You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2016/01/12 20:51:34 UTC
[01/13] accumulo git commit: ACCUMULO-4108 Increase zk timeout from
5s to 15s for ITs.
Repository: accumulo
Updated Branches:
refs/heads/1.6 46ad8368e -> 29c6052d6
refs/heads/1.7 656282825 -> 94f4a19c0
refs/heads/master 18725dd6c -> 081eb1fac
ACCUMULO-4108 Increase zk timeout from 5s to 15s for ITs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/29c6052d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/29c6052d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/29c6052d
Branch: refs/heads/1.6
Commit: 29c6052d64de0705729bf32136e44505e0c788cc
Parents: 46ad836
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 12 12:09:49 2016 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jan 12 12:09:49 2016 -0500
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java | 2 +-
test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 2 +-
.../test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/MasterFailoverIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/RestartIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/RestartStressIT.java | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java b/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
index eff7251..8a0086b 100644
--- a/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
+++ b/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
@@ -48,7 +48,7 @@ public class Accumulo3010IT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// file system supports recovery
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java b/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
index d55e427..323888a 100644
--- a/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
@@ -56,7 +56,7 @@ public class ExistingMacIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
index 72dabdf..f1372dc 100644
--- a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
@@ -59,7 +59,7 @@ public class MasterRepairsDualAssignmentIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "5s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java b/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
index 3b25655..116092b 100644
--- a/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
@@ -45,7 +45,7 @@ public class MultiTableRecoveryIT extends ConfigurableMacIT {
@Override
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
index 7634f10..0c2631f 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
@@ -35,7 +35,7 @@ public class MasterFailoverIT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s"));
+ cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s"));
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index 4e55ab4..b498412 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@ -67,7 +67,7 @@ public class RestartIT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
props.put(Property.GC_CYCLE_DELAY.getKey(), "1s");
props.put(Property.GC_CYCLE_START.getKey(), "1s");
cfg.setSiteConfig(props);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index b965420..c4b3afd 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@ -55,7 +55,7 @@ public class RestartStressIT extends AccumuloClusterIT {
opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
- opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
cfg.setSiteConfig(opts);
cfg.useMiniDFS(true);
[02/13] accumulo git commit: ACCUMULO-4108 Increase zk timeout from
5s to 15s for ITs.
Posted by el...@apache.org.
ACCUMULO-4108 Increase zk timeout from 5s to 15s for ITs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/29c6052d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/29c6052d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/29c6052d
Branch: refs/heads/1.7
Commit: 29c6052d64de0705729bf32136e44505e0c788cc
Parents: 46ad836
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 12 12:09:49 2016 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jan 12 12:09:49 2016 -0500
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java | 2 +-
test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 2 +-
.../test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/MasterFailoverIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/RestartIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/RestartStressIT.java | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java b/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
index eff7251..8a0086b 100644
--- a/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
+++ b/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
@@ -48,7 +48,7 @@ public class Accumulo3010IT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// file system supports recovery
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java b/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
index d55e427..323888a 100644
--- a/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
@@ -56,7 +56,7 @@ public class ExistingMacIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
index 72dabdf..f1372dc 100644
--- a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
@@ -59,7 +59,7 @@ public class MasterRepairsDualAssignmentIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "5s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java b/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
index 3b25655..116092b 100644
--- a/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
@@ -45,7 +45,7 @@ public class MultiTableRecoveryIT extends ConfigurableMacIT {
@Override
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
index 7634f10..0c2631f 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
@@ -35,7 +35,7 @@ public class MasterFailoverIT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s"));
+ cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s"));
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index 4e55ab4..b498412 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@ -67,7 +67,7 @@ public class RestartIT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
props.put(Property.GC_CYCLE_DELAY.getKey(), "1s");
props.put(Property.GC_CYCLE_START.getKey(), "1s");
cfg.setSiteConfig(props);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index b965420..c4b3afd 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@ -55,7 +55,7 @@ public class RestartStressIT extends AccumuloClusterIT {
opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
- opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
cfg.setSiteConfig(opts);
cfg.useMiniDFS(true);
[13/13] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/081eb1fa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/081eb1fa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/081eb1fa
Branch: refs/heads/master
Commit: 081eb1facb7d7402add733b332f5a33152e19e81
Parents: 18725dd 94f4a19
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 12 12:25:40 2016 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jan 12 14:51:20 2016 -0500
----------------------------------------------------------------------
test/src/main/java/org/apache/accumulo/test/CleanWalIT.java | 2 +-
.../java/org/apache/accumulo/test/DetectDeadTabletServersIT.java | 2 +-
test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 2 +-
test/src/main/java/org/apache/accumulo/test/MetaRecoveryIT.java | 2 +-
.../main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java | 2 +-
.../java/org/apache/accumulo/test/TabletServerGivesUpIT.java | 2 +-
.../java/org/apache/accumulo/test/TabletServerHdfsRestartIT.java | 2 +-
.../java/org/apache/accumulo/test/VerifySerialRecoveryIT.java | 2 +-
test/src/main/java/org/apache/accumulo/test/VolumeIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/BinaryStressIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/CleanTmpIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/CompactionIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/DurabilityIT.java | 2 +-
.../org/apache/accumulo/test/functional/GarbageCollectorIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/KerberosIT.java | 2 +-
.../org/apache/accumulo/test/functional/KerberosRenewalIT.java | 2 +-
.../org/apache/accumulo/test/functional/MasterFailoverIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/ReadWriteIT.java | 2 +-
.../main/java/org/apache/accumulo/test/functional/RestartIT.java | 2 +-
.../org/apache/accumulo/test/functional/RestartStressIT.java | 2 +-
.../org/apache/accumulo/test/functional/SessionDurabilityIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/WALSunnyDayIT.java | 2 +-
.../org/apache/accumulo/test/functional/WriteAheadLogIT.java | 2 +-
.../org/apache/accumulo/test/functional/ZookeeperRestartIT.java | 2 +-
.../java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java | 4 ++--
.../replication/GarbageCollectorCommunicatesWithTServersIT.java | 2 +-
.../accumulo/test/replication/MultiInstanceReplicationIT.java | 2 +-
.../java/org/apache/accumulo/test/replication/ReplicationIT.java | 3 +--
.../test/replication/UnorderedWorkAssignerReplicationIT.java | 2 +-
31 files changed, 32 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/CleanWalIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/CleanWalIT.java
index 91b929f,0000000..7146a9f
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/CleanWalIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/CleanWalIT.java
@@@ -1,147 -1,0 +1,147 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class CleanWalIT extends AccumuloClusterHarness {
+ private static final Logger log = LoggerFactory.getLogger(CleanWalIT.class);
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Before
+ public void offlineTraceTable() throws Exception {
+ Connector conn = getConnector();
+ String traceTable = conn.instanceOperations().getSystemConfiguration().get(Property.TRACE_TABLE.getKey());
+ if (conn.tableOperations().exists(traceTable)) {
+ conn.tableOperations().offline(traceTable, true);
+ }
+ }
+
+ @After
+ public void onlineTraceTable() throws Exception {
+ if (null != cluster) {
+ Connector conn = getConnector();
+ String traceTable = conn.instanceOperations().getSystemConfiguration().get(Property.TRACE_TABLE.getKey());
+ if (conn.tableOperations().exists(traceTable)) {
+ conn.tableOperations().online(traceTable, true);
+ }
+ }
+ }
+
+ // test for ACCUMULO-1830
+ @Test
+ public void test() throws Exception {
+ Connector conn = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ conn.tableOperations().create(tableName);
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.close();
+ getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ // all 3 tables should do recovery, but the bug doesn't really remove the log file references
+
+ getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+
+ for (String table : new String[] {MetadataTable.NAME, RootTable.NAME})
+ conn.tableOperations().flush(table, null, null, true);
+ log.debug("Checking entries for " + tableName);
+ assertEquals(1, count(tableName, conn));
+ for (String table : new String[] {MetadataTable.NAME, RootTable.NAME}) {
+ log.debug("Checking logs for " + table);
+ assertEquals("Found logs for " + table, 0, countLogs(table, conn));
+ }
+
+ bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ m = new Mutation("row");
+ m.putDelete("cf", "cq");
+ bw.addMutation(m);
+ bw.close();
+ assertEquals(0, count(tableName, conn));
+ conn.tableOperations().flush(tableName, null, null, true);
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ conn.tableOperations().flush(RootTable.NAME, null, null, true);
+ try {
+ getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ sleepUninterruptibly(3, TimeUnit.SECONDS);
+ } finally {
+ getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+ }
+ assertEquals(0, count(tableName, conn));
+ }
+
+ private int countLogs(String tableName, Connector conn) throws TableNotFoundException {
+ Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.LogColumnFamily.NAME);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ int count = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ log.debug("Saw " + entry.getKey() + "=" + entry.getValue());
+ count++;
+ }
+ return count;
+ }
+
+ int count(String tableName, Connector conn) throws Exception {
+ Scanner s = conn.createScanner(tableName, Authorizations.EMPTY);
+ return Iterators.size(s.iterator());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
index e4e0962,0000000..f207353
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
@@@ -1,97 -1,0 +1,97 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER;
+import static org.junit.Assert.assertEquals;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
+import org.apache.accumulo.core.master.thrift.MasterMonitorInfo;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.trace.Tracer;
+import org.apache.accumulo.fate.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class DetectDeadTabletServersIT extends ConfigurableMacBase {
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ log.info("verifying that everything is up");
+ Iterators.size(c.createScanner(MetadataTable.NAME, Authorizations.EMPTY).iterator());
+
+ MasterMonitorInfo stats = getStats(c);
+ assertEquals(2, stats.tServerInfo.size());
+ assertEquals(0, stats.badTServers.size());
+ assertEquals(0, stats.deadTabletServers.size());
+ log.info("Killing a tablet server");
+ getCluster().killProcess(TABLET_SERVER, getCluster().getProcesses().get(TABLET_SERVER).iterator().next());
+
+ while (true) {
+ stats = getStats(c);
+ if (2 != stats.tServerInfo.size()) {
+ break;
+ }
+ UtilWaitThread.sleep(500);
+ }
+ assertEquals(1, stats.tServerInfo.size());
+ assertEquals(1, stats.badTServers.size() + stats.deadTabletServers.size());
+ while (true) {
+ stats = getStats(c);
+ if (0 != stats.deadTabletServers.size()) {
+ break;
+ }
+ UtilWaitThread.sleep(500);
+ }
+ assertEquals(1, stats.tServerInfo.size());
+ assertEquals(0, stats.badTServers.size());
+ assertEquals(1, stats.deadTabletServers.size());
+ }
+
+ private MasterMonitorInfo getStats(Connector c) throws Exception {
+ Credentials creds = new Credentials("root", new PasswordToken(ROOT_PASSWORD));
+ ClientContext context = new ClientContext(c.getInstance(), creds, getClientConfig());
+ Client client = null;
+ try {
+ client = MasterClient.getConnectionWithRetry(context);
+ log.info("Fetching master stats");
+ return client.getMasterStats(Tracer.traceInfo(), context.rpcCreds());
+ } finally {
+ if (client != null) {
+ MasterClient.close(client);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
index d4f4d58,0000000..7984393
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/ExistingMacIT.java
@@@ -1,171 -1,0 +1,171 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collection;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class ExistingMacIT extends ConfigurableMacBase {
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ private void createEmptyConfig(File confFile) throws IOException {
+ Configuration conf = new Configuration(false);
+ OutputStream hcOut = new FileOutputStream(confFile);
+ conf.writeXml(hcOut);
+ hcOut.close();
+ }
+
+ @Test
+ public void testExistingInstance() throws Exception {
+
+ Connector conn = getCluster().getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ conn.tableOperations().create("table1");
+
+ BatchWriter bw = conn.createBatchWriter("table1", new BatchWriterConfig());
+
+ Mutation m1 = new Mutation("00081");
+ m1.put("math", "sqroot", "9");
+ m1.put("math", "sq", "6560");
+
+ bw.addMutation(m1);
+ bw.close();
+
+ conn.tableOperations().flush("table1", null, null, true);
+ // TOOD use constants
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ conn.tableOperations().flush(RootTable.NAME, null, null, true);
+
+ Set<Entry<ServerType,Collection<ProcessReference>>> procs = getCluster().getProcesses().entrySet();
+ for (Entry<ServerType,Collection<ProcessReference>> entry : procs) {
+ if (entry.getKey() == ServerType.ZOOKEEPER)
+ continue;
+ for (ProcessReference pr : entry.getValue())
+ getCluster().killProcess(entry.getKey(), pr);
+ }
+
+ // TODO clean out zookeeper? following sleep waits for ephemeral nodes to go away
+ sleepUninterruptibly(10, TimeUnit.SECONDS);
+
+ File hadoopConfDir = createTestDir(ExistingMacIT.class.getSimpleName() + "_hadoop_conf");
+ FileUtils.deleteQuietly(hadoopConfDir);
+ assertTrue(hadoopConfDir.mkdirs());
+ createEmptyConfig(new File(hadoopConfDir, "core-site.xml"));
+ createEmptyConfig(new File(hadoopConfDir, "hdfs-site.xml"));
+
+ File testDir2 = createTestDir(ExistingMacIT.class.getSimpleName() + "_2");
+ FileUtils.deleteQuietly(testDir2);
+
+ MiniAccumuloConfigImpl macConfig2 = new MiniAccumuloConfigImpl(testDir2, "notused");
+ macConfig2.useExistingInstance(new File(getCluster().getConfig().getConfDir(), "accumulo-site.xml"), hadoopConfDir);
+
+ MiniAccumuloClusterImpl accumulo2 = new MiniAccumuloClusterImpl(macConfig2);
+ accumulo2.start();
+
+ conn = accumulo2.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ Scanner scanner = conn.createScanner("table1", Authorizations.EMPTY);
+
+ int sum = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ sum += Integer.parseInt(entry.getValue().toString());
+ }
+
+ Assert.assertEquals(6569, sum);
+
+ accumulo2.stop();
+ }
+
+ @Test
+ public void testExistingRunningInstance() throws Exception {
+ final String table = getUniqueNames(1)[0];
+ Connector conn = getConnector();
+ // Ensure that a master and tserver are up so the existing instance check won't fail.
+ conn.tableOperations().create(table);
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("foo");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ File hadoopConfDir = createTestDir(ExistingMacIT.class.getSimpleName() + "_hadoop_conf_2");
+ FileUtils.deleteQuietly(hadoopConfDir);
+ assertTrue(hadoopConfDir.mkdirs());
+ createEmptyConfig(new File(hadoopConfDir, "core-site.xml"));
+ createEmptyConfig(new File(hadoopConfDir, "hdfs-site.xml"));
+
+ File testDir2 = createTestDir(ExistingMacIT.class.getSimpleName() + "_3");
+ FileUtils.deleteQuietly(testDir2);
+
+ MiniAccumuloConfigImpl macConfig2 = new MiniAccumuloConfigImpl(testDir2, "notused");
+ macConfig2.useExistingInstance(new File(getCluster().getConfig().getConfDir(), "accumulo-site.xml"), hadoopConfDir);
+
+ System.out.println("conf " + new File(getCluster().getConfig().getConfDir(), "accumulo-site.xml"));
+
+ MiniAccumuloClusterImpl accumulo2 = new MiniAccumuloClusterImpl(macConfig2);
+ try {
+ accumulo2.start();
+ Assert.fail("A 2nd MAC instance should not be able to start over an existing MAC instance");
+ } catch (RuntimeException e) {
+ // TODO check message or throw more explicit exception
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
index a4f067e,0000000..e54e451
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
@@@ -1,162 -1,0 +1,162 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.fate.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.master.state.ClosableIterator;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.RootTabletStateStore;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class MasterRepairsDualAssignmentIT extends ConfigurableMacBase {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "5s");
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test
+ public void test() throws Exception {
+ // make some tablets, spread 'em around
+ Connector c = getConnector();
+ ClientContext context = new ClientContext(c.getInstance(), new Credentials("root", new PasswordToken(ROOT_PASSWORD)), getClientConfig());
+ String table = this.getUniqueNames(1)[0];
+ c.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+ c.securityOperations().grantTablePermission("root", RootTable.NAME, TablePermission.WRITE);
+ c.tableOperations().create(table);
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ for (String part : "a b c d e f g h i j k l m n o p q r s t u v w x y z".split(" ")) {
+ partitions.add(new Text(part));
+ }
+ c.tableOperations().addSplits(table, partitions);
+ // scan the metadata table and get the two table location states
+ Set<TServerInstance> states = new HashSet<TServerInstance>();
+ Set<TabletLocationState> oldLocations = new HashSet<TabletLocationState>();
+ MetaDataStateStore store = new MetaDataStateStore(context, null);
+ while (states.size() < 2) {
+ UtilWaitThread.sleep(250);
+ oldLocations.clear();
+ for (TabletLocationState tls : store) {
+ if (tls.current != null) {
+ states.add(tls.current);
+ oldLocations.add(tls);
+ }
+ }
+ }
+ assertEquals(2, states.size());
+ // Kill a tablet server... we don't care which one... wait for everything to be reassigned
+ cluster.killProcess(ServerType.TABLET_SERVER, cluster.getProcesses().get(ServerType.TABLET_SERVER).iterator().next());
+ Set<TServerInstance> replStates = new HashSet<>();
+ // Find out which tablet server remains
+ while (true) {
+ UtilWaitThread.sleep(1000);
+ states.clear();
+ replStates.clear();
+ boolean allAssigned = true;
+ for (TabletLocationState tls : store) {
+ if (tls != null && tls.current != null) {
+ states.add(tls.current);
+ } else if (tls != null && tls.extent.equals(new KeyExtent(new Text(ReplicationTable.ID), null, null))) {
+ replStates.add(tls.current);
+ } else {
+ allAssigned = false;
+ }
+ }
+ System.out.println(states + " size " + states.size() + " allAssigned " + allAssigned);
+ if (states.size() != 2 && allAssigned == true)
+ break;
+ }
+ assertEquals(1, replStates.size());
+ assertEquals(1, states.size());
+ // pick an assigned tablet and assign it to the old tablet
+ TabletLocationState moved = null;
+ for (TabletLocationState old : oldLocations) {
+ if (!states.contains(old.current)) {
+ moved = old;
+ }
+ }
+ assertNotEquals(null, moved);
+ // throw a mutation in as if we were the dying tablet
+ BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ Mutation assignment = new Mutation(moved.extent.getMetadataEntry());
+ moved.current.putLocation(assignment);
+ bw.addMutation(assignment);
+ bw.close();
+ // wait for the master to fix the problem
+ waitForCleanStore(store);
+ // now jam up the metadata table
+ bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ assignment = new Mutation(new KeyExtent(new Text(MetadataTable.ID), null, null).getMetadataEntry());
+ moved.current.putLocation(assignment);
+ bw.addMutation(assignment);
+ bw.close();
+ waitForCleanStore(new RootTabletStateStore(context, null));
+ }
+
+ private void waitForCleanStore(MetaDataStateStore store) {
+ while (true) {
+ try (ClosableIterator<TabletLocationState> iter = store.iterator()) {
+ Iterators.size(iter);
+ } catch (Exception ex) {
+ System.out.println(ex);
+ UtilWaitThread.sleep(250);
+ continue;
+ }
+ break;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/MetaRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/MetaRecoveryIT.java
index 9f93381,0000000..0c16a5f
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/MetaRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MetaRecoveryIT.java
@@@ -1,96 -1,0 +1,96 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collections;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+// ACCUMULO-3211
+public class MetaRecoveryIT extends ConfigurableMacBase {
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.TSERV_ARCHIVE_WALOGS, "true");
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1048576");
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void test() throws Exception {
+ String[] tables = getUniqueNames(10);
+ Connector c = getConnector();
+ int i = 0;
+ for (String table : tables) {
+ log.info("Creating table {}", i);
+ c.tableOperations().create(table);
+ BatchWriter bw = c.createBatchWriter(table, null);
+ for (int j = 0; j < 1000; j++) {
+ Mutation m = new Mutation("" + j);
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ }
+ bw.close();
+ log.info("Data written to table {}", i);
+ i++;
+ }
+ c.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ c.tableOperations().flush(RootTable.NAME, null, null, true);
+ SortedSet<Text> splits = new TreeSet<>();
+ for (i = 1; i < tables.length; i++) {
+ splits.add(new Text("" + i));
+ }
+ c.tableOperations().addSplits(MetadataTable.NAME, splits);
+ log.info("Added {} splits to {}", splits.size(), MetadataTable.NAME);
+ c.instanceOperations().waitForBalance();
+ log.info("Restarting");
+ getCluster().getClusterControl().kill(ServerType.TABLET_SERVER, "localhost");
+ getCluster().start();
+ log.info("Verifying");
+ for (String table : tables) {
+ BatchScanner scanner = c.createBatchScanner(table, Authorizations.EMPTY, 5);
+ scanner.setRanges(Collections.singletonList(new Range()));
+ assertEquals(1000, Iterators.size(scanner.iterator()));
+ scanner.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
index d584613,0000000..e62b5ad
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
@@@ -1,135 -1,0 +1,135 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class MultiTableRecoveryIT extends ConfigurableMacBase {
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testRecoveryOverMultipleTables() throws Exception {
+ final int N = 3;
+ final Connector c = getConnector();
+ final String[] tables = getUniqueNames(N);
+ final BatchWriter[] writers = new BatchWriter[N];
+ final byte[][] values = new byte[N][];
+ int i = 0;
+ System.out.println("Creating tables");
+ for (String tableName : tables) {
+ c.tableOperations().create(tableName);
+ values[i] = Integer.toString(i).getBytes();
+ writers[i] = c.createBatchWriter(tableName, null);
+ i++;
+ }
+ System.out.println("Creating agitator");
+ final AtomicBoolean stop = new AtomicBoolean(false);
+ final Thread agitator = agitator(stop);
+ agitator.start();
+ System.out.println("writing");
+ final Random random = new Random();
+ for (i = 0; i < 1_000_000; i++) {
+ // make non-negative avoiding Math.abs, because that can still be negative
+ long randomRow = random.nextLong() & Long.MAX_VALUE;
+ assertTrue(randomRow >= 0);
+ final int table = (int) (randomRow % N);
+ final Mutation m = new Mutation(Long.toHexString(randomRow));
+ m.put(new byte[0], new byte[0], values[table]);
+ writers[table].addMutation(m);
+ if (i % 10_000 == 0) {
+ System.out.println("flushing");
+ for (int w = 0; w < N; w++) {
+ writers[w].flush();
+ }
+ }
+ }
+ System.out.println("closing");
+ for (int w = 0; w < N; w++) {
+ writers[w].close();
+ }
+ System.out.println("stopping the agitator");
+ stop.set(true);
+ agitator.join();
+ System.out.println("checking the data");
+ long count = 0;
+ for (int w = 0; w < N; w++) {
+ Scanner scanner = c.createScanner(tables[w], Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : scanner) {
+ int value = Integer.parseInt(entry.getValue().toString());
+ assertEquals(w, value);
+ count++;
+ }
+ scanner.close();
+ }
+ assertEquals(1_000_000, count);
+ }
+
+ private Thread agitator(final AtomicBoolean stop) {
+ return new Thread() {
+ @Override
+ public void run() {
+ try {
+ int i = 0;
+ while (!stop.get()) {
+ sleepUninterruptibly(10, TimeUnit.SECONDS);
+ System.out.println("Restarting");
+ getCluster().getClusterControl().stop(ServerType.TABLET_SERVER);
+ getCluster().start();
+ // read the metadata table to know everything is back up
+ Iterators.size(getConnector().createScanner(MetadataTable.NAME, Authorizations.EMPTY).iterator());
+ i++;
+ }
+ System.out.println("Restarted " + i + " times");
+ } catch (Exception ex) {
+ log.error("{}", ex.getMessage(), ex);
+ }
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
index 60b3cf7,0000000..f79e174
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
@@@ -1,101 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+// Accumulo3010
+public class RecoveryCompactionsAreFlushesIT extends AccumuloClusterHarness {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ // file system supports recovery
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test
+ public void test() throws Exception {
+ // create a table
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "100");
+ c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "3");
+ // create 3 flush files
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", new Value("v".getBytes()));
+ for (int i = 0; i < 3; i++) {
+ bw.addMutation(m);
+ bw.flush();
+ c.tableOperations().flush(tableName, null, null, true);
+ }
+ // create an unsaved mutation
+ bw.addMutation(m);
+ bw.close();
+
+ ClusterControl control = cluster.getClusterControl();
+
+ // kill the tablet servers
+ control.stopAllServers(ServerType.TABLET_SERVER);
+
+ // recover
+ control.startAllServers(ServerType.TABLET_SERVER);
+
+ // ensure the table is readable
+ Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator());
+
+ // ensure that the recovery was not a merging minor compaction
+ Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : s) {
+ String filename = entry.getKey().getColumnQualifier().toString();
+ String parts[] = filename.split("/");
+ Assert.assertFalse(parts[parts.length - 1].startsWith("M"));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
index f7b11f6,0000000..33c1798
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
@@@ -1,77 -1,0 +1,77 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+// ACCUMULO-2480
+public class TabletServerGivesUpIT extends ConfigurableMacBase {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.useMiniDFS(true);
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_TOLERATED_CREATION_FAILURES, "15");
+ cfg.setProperty(Property.TSERV_WALOG_TOLERATED_MAXIMUM_WAIT_DURATION, "0s");
+ }
+
+ @Test(timeout = 30 * 1000)
+ public void test() throws Exception {
+ final Connector conn = this.getConnector();
+ // Yes, there's a tabletserver
+ assertEquals(1, conn.instanceOperations().getTabletServers().size());
+ final String tableName = getUniqueNames(1)[0];
+ conn.tableOperations().create(tableName);
+ // Kill dfs
+ cluster.getMiniDfs().shutdown();
+ // ask the tserver to do something
+ final AtomicReference<Exception> ex = new AtomicReference<>();
+ Thread splitter = new Thread() {
+ @Override
+ public void run() {
+ try {
+ TreeSet<Text> splits = new TreeSet<>();
+ splits.add(new Text("X"));
+ conn.tableOperations().addSplits(tableName, splits);
+ } catch (Exception e) {
+ ex.set(e);
+ }
+ }
+ };
+ splitter.start();
+ // wait for the tserver to give up on writing to the WAL
+ while (conn.instanceOperations().getTabletServers().size() == 1) {
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/TabletServerHdfsRestartIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/TabletServerHdfsRestartIT.java
index e92c1c5,0000000..1e063f0
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/TabletServerHdfsRestartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/TabletServerHdfsRestartIT.java
@@@ -1,67 -1,0 +1,67 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+// ACCUMULO-3914
+public class TabletServerHdfsRestartIT extends ConfigurableMacBase {
+
+ private static final int N = 1000;
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.useMiniDFS(true);
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ }
+
+ @Test(timeout = 2 * 60 * 1000)
+ public void test() throws Exception {
+ final Connector conn = this.getConnector();
+ // Yes, there's a tabletserver
+ assertEquals(1, conn.instanceOperations().getTabletServers().size());
+ final String tableName = getUniqueNames(1)[0];
+ conn.tableOperations().create(tableName);
+ BatchWriter bw = conn.createBatchWriter(tableName, null);
+ for (int i = 0; i < N; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ conn.tableOperations().flush(tableName, null, null, true);
+
+ // Kill dfs
+ cluster.getMiniDfs().restartNameNode(false);
+
+ assertEquals(N, Iterators.size(conn.createScanner(tableName, Authorizations.EMPTY).iterator()));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
index 6a90730,0000000..1204ee0
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
@@@ -1,107 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class VerifySerialRecoveryIT extends ConfigurableMacBase {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "20");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testSerializedRecovery() throws Exception {
+ // make a table with many splits
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 200; i++) {
+ splits.add(new Text(AssignmentThreadsIT.randomHex(8)));
+ }
+ c.tableOperations().addSplits(tableName, splits);
+ // load data to give the recovery something to do
+ BatchWriter bw = c.createBatchWriter(tableName, null);
+ for (int i = 0; i < 50000; i++) {
+ Mutation m = new Mutation(AssignmentThreadsIT.randomHex(8));
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ // kill the tserver
+ for (ProcessReference ref : getCluster().getProcesses().get(ServerType.TABLET_SERVER))
+ getCluster().killProcess(ServerType.TABLET_SERVER, ref);
+ final Process ts = cluster.exec(TabletServer.class);
+
+ // wait for recovery
+ Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator());
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ ts.waitFor();
+ String result = FunctionalTestUtils.readAll(cluster, TabletServer.class, ts);
+ for (String line : result.split("\n")) {
+ System.out.println(line);
+ }
+ // walk through the output, verifying that only a single normal recovery was running at one time
+ boolean started = false;
+ int recoveries = 0;
+ for (String line : result.split("\n")) {
+ // ignore metadata tables
+ if (line.contains("!0") || line.contains("+r"))
+ continue;
+ if (line.contains("Starting Write-Ahead Log")) {
+ assertFalse(started);
+ started = true;
+ recoveries++;
+ }
+ if (line.contains("Write-Ahead Log recovery complete")) {
+ assertTrue(started);
+ started = false;
+ }
+ }
+ assertFalse(started);
+ assertTrue(recoveries > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/VolumeIT.java
index b325359,0000000..0a06fdf
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/VolumeIT.java
@@@ -1,568 -1,0 +1,568 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.DiskUsage;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.CachedConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.ServerConstants;
+import org.apache.accumulo.server.init.Initialize;
+import org.apache.accumulo.server.log.WalStateManager;
+import org.apache.accumulo.server.log.WalStateManager.WalState;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class VolumeIT extends ConfigurableMacBase {
+
+ private static final Text EMPTY = new Text();
+ private static final Value EMPTY_VALUE = new Value(new byte[] {});
+ private File volDirBase;
+ private Path v1, v2;
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ File baseDir = cfg.getDir();
+ volDirBase = new File(baseDir, "volumes");
+ File v1f = new File(volDirBase, "v1");
+ File v2f = new File(volDirBase, "v2");
+ v1 = new Path("file://" + v1f.getAbsolutePath());
+ v2 = new Path("file://" + v2f.getAbsolutePath());
+
+ // Run MAC on two locations in the local file system
+ URI v1Uri = v1.toUri();
+ cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath());
+ cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost());
+ cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+
+ super.configure(cfg, hadoopCoreSite);
+ }
+
+ @Test
+ public void test() throws Exception {
+ // create a table
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName);
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ // with some splits
+ for (String s : "d,m,t".split(","))
+ partitions.add(new Text(s));
+ connector.tableOperations().addSplits(tableName, partitions);
+ // scribble over the splits
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+ String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ for (String s : rows) {
+ Mutation m = new Mutation(new Text(s));
+ m.put(EMPTY, EMPTY, EMPTY_VALUE);
+ bw.addMutation(m);
+ }
+ bw.close();
+ // write the data to disk, read it back
+ connector.tableOperations().flush(tableName, null, null, true);
+ Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
+ int i = 0;
+ for (Entry<Key,Value> entry : scanner) {
+ assertEquals(rows[i++], entry.getKey().getRow().toString());
+ }
+ // verify the new files are written to the different volumes
+ scanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(new Range("1", "1<"));
+ scanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ int fileCount = 0;
+
+ for (Entry<Key,Value> entry : scanner) {
+ boolean inV1 = entry.getKey().getColumnQualifier().toString().contains(v1.toString());
+ boolean inV2 = entry.getKey().getColumnQualifier().toString().contains(v2.toString());
+ assertTrue(inV1 || inV2);
+ fileCount++;
+ }
+ assertEquals(4, fileCount);
+ List<DiskUsage> diskUsage = connector.tableOperations().getDiskUsage(Collections.singleton(tableName));
+ assertEquals(1, diskUsage.size());
+ long usage = diskUsage.get(0).getUsage().longValue();
+ log.debug("usage {}", usage);
+ assertTrue(usage > 700 && usage < 800);
+ }
+
+ private void verifyData(List<String> expected, Scanner createScanner) {
+
+ List<String> actual = new ArrayList<String>();
+
+ for (Entry<Key,Value> entry : createScanner) {
+ Key k = entry.getKey();
+ actual.add(k.getRow() + ":" + k.getColumnFamily() + ":" + k.getColumnQualifier() + ":" + entry.getValue());
+ }
+
+ Collections.sort(expected);
+ Collections.sort(actual);
+
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void testRelativePaths() throws Exception {
+
+ List<String> expected = new ArrayList<String>();
+
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName, new NewTableConfiguration().withoutDefaultIterators());
+
+ String tableId = connector.tableOperations().tableIdMap().get(tableName);
+
+ SortedSet<Text> partitions = new TreeSet<Text>();
+ // with some splits
+ for (String s : "c,g,k,p,s,v".split(","))
+ partitions.add(new Text(s));
+
+ connector.tableOperations().addSplits(tableName, partitions);
+
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+
+ // create two files in each tablet
+
+ String[] rows = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",");
+ for (String s : rows) {
+ Mutation m = new Mutation(s);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ expected.add(s + ":cf1:cq1:1");
+ }
+
+ bw.flush();
+ connector.tableOperations().flush(tableName, null, null, true);
+
+ for (String s : rows) {
+ Mutation m = new Mutation(s);
+ m.put("cf1", "cq1", "2");
+ bw.addMutation(m);
+ expected.add(s + ":cf1:cq1:2");
+ }
+
+ bw.close();
+ connector.tableOperations().flush(tableName, null, null, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ connector.tableOperations().offline(tableName, true);
+
+ connector.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+
+ Scanner metaScanner = connector.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ BatchWriter mbw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ String cq = entry.getKey().getColumnQualifier().toString();
+ if (cq.startsWith(v1.toString())) {
+ Path path = new Path(cq);
+ String relPath = "/" + path.getParent().getName() + "/" + path.getName();
+ Mutation fileMut = new Mutation(entry.getKey().getRow());
+ fileMut.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier());
+ fileMut.put(entry.getKey().getColumnFamily().toString(), relPath, entry.getValue().toString());
+ mbw.addMutation(fileMut);
+ }
+ }
+
+ mbw.close();
+
+ connector.tableOperations().online(tableName, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ connector.tableOperations().compact(tableName, null, null, true, true);
+
+ verifyData(expected, connector.createScanner(tableName, Authorizations.EMPTY));
+
+ for (Entry<Key,Value> entry : metaScanner) {
+ String cq = entry.getKey().getColumnQualifier().toString();
+ Path path = new Path(cq);
+ Assert.assertTrue("relative path not deleted " + path.toString(), path.depth() > 2);
+ }
+
+ }
+
+ @Test
+ public void testAddVolumes() throws Exception {
+
+ String[] tableNames = getUniqueNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getClientConfig()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ assertTrue(v3f.mkdir() || v3f.isDirectory());
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v1.toString() + "," + v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ verifyVolumesUsed(tableNames[1], false, v1, v2, v3);
+ }
+
+ @Test
+ public void testNonConfiguredVolumes() throws Exception {
+
+ String[] tableNames = getUniqueNames(2);
+
+ // grab this before shutting down cluster
+ String uuid = new ZooKeeperInstance(cluster.getClientConfig()).getInstanceID();
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ File v3f = new File(volDirBase, "v3");
+ assertTrue(v3f.mkdir() || v3f.isDirectory());
+ Path v3 = new Path("file://" + v3f.getAbsolutePath());
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString() + "," + v3.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // initialize volume
+ Assert.assertEquals(0, cluster.exec(Initialize.class, "--add-volumes").waitFor());
+
+ // check that all volumes are initialized
+ for (Path volumePath : Arrays.asList(v1, v2, v3)) {
+ FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance());
+ Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR);
+ FileStatus[] iids = fs.listStatus(vp);
+ Assert.assertEquals(1, iids.length);
+ Assert.assertEquals(uuid, iids[0].getPath().getName());
+ }
+
+ // start cluster and verify that new volume is used
+ cluster.start();
+
+ // Make sure we can still read the tables (tableNames[0] is very likely to have a file still on v1)
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ verifyData(expected, getConnector().createScanner(tableNames[0], Authorizations.EMPTY));
+
+ // v1 should not have any data for tableNames[1]
+ verifyVolumesUsed(tableNames[1], false, v2, v3);
+ }
+
+ private void writeData(String tableName, Connector conn) throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException,
+ MutationsRejectedException {
+ TreeSet<Text> splits = new TreeSet<Text>();
+ for (int i = 1; i < 100; i++) {
+ splits.add(new Text(String.format("%06d", i * 100)));
+ }
+
+ conn.tableOperations().create(tableName);
+ conn.tableOperations().addSplits(tableName, splits);
+
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ Mutation m = new Mutation(row);
+ m.put("cf1", "cq1", "1");
+ bw.addMutation(m);
+ }
+
+ bw.close();
+ }
+
+ private void verifyVolumesUsed(String tableName, boolean shouldExist, Path... paths) throws Exception {
+
+ Connector conn = getConnector();
+
+ List<String> expected = new ArrayList<String>();
+ for (int i = 0; i < 100; i++) {
+ String row = String.format("%06d", i * 100 + 3);
+ expected.add(row + ":cf1:cq1:1");
+ }
+
+ if (!conn.tableOperations().exists(tableName)) {
+ Assert.assertFalse(shouldExist);
+
+ writeData(tableName, conn);
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ conn.tableOperations().flush(tableName, null, null, true);
+ }
+
+ verifyData(expected, conn.createScanner(tableName, Authorizations.EMPTY));
+
+ String tableId = conn.tableOperations().tableIdMap().get(tableName);
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN.fetch(metaScanner);
+ metaScanner.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ metaScanner.setRange(new KeyExtent(new Text(tableId), null, null).toMetadataRange());
+
+ int counts[] = new int[paths.length];
+
+ outer: for (Entry<Key,Value> entry : metaScanner) {
+ String cf = entry.getKey().getColumnFamily().toString();
+ String cq = entry.getKey().getColumnQualifier().toString();
+
+ String path;
+ if (cf.equals(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME.toString()))
+ path = cq;
+ else
+ path = entry.getValue().toString();
+
+ for (int i = 0; i < paths.length; i++) {
+ if (path.startsWith(paths[i].toString())) {
+ counts[i]++;
+ continue outer;
+ }
+ }
+
+ Assert.fail("Unexpected volume " + path);
+ }
+
+ Instance i = conn.getInstance();
+ ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
+ WalStateManager wals = new WalStateManager(i, zk);
+ outer: for (Entry<Path,WalState> entry : wals.getAllState().entrySet()) {
+ for (Path path : paths) {
+ if (entry.getKey().toString().startsWith(path.toString())) {
+ continue outer;
+ }
+ }
+ Assert.fail("Unexpected volume " + entry.getKey());
+ }
+
+ // if a volume is chosen randomly for each tablet, then the probability that a volume will not be chosen for any tablet is ((num_volumes -
+ // 1)/num_volumes)^num_tablets. For 100 tablets and 3 volumes the probability that only 2 volumes would be chosen is 2.46e-18
+
+ int sum = 0;
+ for (int count : counts) {
+ Assert.assertTrue(count > 0);
+ sum += count;
+ }
+
+ Assert.assertEquals(200, sum);
+
+ }
+
+ @Test
+ public void testRemoveVolumes() throws Exception {
+ String[] tableNames = getUniqueNames(2);
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ cluster.stop();
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v2.toString());
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volume was decommisioned
+ cluster.start();
+
+ Connector conn = cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+ conn.tableOperations().compact(tableNames[0], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], true, v2);
+
+ // check that root tablet is not on volume 1
+ ZooReader zreader = new ZooReader(cluster.getZooKeepers(), 30000);
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getClientConfig())) + RootTable.ZROOT_TABLET_PATH;
+ String rootTabletDir = new String(zreader.getData(zpath, false, null), UTF_8);
+ Assert.assertTrue(rootTabletDir.startsWith(v2.toString()));
+
+ conn.tableOperations().clone(tableNames[0], tableNames[1], true, new HashMap<String,String>(), new HashSet<String>());
+
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+ conn.tableOperations().flush(RootTable.NAME, null, null, true);
+
+ verifyVolumesUsed(tableNames[0], true, v2);
+ verifyVolumesUsed(tableNames[1], true, v2);
+
+ }
+
+ private void testReplaceVolume(boolean cleanShutdown) throws Exception {
+ String[] tableNames = getUniqueNames(3);
+
+ verifyVolumesUsed(tableNames[0], false, v1, v2);
+
+ // write to 2nd table, but do not flush data to disk before shutdown
+ writeData(tableNames[1], cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD)));
+
+ if (cleanShutdown)
+ Assert.assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+
+ cluster.stop();
+
+ File v1f = new File(v1.toUri());
+ File v8f = new File(new File(v1.getParent().toUri()), "v8");
+ Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f));
+ Path v8 = new Path(v8f.toURI());
+
+ File v2f = new File(v2.toUri());
+ File v9f = new File(new File(v2.getParent().toUri()), "v9");
+ Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f));
+ Path v9 = new Path(v9f.toURI());
+
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(cluster.getConfig().getConfDir().toURI().toString(), "accumulo-site.xml"));
+
+ conf.set(Property.INSTANCE_VOLUMES.getKey(), v8 + "," + v9);
+ conf.set(Property.INSTANCE_VOLUMES_REPLACEMENTS.getKey(), v1 + " " + v8 + "," + v2 + " " + v9);
+ BufferedOutputStream fos = new BufferedOutputStream(new FileOutputStream(new File(cluster.getConfig().getConfDir(), "accumulo-site.xml")));
+ conf.writeXml(fos);
+ fos.close();
+
+ // start cluster and verify that volumes were replaced
+ cluster.start();
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+
+ // verify writes to new dir
+ getConnector().tableOperations().compact(tableNames[0], null, null, true, true);
+ getConnector().tableOperations().compact(tableNames[1], null, null, true, true);
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+
+ // check that root tablet is not on volume 1 or 2
+ ZooReader zreader = new ZooReader(cluster.getZooKeepers(), 30000);
+ String zpath = ZooUtil.getRoot(new ZooKeeperInstance(cluster.getClientConfig())) + RootTable.ZROOT_TABLET_PATH;
+ String rootTabletDir = new String(zreader.getData(zpath, false, null), UTF_8);
+ Assert.assertTrue(rootTabletDir.startsWith(v8.toString()) || rootTabletDir.startsWith(v9.toString()));
+
+ getConnector().tableOperations().clone(tableNames[1], tableNames[2], true, new HashMap<String,String>(), new HashSet<String>());
+
+ getConnector().tableOperations().flush(MetadataTable.NAME, null, null, true);
+ getConnector().tableOperations().flush(RootTable.NAME, null, null, true);
+
+ verifyVolumesUsed(tableNames[0], true, v8, v9);
+ verifyVolumesUsed(tableNames[1], true, v8, v9);
+ verifyVolumesUsed(tableNames[2], true, v8, v9);
+ }
+
+ @Test
+ public void testCleanReplaceVolumes() throws Exception {
+ testReplaceVolume(true);
+ }
+
+ @Test
+ public void testDirtyReplaceVolumes() throws Exception {
+ testReplaceVolume(false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
index 440d2cf,0000000..9ce221a
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@@ -1,107 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class BinaryStressIT extends AccumuloClusterHarness {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_MAXMEM, "50K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
+ }
+
+ private String majcDelay, maxMem;
+
+ @Before
+ public void alterConfig() throws Exception {
+ if (ClusterType.MINI == getClusterType()) {
+ return;
+ }
+
+ InstanceOperations iops = getConnector().instanceOperations();
+ Map<String,String> conf = iops.getSystemConfiguration();
+ majcDelay = conf.get(Property.TSERV_MAJC_DELAY.getKey());
+ maxMem = conf.get(Property.TSERV_MAXMEM.getKey());
+
+ iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "0");
+ iops.setProperty(Property.TSERV_MAXMEM.getKey(), "50K");
+
+ getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+ }
+
+ @After
+ public void resetConfig() throws Exception {
+ if (null != majcDelay) {
+ InstanceOperations iops = getConnector().instanceOperations();
+ iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), majcDelay);
+ iops.setProperty(Property.TSERV_MAXMEM.getKey(), maxMem);
+
+ getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+ }
+ }
+
+ @Test
+ public void binaryStressTest() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+ BinaryIT.runTest(c, tableName);
+ String id = c.tableOperations().tableIdMap().get(tableName);
+ Set<Text> tablets = new HashSet<>();
+ Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(Range.prefix(id));
+ for (Entry<Key,Value> entry : s) {
+ tablets.add(entry.getKey().getRow());
+ }
+ assertTrue("Expected at least 8 tablets, saw " + tablets.size(), tablets.size() > 7);
+ }
+
+}
[11/13] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
index 3098251,0000000..7b37a9e
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ReadWriteIT.java
@@@ -1,510 -1,0 +1,510 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URL;
+import java.security.SecureRandom;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.cluster.standalone.StandaloneAccumuloCluster;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.file.rfile.PrintInfo;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.MonitorUtil;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.TestMultiTableIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Iterators;
+
+public class ReadWriteIT extends AccumuloClusterHarness {
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ }
+
+ private static final Logger log = LoggerFactory.getLogger(ReadWriteIT.class);
+
+ static final int ROWS = 100000;
+ static final int COLS = 1;
+ static final String COLF = "colf";
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 6 * 60;
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void invalidInstanceName() throws Exception {
+ final Connector conn = getConnector();
+ new ZooKeeperInstance("fake_instance_name", conn.getInstance().getZooKeepers());
+ }
+
+ @Test
+ public void sunnyDay() throws Exception {
+ // Start accumulo, create a table, insert some data, verify we can read it out.
+ // Shutdown cleanly.
+ log.debug("Starting Monitor");
+ cluster.getClusterControl().startAllServers(ServerType.MONITOR);
+ Connector connector = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS, COLS, 50, 0, tableName);
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS, COLS, 50, 0, tableName);
+ String monitorLocation = null;
+ while (null == monitorLocation) {
+ monitorLocation = MonitorUtil.getLocation(getConnector().getInstance());
+ if (null == monitorLocation) {
+ log.debug("Could not fetch monitor HTTP address from zookeeper");
+ Thread.sleep(2000);
+ }
+ }
+ String scheme = "http://";
+ if (getCluster() instanceof StandaloneAccumuloCluster) {
+ StandaloneAccumuloCluster standaloneCluster = (StandaloneAccumuloCluster) getCluster();
+ File accumuloSite = new File(standaloneCluster.getServerAccumuloConfDir(), "accumulo-site.xml");
+ if (accumuloSite.isFile()) {
+ Configuration conf = new Configuration(false);
+ conf.addResource(new Path(accumuloSite.toURI()));
+ String monitorSslKeystore = conf.get(Property.MONITOR_SSL_KEYSTORE.getKey());
+ if (null != monitorSslKeystore) {
+ log.info("Setting scheme to HTTPS since monitor ssl keystore configuration was observed in {}", accumuloSite);
+ scheme = "https://";
+ SSLContext ctx = SSLContext.getInstance("SSL");
+ TrustManager[] tm = new TrustManager[] {new TestTrustManager()};
+ ctx.init(new KeyManager[0], tm, new SecureRandom());
+ SSLContext.setDefault(ctx);
+ HttpsURLConnection.setDefaultSSLSocketFactory(ctx.getSocketFactory());
+ HttpsURLConnection.setDefaultHostnameVerifier(new TestHostnameVerifier());
+ }
+ } else {
+ log.info("{} is not a normal file, not checking for monitor running with SSL", accumuloSite);
+ }
+ }
+ URL url = new URL(scheme + monitorLocation);
+ log.debug("Fetching web page " + url);
+ String result = FunctionalTestUtils.readAll(url.openStream());
+ assertTrue(result.length() > 100);
+ log.debug("Stopping accumulo cluster");
+ ClusterControl control = cluster.getClusterControl();
+ control.adminStopAll();
+ ZooReader zreader = new ZooReader(connector.getInstance().getZooKeepers(), connector.getInstance().getZooKeepersSessionTimeOut());
+ ZooCache zcache = new ZooCache(zreader, null);
+ byte[] masterLockData;
+ do {
+ masterLockData = ZooLock.getLockData(zcache, ZooUtil.getRoot(connector.getInstance()) + Constants.ZMASTER_LOCK, null);
+ if (null != masterLockData) {
+ log.info("Master lock is still held");
+ Thread.sleep(1000);
+ }
+ } while (null != masterLockData);
+
+ control.stopAllServers(ServerType.GARBAGE_COLLECTOR);
+ control.stopAllServers(ServerType.MONITOR);
+ control.stopAllServers(ServerType.TRACER);
+ log.debug("success!");
+ // Restarting everything
+ cluster.start();
+ }
+
+ public static void ingest(Connector connector, ClientConfiguration clientConfig, String principal, int rows, int cols, int width, int offset, String tableName)
+ throws Exception {
+ ingest(connector, clientConfig, principal, rows, cols, width, offset, COLF, tableName);
+ }
+
+ public static void ingest(Connector connector, ClientConfiguration clientConfig, String principal, int rows, int cols, int width, int offset, String colf,
+ String tableName) throws Exception {
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.rows = rows;
+ opts.cols = cols;
+ opts.dataSize = width;
+ opts.startRow = offset;
+ opts.columnFamily = colf;
+ opts.createTable = true;
+ opts.setTableName(tableName);
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConfig);
+ } else {
+ opts.setPrincipal(principal);
+ }
+
+ TestIngest.ingest(connector, opts, new BatchWriterOpts());
+ }
+
+ public static void verify(Connector connector, ClientConfiguration clientConfig, String principal, int rows, int cols, int width, int offset, String tableName)
+ throws Exception {
+ verify(connector, clientConfig, principal, rows, cols, width, offset, COLF, tableName);
+ }
+
+ private static void verify(Connector connector, ClientConfiguration clientConfig, String principal, int rows, int cols, int width, int offset, String colf,
+ String tableName) throws Exception {
+ ScannerOpts scannerOpts = new ScannerOpts();
+ VerifyIngest.Opts opts = new VerifyIngest.Opts();
+ opts.rows = rows;
+ opts.cols = cols;
+ opts.dataSize = width;
+ opts.startRow = offset;
+ opts.columnFamily = colf;
+ opts.setTableName(tableName);
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConfig);
+ } else {
+ opts.setPrincipal(principal);
+ }
+
+ VerifyIngest.verifyIngest(connector, opts, scannerOpts);
+ }
+
+ public static String[] args(String... args) {
+ return args;
+ }
+
+ @Test
+ public void multiTableTest() throws Exception {
+ // Write to multiple tables
+ final String instance = cluster.getInstanceName();
+ final String keepers = cluster.getZooKeepers();
+ final ClusterControl control = cluster.getClusterControl();
+ final String prefix = getClass().getSimpleName() + "_" + testName.getMethodName();
+ ExecutorService svc = Executors.newFixedThreadPool(2);
+ Future<Integer> p1 = svc.submit(new Callable<Integer>() {
+ @Override
+ public Integer call() {
+ try {
+ ClientConfiguration clientConf = cluster.getClientConfig();
+ // Invocation is different for SASL. We're only logged in via this processes memory (not via some credentials cache on disk)
+ // Need to pass along the keytab because of that.
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ String principal = getAdminPrincipal();
+ AuthenticationToken token = getAdminToken();
+ assertTrue("Expected KerberosToken, but was " + token.getClass(), token instanceof KerberosToken);
+ KerberosToken kt = (KerberosToken) token;
+ assertNotNull("Expected keytab in token", kt.getKeytab());
+ return control.exec(
+ TestMultiTableIngest.class,
+ args("--count", Integer.toString(ROWS), "-i", instance, "-z", keepers, "--tablePrefix", prefix, "--keytab", kt.getKeytab().getAbsolutePath(),
+ "-u", principal));
+ }
+
+ return control.exec(
+ TestMultiTableIngest.class,
+ args("--count", Integer.toString(ROWS), "-u", getAdminPrincipal(), "-i", instance, "-z", keepers, "-p", new String(
+ ((PasswordToken) getAdminToken()).getPassword(), Charsets.UTF_8), "--tablePrefix", prefix));
+ } catch (IOException e) {
+ log.error("Error running MultiTableIngest", e);
+ return -1;
+ }
+ }
+ });
+ Future<Integer> p2 = svc.submit(new Callable<Integer>() {
+ @Override
+ public Integer call() {
+ try {
+ ClientConfiguration clientConf = cluster.getClientConfig();
+ // Invocation is different for SASL. We're only logged in via this processes memory (not via some credentials cache on disk)
+ // Need to pass along the keytab because of that.
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ String principal = getAdminPrincipal();
+ AuthenticationToken token = getAdminToken();
+ assertTrue("Expected KerberosToken, but was " + token.getClass(), token instanceof KerberosToken);
+ KerberosToken kt = (KerberosToken) token;
+ assertNotNull("Expected keytab in token", kt.getKeytab());
+ return control.exec(
+ TestMultiTableIngest.class,
+ args("--count", Integer.toString(ROWS), "--readonly", "-i", instance, "-z", keepers, "--tablePrefix", prefix, "--keytab", kt.getKeytab()
+ .getAbsolutePath(), "-u", principal));
+ }
+
+ return control.exec(
+ TestMultiTableIngest.class,
+ args("--count", Integer.toString(ROWS), "--readonly", "-u", getAdminPrincipal(), "-i", instance, "-z", keepers, "-p", new String(
+ ((PasswordToken) getAdminToken()).getPassword(), Charsets.UTF_8), "--tablePrefix", prefix));
+ } catch (IOException e) {
+ log.error("Error running MultiTableIngest", e);
+ return -1;
+ }
+ }
+ });
+ svc.shutdown();
+ while (!svc.isTerminated()) {
+ svc.awaitTermination(15, TimeUnit.SECONDS);
+ }
+ assertEquals(0, p1.get().intValue());
+ assertEquals(0, p2.get().intValue());
+ }
+
+ @Test
+ public void largeTest() throws Exception {
+ // write a few large values
+ Connector connector = getConnector();
+ String table = getUniqueNames(1)[0];
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), 2, 1, 500000, 0, table);
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), 2, 1, 500000, 0, table);
+ }
+
+ @Test
+ public void interleaved() throws Exception {
+ // read and write concurrently
+ final Connector connector = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ interleaveTest(connector, tableName);
+ }
+
+ static void interleaveTest(final Connector connector, final String tableName) throws Exception {
+ final AtomicBoolean fail = new AtomicBoolean(false);
+ final int CHUNKSIZE = ROWS / 10;
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), CHUNKSIZE, 1, 50, 0, tableName);
+ int i;
+ for (i = 0; i < ROWS; i += CHUNKSIZE) {
+ final int start = i;
+ Thread verify = new Thread() {
+ @Override
+ public void run() {
+ try {
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), CHUNKSIZE, 1, 50, start, tableName);
+ } catch (Exception ex) {
+ fail.set(true);
+ }
+ }
+ };
+ verify.start();
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), CHUNKSIZE, 1, 50, i + CHUNKSIZE, tableName);
+ verify.join();
+ assertFalse(fail.get());
+ }
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), CHUNKSIZE, 1, 50, i, tableName);
+ }
+
+ public static Text t(String s) {
+ return new Text(s);
+ }
+
+ public static Mutation m(String row, String cf, String cq, String value) {
+ Mutation m = new Mutation(t(row));
+ m.put(t(cf), t(cq), new Value(value.getBytes()));
+ return m;
+ }
+
+ @Test
+ public void localityGroupPerf() throws Exception {
+ // verify that locality groups can make look-ups faster
+ final Connector connector = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName);
+ connector.tableOperations().setProperty(tableName, "table.group.g1", "colf");
+ connector.tableOperations().setProperty(tableName, "table.groups.enabled", "g1");
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), 2000, 1, 50, 0, tableName);
+ connector.tableOperations().compact(tableName, null, null, true, true);
+ BatchWriter bw = connector.createBatchWriter(tableName, new BatchWriterConfig());
+ bw.addMutation(m("zzzzzzzzzzz", "colf2", "cq", "value"));
+ bw.close();
+ long now = System.currentTimeMillis();
+ Scanner scanner = connector.createScanner(tableName, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(new Text("colf"));
+ Iterators.size(scanner.iterator());
+ long diff = System.currentTimeMillis() - now;
+ now = System.currentTimeMillis();
+ scanner = connector.createScanner(tableName, Authorizations.EMPTY);
+ scanner.fetchColumnFamily(new Text("colf2"));
+ Iterators.size(scanner.iterator());
+ bw.close();
+ long diff2 = System.currentTimeMillis() - now;
+ assertTrue(diff2 < diff);
+ }
+
+ @Test
+ public void sunnyLG() throws Exception {
+ // create a locality group, write to it and ensure it exists in the RFiles that result
+ final Connector connector = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ connector.tableOperations().create(tableName);
+ Map<String,Set<Text>> groups = new TreeMap<String,Set<Text>>();
+ groups.put("g1", Collections.singleton(t("colf")));
+ connector.tableOperations().setLocalityGroups(tableName, groups);
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), 2000, 1, 50, 0, tableName);
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), 2000, 1, 50, 0, tableName);
+ connector.tableOperations().flush(tableName, null, null, true);
+ BatchScanner bscanner = connector.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 1);
+ String tableId = connector.tableOperations().tableIdMap().get(tableName);
+ bscanner.setRanges(Collections.singletonList(new Range(new Text(tableId + ";"), new Text(tableId + "<"))));
+ bscanner.fetchColumnFamily(DataFileColumnFamily.NAME);
+ boolean foundFile = false;
+ for (Entry<Key,Value> entry : bscanner) {
+ foundFile = true;
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream newOut = new PrintStream(baos);
+ PrintStream oldOut = System.out;
+ try {
+ System.setOut(newOut);
+ List<String> args = new ArrayList<>();
+ args.add(entry.getKey().getColumnQualifier().toString());
+ if (ClusterType.STANDALONE == getClusterType() && cluster.getClientConfig().getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ args.add("--config");
+ StandaloneAccumuloCluster sac = (StandaloneAccumuloCluster) cluster;
+ String hadoopConfDir = sac.getHadoopConfDir();
+ args.add(new Path(hadoopConfDir, "core-site.xml").toString());
+ args.add(new Path(hadoopConfDir, "hdfs-site.xml").toString());
+ }
+ log.info("Invoking PrintInfo with " + args);
+ PrintInfo.main(args.toArray(new String[args.size()]));
+ newOut.flush();
+ String stdout = baos.toString();
+ assertTrue(stdout.contains("Locality group : g1"));
+ assertTrue(stdout.contains("families : [colf]"));
+ } finally {
+ newOut.close();
+ System.setOut(oldOut);
+ }
+ }
+ bscanner.close();
+ assertTrue(foundFile);
+ }
+
+ @Test
+ public void localityGroupChange() throws Exception {
+ // Make changes to locality groups and ensure nothing is lost
+ final Connector connector = getConnector();
+ String table = getUniqueNames(1)[0];
+ TableOperations to = connector.tableOperations();
+ to.create(table);
+ String[] config = new String[] {"lg1:colf", null, "lg1:colf,xyz", "lg1:colf,xyz;lg2:c1,c2"};
+ int i = 0;
+ for (String cfg : config) {
+ to.setLocalityGroups(table, getGroups(cfg));
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS * (i + 1), 1, 50, ROWS * i, table);
+ to.flush(table, null, null, true);
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), 0, 1, 50, ROWS * (i + 1), table);
+ i++;
+ }
+ to.delete(table);
+ to.create(table);
+ config = new String[] {"lg1:colf", null, "lg1:colf,xyz", "lg1:colf;lg2:colf",};
+ i = 1;
+ for (String cfg : config) {
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS * i, 1, 50, 0, table);
+ ingest(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS * i, 1, 50, 0, "xyz", table);
+ to.setLocalityGroups(table, getGroups(cfg));
+ to.flush(table, null, null, true);
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS * i, 1, 50, 0, table);
+ verify(connector, getCluster().getClientConfig(), getAdminPrincipal(), ROWS * i, 1, 50, 0, "xyz", table);
+ i++;
+ }
+ }
+
+ private Map<String,Set<Text>> getGroups(String cfg) {
+ Map<String,Set<Text>> groups = new TreeMap<String,Set<Text>>();
+ if (cfg != null) {
+ for (String group : cfg.split(";")) {
+ String[] parts = group.split(":");
+ Set<Text> cols = new HashSet<Text>();
+ for (String col : parts[1].split(",")) {
+ cols.add(t(col));
+ }
+ groups.put(parts[1], cols);
+ }
+ }
+ return groups;
+ }
+
+ private static class TestTrustManager implements X509TrustManager {
+ @Override
+ public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
+
+ @Override
+ public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
+
+ @Override
+ public X509Certificate[] getAcceptedIssuers() {
+ return null;
+ }
+ }
+
+ private static class TestHostnameVerifier implements HostnameVerifier {
+ @Override
+ public boolean verify(String hostname, SSLSession session) {
+ return true;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java
index d73bf3e,0000000..4ef0cab
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RestartIT.java
@@@ -1,367 -1,0 +1,367 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.Map.Entry;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.cluster.ClusterUser;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class RestartIT extends AccumuloClusterHarness {
+ private static final Logger log = LoggerFactory.getLogger(RestartIT.class);
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 10 * 60;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ private static final ScannerOpts SOPTS = new ScannerOpts();
+ private static final VerifyIngest.Opts VOPTS = new VerifyIngest.Opts();
+ private static final TestIngest.Opts OPTS = new TestIngest.Opts();
+ private static final BatchWriterOpts BWOPTS = new BatchWriterOpts();
+ static {
+ OPTS.rows = VOPTS.rows = 10 * 1000;
+ }
+
+ private ExecutorService svc;
+
+ @Before
+ public void setup() throws Exception {
+ svc = Executors.newFixedThreadPool(1);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (null == svc) {
+ return;
+ }
+
+ if (!svc.isShutdown()) {
+ svc.shutdown();
+ }
+
+ while (!svc.awaitTermination(10, TimeUnit.SECONDS)) {
+ log.info("Waiting for threadpool to terminate");
+ }
+ }
+
+ @Test
+ public void restartMaster() throws Exception {
+ Connector c = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ OPTS.setTableName(tableName);
+ VOPTS.setTableName(tableName);
+ c.tableOperations().create(tableName);
+ final AuthenticationToken token = getAdminToken();
+ final ClusterControl control = getCluster().getClusterControl();
+
+ final String[] args;
+ if (token instanceof PasswordToken) {
+ byte[] password = ((PasswordToken) token).getPassword();
+ args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, Charsets.UTF_8), "-i", cluster.getInstanceName(), "-z",
+ cluster.getZooKeepers(), "--rows", "" + OPTS.rows, "--table", tableName};
+ OPTS.setPrincipal(getAdminPrincipal());
+ VOPTS.setPrincipal(getAdminPrincipal());
+ } else if (token instanceof KerberosToken) {
+ ClusterUser rootUser = getAdminUser();
+ args = new String[] {"-u", getAdminPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-i", cluster.getInstanceName(), "-z",
+ cluster.getZooKeepers(), "--rows", "" + OPTS.rows, "--table", tableName};
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ OPTS.updateKerberosCredentials(clientConfig);
+ VOPTS.updateKerberosCredentials(clientConfig);
+ } else {
+ throw new RuntimeException("Unknown token");
+ }
+
+ Future<Integer> ret = svc.submit(new Callable<Integer>() {
+ @Override
+ public Integer call() {
+ try {
+ return control.exec(TestIngest.class, args);
+ } catch (IOException e) {
+ log.error("Error running TestIngest", e);
+ return -1;
+ }
+ }
+ });
+
+ control.stopAllServers(ServerType.MASTER);
+ control.startAllServers(ServerType.MASTER);
+ assertEquals(0, ret.get().intValue());
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ }
+
+ @Test
+ public void restartMasterRecovery() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ OPTS.setTableName(tableName);
+ VOPTS.setTableName(tableName);
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ OPTS.updateKerberosCredentials(clientConfig);
+ VOPTS.updateKerberosCredentials(clientConfig);
+ } else {
+ OPTS.setPrincipal(getAdminPrincipal());
+ VOPTS.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, OPTS, BWOPTS);
+ ClusterControl control = getCluster().getClusterControl();
+
+ // TODO implement a kill all too?
+ // cluster.stop() would also stop ZooKeeper
+ control.stopAllServers(ServerType.MASTER);
+ control.stopAllServers(ServerType.TRACER);
+ control.stopAllServers(ServerType.TABLET_SERVER);
+ control.stopAllServers(ServerType.GARBAGE_COLLECTOR);
+ control.stopAllServers(ServerType.MONITOR);
+
+ ZooReader zreader = new ZooReader(c.getInstance().getZooKeepers(), c.getInstance().getZooKeepersSessionTimeOut());
+ ZooCache zcache = new ZooCache(zreader, null);
+ byte[] masterLockData;
+ do {
+ masterLockData = ZooLock.getLockData(zcache, ZooUtil.getRoot(c.getInstance()) + Constants.ZMASTER_LOCK, null);
+ if (null != masterLockData) {
+ log.info("Master lock is still held");
+ Thread.sleep(1000);
+ }
+ } while (null != masterLockData);
+
+ cluster.start();
+ sleepUninterruptibly(5, TimeUnit.MILLISECONDS);
+ control.stopAllServers(ServerType.MASTER);
+
+ masterLockData = new byte[0];
+ do {
+ masterLockData = ZooLock.getLockData(zcache, ZooUtil.getRoot(c.getInstance()) + Constants.ZMASTER_LOCK, null);
+ if (null != masterLockData) {
+ log.info("Master lock is still held");
+ Thread.sleep(1000);
+ }
+ } while (null != masterLockData);
+ cluster.start();
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ }
+
+ @Test
+ public void restartMasterSplit() throws Exception {
+ Connector c = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ final AuthenticationToken token = getAdminToken();
+ final ClusterControl control = getCluster().getClusterControl();
+ VOPTS.setTableName(tableName);
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
+
+ final String[] args;
+ if (token instanceof PasswordToken) {
+ byte[] password = ((PasswordToken) token).getPassword();
+ args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, Charsets.UTF_8), "-i", cluster.getInstanceName(), "-z",
+ cluster.getZooKeepers(), "--rows", Integer.toString(VOPTS.rows), "--table", tableName};
+ OPTS.setPrincipal(getAdminPrincipal());
+ VOPTS.setPrincipal(getAdminPrincipal());
+ } else if (token instanceof KerberosToken) {
+ ClusterUser rootUser = getAdminUser();
+ args = new String[] {"-u", getAdminPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-i", cluster.getInstanceName(), "-z",
+ cluster.getZooKeepers(), "--rows", Integer.toString(VOPTS.rows), "--table", tableName};
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ OPTS.updateKerberosCredentials(clientConfig);
+ VOPTS.updateKerberosCredentials(clientConfig);
+ } else {
+ throw new RuntimeException("Unknown token");
+ }
+
+ Future<Integer> ret = svc.submit(new Callable<Integer>() {
+ @Override
+ public Integer call() {
+ try {
+ return control.exec(TestIngest.class, args);
+ } catch (Exception e) {
+ log.error("Error running TestIngest", e);
+ return -1;
+ }
+ }
+ });
+
+ control.stopAllServers(ServerType.MASTER);
+
+ ZooReader zreader = new ZooReader(c.getInstance().getZooKeepers(), c.getInstance().getZooKeepersSessionTimeOut());
+ ZooCache zcache = new ZooCache(zreader, null);
+ byte[] masterLockData;
+ do {
+ masterLockData = ZooLock.getLockData(zcache, ZooUtil.getRoot(c.getInstance()) + Constants.ZMASTER_LOCK, null);
+ if (null != masterLockData) {
+ log.info("Master lock is still held");
+ Thread.sleep(1000);
+ }
+ } while (null != masterLockData);
+
+ cluster.start();
+ assertEquals(0, ret.get().intValue());
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ }
+
+ @Test
+ public void killedTabletServer() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ OPTS.setTableName(tableName);
+ VOPTS.setTableName(tableName);
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ OPTS.updateKerberosCredentials(clientConfig);
+ VOPTS.updateKerberosCredentials(clientConfig);
+ } else {
+ OPTS.setPrincipal(getAdminPrincipal());
+ VOPTS.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, OPTS, BWOPTS);
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ cluster.getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ cluster.start();
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ }
+
+ @Test
+ public void killedTabletServer2() throws Exception {
+ final Connector c = getConnector();
+ final String[] names = getUniqueNames(2);
+ final String tableName = names[0];
+ final ClusterControl control = getCluster().getClusterControl();
+ c.tableOperations().create(tableName);
+ // Original test started and then stopped a GC. Not sure why it did this. The GC was
+ // already running by default, and it would have nothing to do after only creating a table
+ control.stopAllServers(ServerType.TABLET_SERVER);
+
+ cluster.start();
+ c.tableOperations().create(names[1]);
+ }
+
+ @Test
+ public void killedTabletServerDuringShutdown() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ OPTS.setTableName(tableName);
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ OPTS.updateKerberosCredentials(clientConfig);
+ } else {
+ OPTS.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, OPTS, BWOPTS);
+ try {
+ getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ getCluster().getClusterControl().adminStopAll();
+ } finally {
+ getCluster().start();
+ }
+ }
+
+ @Test
+ public void shutdownDuringCompactingSplitting() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ VOPTS.setTableName(tableName);
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ OPTS.updateKerberosCredentials(clientConfig);
+ VOPTS.updateKerberosCredentials(clientConfig);
+ } else {
+ OPTS.setPrincipal(getAdminPrincipal());
+ VOPTS.setPrincipal(getAdminPrincipal());
+ }
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
+ String splitThreshold = null;
+ for (Entry<String,String> entry : c.tableOperations().getProperties(tableName)) {
+ if (entry.getKey().equals(Property.TABLE_SPLIT_THRESHOLD.getKey())) {
+ splitThreshold = entry.getValue();
+ break;
+ }
+ }
+ Assert.assertNotNull(splitThreshold);
+ try {
+ c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), "20K");
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.setTableName(tableName);
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConfig);
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, opts, BWOPTS);
+ c.tableOperations().flush(tableName, null, null, false);
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ getCluster().stop();
+ } finally {
+ if (getClusterType() == ClusterType.STANDALONE) {
+ getCluster().start();
+ c.tableOperations().setProperty(MetadataTable.NAME, Property.TABLE_SPLIT_THRESHOLD.getKey(), splitThreshold);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java
index f607d16,0000000..f58db38
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@@ -1,153 -1,0 +1,153 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.cluster.ClusterUser;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class RestartStressIT extends AccumuloClusterHarness {
+ private static final Logger log = LoggerFactory.getLogger(RestartStressIT.class);
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ Map<String,String> opts = cfg.getSiteConfig();
+ opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
+ opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
+ opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
- opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
++ opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
+ cfg.setSiteConfig(opts);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 10 * 60;
+ }
+
+ private ExecutorService svc;
+
+ @Before
+ public void setup() throws Exception {
+ svc = Executors.newFixedThreadPool(1);
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (null == svc) {
+ return;
+ }
+
+ if (!svc.isShutdown()) {
+ svc.shutdown();
+ }
+
+ while (!svc.awaitTermination(10, TimeUnit.SECONDS)) {
+ log.info("Waiting for threadpool to terminate");
+ }
+ }
+
+ private static final VerifyIngest.Opts VOPTS;
+ static {
+ VOPTS = new VerifyIngest.Opts();
+ VOPTS.rows = 10 * 1000;
+ }
+ private static final ScannerOpts SOPTS = new ScannerOpts();
+
+ @Test
+ public void test() throws Exception {
+ final Connector c = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ final AuthenticationToken token = getAdminToken();
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "500K");
+ final ClusterControl control = getCluster().getClusterControl();
+ final String[] args;
+ if (token instanceof PasswordToken) {
+ byte[] password = ((PasswordToken) token).getPassword();
+ args = new String[] {"-u", getAdminPrincipal(), "-p", new String(password, Charsets.UTF_8), "-i", cluster.getInstanceName(), "-z",
+ cluster.getZooKeepers(), "--rows", "" + VOPTS.rows, "--table", tableName};
+ } else if (token instanceof KerberosToken) {
+ ClusterUser rootUser = getAdminUser();
+ args = new String[] {"-u", getAdminPrincipal(), "--keytab", rootUser.getKeytab().getAbsolutePath(), "-i", cluster.getInstanceName(), "-z",
+ cluster.getZooKeepers(), "--rows", "" + VOPTS.rows, "--table", tableName};
+ } else {
+ throw new RuntimeException("Unrecognized token");
+ }
+
+ Future<Integer> retCode = svc.submit(new Callable<Integer>() {
+ @Override
+ public Integer call() {
+ try {
+ return control.exec(TestIngest.class, args);
+ } catch (Exception e) {
+ log.error("Error running TestIngest", e);
+ return -1;
+ }
+ }
+ });
+
+ for (int i = 0; i < 2; i++) {
+ sleepUninterruptibly(10, TimeUnit.SECONDS);
+ control.stopAllServers(ServerType.TABLET_SERVER);
+ control.startAllServers(ServerType.TABLET_SERVER);
+ }
+ assertEquals(0, retCode.get().intValue());
+ VOPTS.setTableName(tableName);
+
+ if (token instanceof PasswordToken) {
+ VOPTS.setPrincipal(getAdminPrincipal());
+ } else if (token instanceof KerberosToken) {
+ VOPTS.updateKerberosCredentials(cluster.getClientConfig());
+ } else {
+ throw new RuntimeException("Unrecognized token");
+ }
+
+ VerifyIngest.verifyIngest(c, VOPTS, SOPTS);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
index 36bdd7a,0000000..5f7ca88
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
@@@ -1,153 -1,0 +1,153 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriter.Status;
+import org.apache.accumulo.core.client.ConditionalWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Durability;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Condition;
+import org.apache.accumulo.core.data.ConditionalMutation;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class SessionDurabilityIT extends ConfigurableMacBase {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void nondurableTableHasDurableWrites() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default has no durability
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ // send durable writes
+ BatchWriterConfig cfg = new BatchWriterConfig();
+ cfg.setDurability(Durability.SYNC);
+ writeSome(tableName, 10, cfg);
+ assertEquals(10, count(tableName));
+ // verify writes servive restart
+ restartTServer();
+ assertEquals(10, count(tableName));
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void durableTableLosesNonDurableWrites() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ // write with no durability
+ BatchWriterConfig cfg = new BatchWriterConfig();
+ cfg.setDurability(Durability.NONE);
+ writeSome(tableName, 10, cfg);
+ // verify writes are lost on restart
+ restartTServer();
+ assertTrue(10 > count(tableName));
+ }
+
+ private int count(String tableName) throws Exception {
+ return Iterators.size(getConnector().createScanner(tableName, Authorizations.EMPTY).iterator());
+ }
+
+ private void writeSome(String tableName, int n, BatchWriterConfig cfg) throws Exception {
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(tableName, cfg);
+ for (int i = 0; i < n; i++) {
+ Mutation m = new Mutation(i + "");
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void testConditionDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ // write without durability
+ ConditionalWriterConfig cfg = new ConditionalWriterConfig();
+ cfg.setDurability(Durability.NONE);
+ conditionWriteSome(tableName, 10, cfg);
+ // everything in there?
+ assertEquals(10, count(tableName));
+ // restart the server and verify the updates are lost
+ restartTServer();
+ assertEquals(0, count(tableName));
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void testConditionDurability2() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ // write with durability
+ ConditionalWriterConfig cfg = new ConditionalWriterConfig();
+ cfg.setDurability(Durability.SYNC);
+ conditionWriteSome(tableName, 10, cfg);
+ // everything in there?
+ assertEquals(10, count(tableName));
+ // restart the server and verify the updates are still there
+ restartTServer();
+ assertEquals(10, count(tableName));
+ }
+
+ private void conditionWriteSome(String tableName, int n, ConditionalWriterConfig cfg) throws Exception {
+ Connector c = getConnector();
+ ConditionalWriter cw = c.createConditionalWriter(tableName, cfg);
+ for (int i = 0; i < n; i++) {
+ ConditionalMutation m = new ConditionalMutation((CharSequence) (i + ""), new Condition("", ""));
+ m.put("", "", "X");
+ assertEquals(Status.ACCEPTED, cw.write(m).getStatus());
+ }
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
index ab3c662,0000000..6f0bc7a
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/WALSunnyDayIT.java
@@@ -1,235 -1,0 +1,235 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.apache.accumulo.core.conf.Property.GC_CYCLE_DELAY;
+import static org.apache.accumulo.core.conf.Property.GC_CYCLE_START;
+import static org.apache.accumulo.core.conf.Property.INSTANCE_ZK_TIMEOUT;
+import static org.apache.accumulo.core.conf.Property.TSERV_WALOG_MAX_SIZE;
+import static org.apache.accumulo.core.conf.Property.TSERV_WAL_REPLICATION;
+import static org.apache.accumulo.core.security.Authorizations.EMPTY;
+import static org.apache.accumulo.minicluster.ServerType.GARBAGE_COLLECTOR;
+import static org.apache.accumulo.minicluster.ServerType.TABLET_SERVER;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
+import org.apache.accumulo.master.state.SetGoalState;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterControl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.log.WalStateManager;
+import org.apache.accumulo.server.log.WalStateManager.WalState;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class WALSunnyDayIT extends ConfigurableMacBase {
+
+ private static final Text CF = new Text(new byte[0]);
+
+ @Override
+ protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setProperty(GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(GC_CYCLE_START, "0s");
+ cfg.setProperty(TSERV_WALOG_MAX_SIZE, "1M");
+ cfg.setProperty(TSERV_WAL_REPLICATION, "1");
- cfg.setProperty(INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ int countTrue(Collection<Boolean> bools) {
+ int result = 0;
+ for (Boolean b : bools) {
+ if (b.booleanValue())
+ result++;
+ }
+ return result;
+ }
+
+ @Test
+ public void test() throws Exception {
+ MiniAccumuloClusterImpl mac = getCluster();
+ MiniAccumuloClusterControl control = mac.getClusterControl();
+ control.stop(GARBAGE_COLLECTOR);
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ writeSomeData(c, tableName, 1, 1);
+
+ // wal markers are added lazily
+ Map<String,Boolean> wals = getWals(c);
+ assertEquals(wals.toString(), 2, wals.size());
+ for (Boolean b : wals.values()) {
+ assertTrue("logs should be in use", b.booleanValue());
+ }
+
+ // roll log, get a new next
+ writeSomeData(c, tableName, 1001, 50);
+ Map<String,Boolean> walsAfterRoll = getWals(c);
+ assertEquals("should have 3 WALs after roll", 3, walsAfterRoll.size());
+ assertTrue("new WALs should be a superset of the old WALs", walsAfterRoll.keySet().containsAll(wals.keySet()));
+ assertEquals("all WALs should be in use", 3, countTrue(walsAfterRoll.values()));
+
+ // flush the tables
+ for (String table : new String[] {tableName, MetadataTable.NAME, RootTable.NAME}) {
+ c.tableOperations().flush(table, null, null, true);
+ }
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ // rolled WAL is no longer in use, but needs to be GC'd
+ Map<String,Boolean> walsAfterflush = getWals(c);
+ assertEquals(walsAfterflush.toString(), 3, walsAfterflush.size());
+ assertEquals("inUse should be 2", 2, countTrue(walsAfterflush.values()));
+
+ // let the GC run for a little bit
+ control.start(GARBAGE_COLLECTOR);
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ // make sure the unused WAL goes away
+ Map<String,Boolean> walsAfterGC = getWals(c);
+ assertEquals(walsAfterGC.toString(), 2, walsAfterGC.size());
+ control.stop(GARBAGE_COLLECTOR);
+ // restart the tserver, but don't run recovery on all tablets
+ control.stop(TABLET_SERVER);
+ // this delays recovery on the normal tables
+ assertEquals(0, cluster.exec(SetGoalState.class, "SAFE_MODE").waitFor());
+ control.start(TABLET_SERVER);
+
+ // wait for the metadata table to go back online
+ getRecoveryMarkers(c);
+ // allow a little time for the master to notice ASSIGNED_TO_DEAD_SERVER tablets
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ Map<KeyExtent,List<String>> markers = getRecoveryMarkers(c);
+ // log.debug("markers " + markers);
+ assertEquals("one tablet should have markers", 1, markers.keySet().size());
+ assertEquals("tableId of the keyExtent should be 1", markers.keySet().iterator().next().getTableId(), new Text("1"));
+
+ // put some data in the WAL
+ assertEquals(0, cluster.exec(SetGoalState.class, "NORMAL").waitFor());
+ verifySomeData(c, tableName, 1001 * 50 + 1);
+ writeSomeData(c, tableName, 100, 100);
+
+ Map<String,Boolean> walsAfterRestart = getWals(c);
+ // log.debug("wals after " + walsAfterRestart);
+ assertEquals("used WALs after restart should be 4", 4, countTrue(walsAfterRestart.values()));
+ control.start(GARBAGE_COLLECTOR);
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ Map<String,Boolean> walsAfterRestartAndGC = getWals(c);
+ assertEquals("wals left should be 2", 2, walsAfterRestartAndGC.size());
+ assertEquals("logs in use should be 2", 2, countTrue(walsAfterRestartAndGC.values()));
+ }
+
+ private void verifySomeData(Connector c, String tableName, int expected) throws Exception {
+ Scanner scan = c.createScanner(tableName, EMPTY);
+ int result = Iterators.size(scan.iterator());
+ scan.close();
+ Assert.assertEquals(expected, result);
+ }
+
+ private void writeSomeData(Connector conn, String tableName, int row, int col) throws Exception {
+ Random rand = new Random();
+ BatchWriter bw = conn.createBatchWriter(tableName, null);
+ byte[] rowData = new byte[10];
+ byte[] cq = new byte[10];
+ byte[] value = new byte[10];
+
+ for (int r = 0; r < row; r++) {
+ rand.nextBytes(rowData);
+ Mutation m = new Mutation(rowData);
+ for (int c = 0; c < col; c++) {
+ rand.nextBytes(cq);
+ rand.nextBytes(value);
+ m.put(CF, new Text(cq), new Value(value));
+ }
+ bw.addMutation(m);
+ if (r % 100 == 0) {
+ bw.flush();
+ }
+ }
+ bw.close();
+ }
+
+ private Map<String,Boolean> getWals(Connector c) throws Exception {
+ Map<String,Boolean> result = new HashMap<>();
+ Instance i = c.getInstance();
+ ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
+ WalStateManager wals = new WalStateManager(c.getInstance(), zk);
+ for (Entry<Path,WalState> entry : wals.getAllState().entrySet()) {
+ // WALs are in use if they are not unreferenced
+ result.put(entry.getKey().toString(), entry.getValue() != WalState.UNREFERENCED);
+ }
+ return result;
+ }
+
+ private Map<KeyExtent,List<String>> getRecoveryMarkers(Connector c) throws Exception {
+ Map<KeyExtent,List<String>> result = new HashMap<>();
+ Scanner root = c.createScanner(RootTable.NAME, EMPTY);
+ root.setRange(TabletsSection.getRange());
+ root.fetchColumnFamily(TabletsSection.LogColumnFamily.NAME);
+ TabletColumnFamily.PREV_ROW_COLUMN.fetch(root);
+
+ Scanner meta = c.createScanner(MetadataTable.NAME, EMPTY);
+ meta.setRange(TabletsSection.getRange());
+ meta.fetchColumnFamily(TabletsSection.LogColumnFamily.NAME);
+ TabletColumnFamily.PREV_ROW_COLUMN.fetch(meta);
+
+ List<String> logs = new ArrayList<>();
+ Iterator<Entry<Key,Value>> both = Iterators.concat(root.iterator(), meta.iterator());
+ while (both.hasNext()) {
+ Entry<Key,Value> entry = both.next();
+ Key key = entry.getKey();
+ if (key.getColumnFamily().equals(TabletsSection.LogColumnFamily.NAME)) {
+ logs.add(key.getColumnQualifier().toString());
+ }
+ if (TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key) && !logs.isEmpty()) {
+ KeyExtent extent = new KeyExtent(key.getRow(), entry.getValue());
+ result.put(extent, logs);
+ logs = new ArrayList<String>();
+ }
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
index d877969,0000000..0074f5f
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
@@@ -1,79 -1,0 +1,79 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+
+public class WriteAheadLogIT extends AccumuloClusterHarness {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "1s");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "4s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 10 * 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "750K");
+ TestIngest.Opts opts = new TestIngest.Opts();
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ opts.setTableName(tableName);
+
+ ClientConfiguration clientConfig = cluster.getClientConfig();
+ if (clientConfig.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConfig);
+ vopts.updateKerberosCredentials(clientConfig);
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ vopts.setPrincipal(getAdminPrincipal());
+ }
+
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+ vopts.setTableName(tableName);
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ getCluster().getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ getCluster().getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
index b19ec2f,0000000..5ece1ac
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
@@@ -1,89 -1,0 +1,89 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class ZookeeperRestartIT extends ConfigurableMacBase {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
++ siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ BatchWriter bw = c.createBatchWriter("test_ingest", null);
+ Mutation m = new Mutation("row");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // kill zookeeper
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.ZOOKEEPER))
+ cluster.killProcess(ServerType.ZOOKEEPER, proc);
+
+ // give the servers time to react
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+
+ // start zookeeper back up
+ cluster.start();
+
+ // use the tservers
+ Scanner s = c.createScanner("test_ingest", Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> i = s.iterator();
+ assertTrue(i.hasNext());
+ assertEquals("row", i.next().getKey().getRow().toString());
+ assertFalse(i.hasNext());
+ // use the master
+ c.tableOperations().delete("test_ingest");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
index 7f61a08,0000000..133c09c
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/proxy/ProxyDurabilityIT.java
@@@ -1,146 -1,0 +1,146 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.proxy;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.proxy.Proxy;
+import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client;
+import org.apache.accumulo.proxy.thrift.Column;
+import org.apache.accumulo.proxy.thrift.ColumnUpdate;
+import org.apache.accumulo.proxy.thrift.Condition;
+import org.apache.accumulo.proxy.thrift.ConditionalStatus;
+import org.apache.accumulo.proxy.thrift.ConditionalUpdates;
+import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions;
+import org.apache.accumulo.proxy.thrift.Durability;
+import org.apache.accumulo.proxy.thrift.TimeType;
+import org.apache.accumulo.proxy.thrift.WriterOptions;
+import org.apache.accumulo.server.util.PortUtils;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.thrift.protocol.TJSONProtocol;
+import org.apache.thrift.server.TServer;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+import com.google.common.net.HostAndPort;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class ProxyDurabilityIT extends ConfigurableMacBase {
+
+ @Override
+ protected int defaultTimeoutSeconds() {
- return 60;
++ return 120;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ }
+
+ private static ByteBuffer bytes(String value) {
+ return ByteBuffer.wrap(value.getBytes());
+ }
+
+ @Test
+ public void testDurability() throws Exception {
+ Connector c = getConnector();
+ Properties props = new Properties();
+ // Avoid issues with locally installed client configuration files with custom properties
+ File emptyFile = Files.createTempFile(null, null).toFile();
+ emptyFile.deleteOnExit();
+ props.put("instance", c.getInstance().getInstanceName());
+ props.put("zookeepers", c.getInstance().getZooKeepers());
+ props.put("tokenClass", PasswordToken.class.getName());
+ props.put("clientConfigurationFile", emptyFile.toString());
+
+ TJSONProtocol.Factory protocol = new TJSONProtocol.Factory();
+
+ int proxyPort = PortUtils.getRandomFreePort();
+ final TServer proxyServer = Proxy.createProxyServer(HostAndPort.fromParts("localhost", proxyPort), protocol, props).server;
+ while (!proxyServer.isServing())
+ sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ Client client = new TestProxyClient("localhost", proxyPort, protocol).proxy();
+ Map<String,String> properties = new TreeMap<String,String>();
+ properties.put("password", ROOT_PASSWORD);
+ ByteBuffer login = client.login("root", properties);
+
+ String tableName = getUniqueNames(1)[0];
+ client.createTable(login, tableName, true, TimeType.MILLIS);
+ assertTrue(c.tableOperations().exists(tableName));
+
+ WriterOptions options = new WriterOptions();
+ options.setDurability(Durability.NONE);
+ String writer = client.createWriter(login, tableName, options);
+ Map<ByteBuffer,List<ColumnUpdate>> cells = new TreeMap<ByteBuffer,List<ColumnUpdate>>();
+ ColumnUpdate column = new ColumnUpdate(bytes("cf"), bytes("cq"));
+ column.setValue("value".getBytes());
+ cells.put(bytes("row"), Collections.singletonList(column));
+ client.update(writer, cells);
+ client.closeWriter(writer);
+ assertEquals(1, count(tableName));
+ restartTServer();
+ assertEquals(0, count(tableName));
+
+ ConditionalWriterOptions cfg = new ConditionalWriterOptions();
+ cfg.setDurability(Durability.SYNC);
+ String cwriter = client.createConditionalWriter(login, tableName, cfg);
+ ConditionalUpdates updates = new ConditionalUpdates();
+ updates.addToConditions(new Condition(new Column(bytes("cf"), bytes("cq"), bytes(""))));
+ updates.addToUpdates(column);
+ Map<ByteBuffer,ConditionalStatus> status = client.updateRowsConditionally(cwriter, Collections.singletonMap(bytes("row"), updates));
+ assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes("row")));
+ assertEquals(1, count(tableName));
+ restartTServer();
+ assertEquals(1, count(tableName));
+
+ proxyServer.stop();
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+ private int count(String tableName) throws Exception {
+ return Iterators.size((getConnector().createScanner(tableName, Authorizations.EMPTY)).iterator());
+ }
+
+}
[08/13] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
index 7650dd5,0000000..5d4e637
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
@@@ -1,731 -1,0 +1,731 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.master.replication.UnorderedWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class UnorderedWorkAssignerReplicationIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(UnorderedWorkAssignerReplicationIT.class);
+
+ private ExecutorService executor;
+ private int timeoutFactor = 1;
+
+ @Before
+ public void createExecutor() {
+ executor = Executors.newSingleThreadExecutor();
+
+ try {
+ timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
+ } catch (NumberFormatException exception) {
+ log.warn("Could not parse timeout.factor, not increasing timeout.");
+ }
+
+ Assert.assertTrue("The timeout factor must be a positive, non-zero value", timeoutFactor > 0);
+ }
+
+ @After
+ public void stopExecutor() {
+ if (null != executor) {
+ executor.shutdownNow();
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60 * 5;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, UnorderedWorkAssigner.class.getName());
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+ */
+ private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+ // Set the same SSL information from the primary when present
+ Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+ if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+ Map<String,String> peerSiteConfig = new HashMap<String,String>();
+ peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+ String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+ Assert.assertNotNull("Keystore Path was null", keystorePath);
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+ String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+ Assert.assertNotNull("Truststore Path was null", truststorePath);
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+ // Passwords might be stored in CredentialProvider
+ String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+ if (null != keystorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+ }
+ String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+ if (null != truststorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+ }
+
+ System.out.println("Setting site configuration for peer " + peerSiteConfig);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+
+ // Use the CredentialProvider if the primary also uses one
+ String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+ if (null != credProvider) {
+ Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+ peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeer() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ try {
+ final Connector connMaster = getConnector();
+ final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ ReplicationTable.setOnline(connMaster);
+
+ String peerUserName = "peer", peerPassword = "foo";
+
+ String peerClusterName = "peer";
+
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ final String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Wait for zookeeper updates (configuration) to propagate
+ sleepUninterruptibly(3, TimeUnit.SECONDS);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("TabletServer restarted");
+ Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+ log.info("TabletServer is online");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+ @Override
+ public Boolean call() throws Exception {
+ connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+ log.info("Drain completed");
+ return true;
+ }
+
+ });
+
+ long timeoutSeconds = timeoutFactor * 30;
+ try {
+ future.get(timeoutSeconds, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ Assert.fail("Drain did not finish within " + timeoutSeconds + " seconds");
+ }
+
+ log.info("drain completed");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Entry<Key,Value> masterEntry = null, peerEntry = null;
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ masterEntry = masterIter.next();
+ peerEntry = peerIter.next();
+ Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ log.info("Last master entry: " + masterEntry);
+ log.info("Last peer entry: " + peerEntry);
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+ } finally {
+ peerCluster.stop();
+ }
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTable() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+ String peerUserName = "peer", peerPassword = "foo";
+
+ // Create local user
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ // Create tables
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Grant write permission
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Wait for zookeeper updates (configuration) to propogate
+ sleepUninterruptibly(3, TimeUnit.SECONDS);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ long masterTable1Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable1Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ long masterTable2Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable2Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+ masterTable2);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ Thread.sleep(500);
+ }
+
+ // Restart the tserver to force a close on the WAL
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("Restarted the tserver");
+
+ // Read the data -- the tserver is back up and running
+ Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+ // Wait for both tables to be replicated
+ log.info("Waiting for {} for {}", filesFor1, masterTable1);
+ connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+ log.info("Waiting for {} for {}", filesFor2, masterTable2);
+ connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+ long countTable = 0l;
+ for (int i = 0; i < 5; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+
+ if (masterTable1Records != countTable) {
+ log.warn("Did not find {} expected records in {}, only found {}", masterTable1Records, peerTable1, countTable);
+ }
+ }
+
+ Assert.assertEquals(masterTable1Records, countTable);
+
+ for (int i = 0; i < 5; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+
+ if (masterTable2Records != countTable) {
+ log.warn("Did not find {} expected records in {}, only found {}", masterTable2Records, peerTable2, countTable);
+ }
+ }
+
+ Assert.assertEquals(masterTable2Records, countTable);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Give our replication user the ability to write to the table
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+ for (String s : files) {
+ log.info("Found referenced file for " + masterTable + ": " + s);
+ }
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+ for (Entry<Key,Value> kv : connMaster.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
+ log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ connMaster.replicationOperations().drain(masterTable, files);
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Assert.assertTrue("No data in master table", masterIter.hasNext());
+ Assert.assertTrue("No data in peer table", peerIter.hasNext());
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+ Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+ peerCluster.stop();
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Give our replication user the ability to write to the tables
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Wait for zookeeper updates (configuration) to propagate
+ sleepUninterruptibly(3, TimeUnit.SECONDS);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ Thread.sleep(500);
+ }
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ sleepUninterruptibly(timeoutFactor * 2, TimeUnit.SECONDS);
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
+ }
+ }
+
+ Assert.assertNotEquals(0, fullyReplicated);
+
+ long countTable = 0l;
+
+ // Check a few times
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+ log.info("Found {} records in {}", countTable, peerTable1);
+ if (0 < countTable) {
+ break;
+ }
+ Thread.sleep(2000);
+ }
+
+ Assert.assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
+
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+ if (0 < countTable) {
+ break;
+ }
+ Thread.sleep(2000);
+ }
+ Assert.assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+}
[06/13] accumulo git commit: Merge branch '1.6' into 1.7
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
index 761d92c,0000000..090ac83
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
@@@ -1,731 -1,0 +1,731 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.master.replication.UnorderedWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class UnorderedWorkAssignerReplicationIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(UnorderedWorkAssignerReplicationIT.class);
+
+ private ExecutorService executor;
+ private int timeoutFactor = 1;
+
+ @Before
+ public void createExecutor() {
+ executor = Executors.newSingleThreadExecutor();
+
+ try {
+ timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
+ } catch (NumberFormatException exception) {
+ log.warn("Could not parse timeout.factor, not increasing timeout.");
+ }
+
+ Assert.assertTrue("The timeout factor must be a positive, non-zero value", timeoutFactor > 0);
+ }
+
+ @After
+ public void stopExecutor() {
+ if (null != executor) {
+ executor.shutdownNow();
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60 * 5;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, UnorderedWorkAssigner.class.getName());
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+ */
+ private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+ // Set the same SSL information from the primary when present
+ Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+ if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+ Map<String,String> peerSiteConfig = new HashMap<String,String>();
+ peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+ String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+ Assert.assertNotNull("Keystore Path was null", keystorePath);
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+ String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+ Assert.assertNotNull("Truststore Path was null", truststorePath);
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+ // Passwords might be stored in CredentialProvider
+ String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+ if (null != keystorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+ }
+ String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+ if (null != truststorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+ }
+
+ System.out.println("Setting site configuration for peer " + peerSiteConfig);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+
+ // Use the CredentialProvider if the primary also uses one
+ String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+ if (null != credProvider) {
+ Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+ peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeer() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ try {
+ final Connector connMaster = getConnector();
+ final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ ReplicationTable.setOnline(connMaster);
+
+ String peerUserName = "peer", peerPassword = "foo";
+
+ String peerClusterName = "peer";
+
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ final String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Wait for zookeeper updates (configuration) to propagate
+ UtilWaitThread.sleep(3 * 1000);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("TabletServer restarted");
+ Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+ log.info("TabletServer is online");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+ @Override
+ public Boolean call() throws Exception {
+ connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+ log.info("Drain completed");
+ return true;
+ }
+
+ });
+
+ long timeoutSeconds = timeoutFactor * 30;
+ try {
+ future.get(timeoutSeconds, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ Assert.fail("Drain did not finish within " + timeoutSeconds + " seconds");
+ }
+
+ log.info("drain completed");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Entry<Key,Value> masterEntry = null, peerEntry = null;
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ masterEntry = masterIter.next();
+ peerEntry = peerIter.next();
+ Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ log.info("Last master entry: " + masterEntry);
+ log.info("Last peer entry: " + peerEntry);
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+ } finally {
+ peerCluster.stop();
+ }
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTable() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+ String peerUserName = "peer", peerPassword = "foo";
+
+ // Create local user
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ // Create tables
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Grant write permission
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Wait for zookeeper updates (configuration) to propogate
+ UtilWaitThread.sleep(3 * 1000);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ long masterTable1Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable1Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ long masterTable2Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable2Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+ masterTable2);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ Thread.sleep(500);
+ }
+
+ // Restart the tserver to force a close on the WAL
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("Restarted the tserver");
+
+ // Read the data -- the tserver is back up and running
+ Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+ // Wait for both tables to be replicated
+ log.info("Waiting for {} for {}", filesFor1, masterTable1);
+ connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+ log.info("Waiting for {} for {}", filesFor2, masterTable2);
+ connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+ long countTable = 0l;
+ for (int i = 0; i < 5; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+
+ if (masterTable1Records != countTable) {
+ log.warn("Did not find {} expected records in {}, only found {}", masterTable1Records, peerTable1, countTable);
+ }
+ }
+
+ Assert.assertEquals(masterTable1Records, countTable);
+
+ for (int i = 0; i < 5; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+
+ if (masterTable2Records != countTable) {
+ log.warn("Did not find {} expected records in {}, only found {}", masterTable2Records, peerTable2, countTable);
+ }
+ }
+
+ Assert.assertEquals(masterTable2Records, countTable);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Give our replication user the ability to write to the table
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+ for (String s : files) {
+ log.info("Found referenced file for " + masterTable + ": " + s);
+ }
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+ for (Entry<Key,Value> kv : connMaster.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
+ log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ connMaster.replicationOperations().drain(masterTable, files);
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Assert.assertTrue("No data in master table", masterIter.hasNext());
+ Assert.assertTrue("No data in peer table", peerIter.hasNext());
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+ Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+ peerCluster.stop();
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Give our replication user the ability to write to the tables
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Wait for zookeeper updates (configuration) to propagate
+ UtilWaitThread.sleep(3 * 1000);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ Thread.sleep(500);
+ }
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ UtilWaitThread.sleep(timeoutFactor * 2000);
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
+ }
+ }
+
+ Assert.assertNotEquals(0, fullyReplicated);
+
+ long countTable = 0l;
+
+ // Check a few times
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+ log.info("Found {} records in {}", countTable, peerTable1);
+ if (0 < countTable) {
+ break;
+ }
+ Thread.sleep(2000);
+ }
+
+ Assert.assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
+
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+ if (0 < countTable) {
+ break;
+ }
+ Thread.sleep(2000);
+ }
+ Assert.assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+}
[09/13] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
index b87bf04,0000000..0ff8c21
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@@ -1,1437 -1,0 +1,1436 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.IteratorSetting.Column;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.TableOfflineException;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
+import org.apache.accumulo.core.iterators.conf.ColumnSet;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.replication.ReplicationTableOfflineException;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.tabletserver.log.LogEntry;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.fate.zookeeper.ZooCacheFactory;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
+import org.apache.accumulo.gc.SimpleGarbageCollector;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.log.WalStateManager;
+import org.apache.accumulo.server.log.WalStateManager.WalState;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusCombiner;
+import org.apache.accumulo.server.replication.StatusFormatter;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.server.util.ReplicationTableUtil;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+import com.google.protobuf.TextFormat;
+
+/**
+ * Replication tests which verify expected functionality using a single MAC instance. A MockReplicaSystem is used to "fake" the peer instance that we're
+ * replicating to. This lets us test replication in a functional way without having to worry about two real systems.
+ */
+public class ReplicationIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationIT.class);
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60 * 10;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ // Run the master replication loop run frequently
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "0");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_PROCESSOR_DELAY, "1s");
+ cfg.setProperty(Property.REPLICATION_WORK_PROCESSOR_PERIOD, "1s");
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setNumTservers(1);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ private Multimap<String,String> getLogs(Connector conn) throws Exception {
+ // Map of server to tableId
+ Multimap<TServerInstance,String> serverToTableID = HashMultimap.create();
+ Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ for (Entry<Key,Value> entry : scanner) {
+ TServerInstance key = new TServerInstance(entry.getValue(), entry.getKey().getColumnQualifier());
+ byte[] tableId = KeyExtent.tableOfMetadataRow(entry.getKey().getRow());
+ serverToTableID.put(key, new String(tableId, UTF_8));
+ }
+ // Map of logs to tableId
+ Multimap<String,String> logs = HashMultimap.create();
+ Instance i = conn.getInstance();
+ ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
+ WalStateManager wals = new WalStateManager(conn.getInstance(), zk);
+ for (Entry<TServerInstance,List<UUID>> entry : wals.getAllMarkers().entrySet()) {
+ for (UUID id : entry.getValue()) {
+ Pair<WalState,Path> state = wals.state(entry.getKey(), id);
+ for (String tableId : serverToTableID.get(entry.getKey())) {
+ logs.put(state.getSecond().toString(), tableId);
+ }
+ }
+ }
+ return logs;
+ }
+
+ private Multimap<String,String> getAllLogs(Connector conn) throws Exception {
+ Multimap<String,String> logs = getLogs(conn);
+ try {
+ Scanner scanner = conn.createScanner(ReplicationTable.NAME, Authorizations.EMPTY);
+ StatusSection.limit(scanner);
+ Text buff = new Text();
+ for (Entry<Key,Value> entry : scanner) {
+ if (Thread.interrupted()) {
+ Thread.currentThread().interrupt();
+ return logs;
+ }
+
+ StatusSection.getFile(entry.getKey(), buff);
+ String file = buff.toString();
+ StatusSection.getTableId(entry.getKey(), buff);
+ String tableId = buff.toString();
+
+ logs.put(file, tableId);
+ }
+ } catch (TableOfflineException e) {
+ log.debug("Replication table isn't online yet");
+ }
+ return logs;
+ }
+
+ private void waitForGCLock(Connector conn) throws InterruptedException {
+ // Check if the GC process has the lock before wasting our retry attempts
+ ZooKeeperInstance zki = (ZooKeeperInstance) conn.getInstance();
+ ZooCacheFactory zcf = new ZooCacheFactory();
+ ZooCache zcache = zcf.getZooCache(zki.getZooKeepers(), zki.getZooKeepersSessionTimeOut());
+ String zkPath = ZooUtil.getRoot(conn.getInstance()) + Constants.ZGC_LOCK;
+ log.info("Looking for GC lock at {}", zkPath);
+ byte[] data = ZooLock.getLockData(zcache, zkPath, null);
+ while (null == data) {
+ log.info("Waiting for GC ZooKeeper lock to be acquired");
+ Thread.sleep(1000);
+ data = ZooLock.getLockData(zcache, zkPath, null);
+ }
+ }
+
+ @Test
+ public void replicationTableCreated() throws AccumuloException, AccumuloSecurityException {
+ Assert.assertTrue(getConnector().tableOperations().exists(ReplicationTable.NAME));
+ Assert.assertEquals(ReplicationTable.ID, getConnector().tableOperations().tableIdMap().get(ReplicationTable.NAME));
+ }
+
+ @Test
+ public void verifyReplicationTableConfig() throws AccumuloException, TableNotFoundException, AccumuloSecurityException {
+ TableOperations tops = getConnector().tableOperations();
+ Map<String,EnumSet<IteratorScope>> iterators = tops.listIterators(ReplicationTable.NAME);
+
+ // verify combiners are only iterators (no versioning)
+ Assert.assertEquals(1, iterators.size());
+
+ // look for combiner
+ Assert.assertTrue(iterators.containsKey(ReplicationTable.COMBINER_NAME));
+ Assert.assertTrue(iterators.get(ReplicationTable.COMBINER_NAME).containsAll(EnumSet.allOf(IteratorScope.class)));
+ for (IteratorScope scope : EnumSet.allOf(IteratorScope.class)) {
+ IteratorSetting is = tops.getIteratorSetting(ReplicationTable.NAME, ReplicationTable.COMBINER_NAME, scope);
+ Assert.assertEquals(30, is.getPriority());
+ Assert.assertEquals(StatusCombiner.class.getName(), is.getIteratorClass());
+ Assert.assertEquals(1, is.getOptions().size());
+ Assert.assertTrue(is.getOptions().containsKey("columns"));
+ String cols = is.getOptions().get("columns");
+ Column statusSectionCol = new Column(StatusSection.NAME);
+ Column workSectionCol = new Column(WorkSection.NAME);
+ Assert.assertEquals(
+ ColumnSet.encodeColumns(statusSectionCol.getColumnFamily(), statusSectionCol.getColumnQualifier()) + ","
+ + ColumnSet.encodeColumns(workSectionCol.getColumnFamily(), workSectionCol.getColumnQualifier()), cols);
+ }
+
+ boolean foundLocalityGroups = false;
+ boolean foundLocalityGroupDef1 = false;
+ boolean foundLocalityGroupDef2 = false;
+ boolean foundFormatter = false;
+ Joiner j = Joiner.on(",");
+ Function<Text,String> textToString = new Function<Text,String>() {
+ @Override
+ public String apply(Text text) {
+ return text.toString();
+ }
+ };
+ for (Entry<String,String> p : tops.getProperties(ReplicationTable.NAME)) {
+ String key = p.getKey();
+ String val = p.getValue();
+ // STATUS_LG_NAME, STATUS_LG_COLFAMS, WORK_LG_NAME, WORK_LG_COLFAMS
+ if (key.equals(Property.TABLE_FORMATTER_CLASS.getKey()) && val.equals(StatusFormatter.class.getName())) {
+ // look for formatter
+ foundFormatter = true;
+ } else if (key.equals(Property.TABLE_LOCALITY_GROUPS.getKey()) && val.equals(j.join(ReplicationTable.LOCALITY_GROUPS.keySet()))) {
+ // look for locality groups enabled
+ foundLocalityGroups = true;
+ } else if (key.startsWith(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey())) {
+ // look for locality group column family definitions
+ if (key.equals(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + ReplicationTable.STATUS_LG_NAME)
+ && val.equals(j.join(Iterables.transform(ReplicationTable.STATUS_LG_COLFAMS, textToString)))) {
+ foundLocalityGroupDef1 = true;
+ } else if (key.equals(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() + ReplicationTable.WORK_LG_NAME)
+ && val.equals(j.join(Iterables.transform(ReplicationTable.WORK_LG_COLFAMS, textToString)))) {
+ foundLocalityGroupDef2 = true;
+ }
+ }
+ }
+ Assert.assertTrue(foundLocalityGroups);
+ Assert.assertTrue(foundLocalityGroupDef1);
+ Assert.assertTrue(foundLocalityGroupDef2);
+ Assert.assertTrue(foundFormatter);
+ }
+
+ @Test
+ public void correctRecordsCompleteFile() throws Exception {
+ Connector conn = getConnector();
+ String table = "table1";
+ conn.tableOperations().create(table);
+ // If we have more than one tserver, this is subject to a race condition.
+ conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
+
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ for (int i = 0; i < 10; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put(new byte[0], new byte[0], new byte[0]);
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // After writing data, we'll get a replication table online
+ boolean online = ReplicationTable.isOnline(conn);
+ int attempts = 10;
+ do {
+ if (!online) {
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ online = ReplicationTable.isOnline(conn);
+ attempts--;
+ }
+ } while (!online && attempts > 0);
+ Assert.assertTrue("Replication table was not online", online);
+
+ for (int i = 0; i < 5; i++) {
+ if (conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ)) {
+ break;
+ }
+ log.info("Could not read replication table, waiting and will retry");
+ Thread.sleep(2000);
+ }
+
+ Assert.assertTrue("'root' user could not read the replication table",
+ conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
+
+ Set<String> replRows = Sets.newHashSet();
+ Scanner scanner;
+ attempts = 5;
+ while (replRows.isEmpty() && attempts > 0) {
+ scanner = ReplicationTable.getScanner(conn);
+ StatusSection.limit(scanner);
+ for (Entry<Key,Value> entry : scanner) {
+ Key k = entry.getKey();
+
+ String fileUri = k.getRow().toString();
+ try {
+ new URI(fileUri);
+ } catch (URISyntaxException e) {
+ Assert.fail("Expected a valid URI: " + fileUri);
+ }
+
+ replRows.add(fileUri);
+ }
+ }
+
+ Set<String> wals = Sets.newHashSet();
+ attempts = 5;
+ Instance i = conn.getInstance();
+ ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
+ while (wals.isEmpty() && attempts > 0) {
+ WalStateManager markers = new WalStateManager(i, zk);
+ for (Entry<Path,WalState> entry : markers.getAllState().entrySet()) {
+ wals.add(entry.getKey().toString());
+ }
+ attempts--;
+ }
+
+ // We only have one file that should need replication (no trace table)
+ // We should find an entry in tablet and in the repl row
+ Assert.assertEquals("Rows found: " + replRows, 1, replRows.size());
+
+ // There should only be one extra WALog that replication doesn't know about
+ replRows.removeAll(wals);
+ Assert.assertEquals(2, wals.size());
+ Assert.assertEquals(0, replRows.size());
+ }
+
+ @Test
+ public void noRecordsWithoutReplication() throws Exception {
+ Connector conn = getConnector();
+ List<String> tables = new ArrayList<>();
+
+ // replication shouldn't be online when we begin
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ for (int i = 0; i < 5; i++) {
+ String name = "table" + i;
+ tables.add(name);
+ conn.tableOperations().create(name);
+ }
+
+ // nor after we create some tables (that aren't being replicated)
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ for (String table : tables) {
+ writeSomeData(conn, table, 5, 5);
+ }
+
+ // After writing data, still no replication table
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ for (String table : tables) {
+ conn.tableOperations().compact(table, null, null, true, true);
+ }
+
+ // After compacting data, still no replication table
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ for (String table : tables) {
+ conn.tableOperations().delete(table);
+ }
+
+ // After deleting tables, still no replication table
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+ }
+
+ @Test
+ public void twoEntriesForTwoTables() throws Exception {
+ Connector conn = getConnector();
+ String table1 = "table1", table2 = "table2";
+
+ // replication shouldn't exist when we begin
+ Assert.assertFalse("Replication table already online at the beginning of the test", ReplicationTable.isOnline(conn));
+
+ // Create two tables
+ conn.tableOperations().create(table1);
+ conn.tableOperations().create(table2);
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.READ);
+ // wait for permission to propagate
+ Thread.sleep(5000);
+
+ // Enable replication on table1
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+
+ // Despite having replication on, we shouldn't have any need to write a record to it (and bring it online)
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ // Write some data to table1
+ writeSomeData(conn, table1, 50, 50);
+
+ // After the commit for these mutations finishes, we'll get a replication entry in accumulo.metadata for table1
+ // Don't want to compact table1 as it ultimately cause the entry in accumulo.metadata to be removed before we can verify it's there
+
+ // After writing data, we'll get a replication table online
+ boolean online = ReplicationTable.isOnline(conn);
+ int attempts = 10;
+ do {
+ if (!online) {
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ online = ReplicationTable.isOnline(conn);
+ attempts--;
+ }
+ } while (!online && attempts > 0);
+ Assert.assertTrue("Replication table did not exist", online);
+
+ Assert.assertTrue(ReplicationTable.isOnline(conn));
+
+ // Verify that we found a single replication record that's for table1
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ for (int i = 0; i < 5; i++) {
+ if (Iterators.size(s.iterator()) == 1) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ Entry<Key,Value> entry = Iterators.getOnlyElement(s.iterator());
+ // We should at least find one status record for this table, we might find a second if another log was started from ingesting the data
+ Assert.assertEquals("Expected to find replication entry for " + table1, conn.tableOperations().tableIdMap().get(table1), entry.getKey()
+ .getColumnQualifier().toString());
+ s.close();
+
+ // Enable replication on table2
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
+
+ // Write some data to table2
+ writeSomeData(conn, table2, 50, 50);
+
+ // After the commit on these mutations, we'll get a replication entry in accumulo.metadata for table2
+ // Don't want to compact table2 as it ultimately cause the entry in accumulo.metadata to be removed before we can verify it's there
+
+ Set<String> tableIds = Sets.newHashSet(conn.tableOperations().tableIdMap().get(table1), conn.tableOperations().tableIdMap().get(table2));
+ Set<String> tableIdsForMetadata = Sets.newHashSet(tableIds);
+
+ List<Entry<Key,Value>> records = new ArrayList<>();
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(MetadataSchema.ReplicationSection.getRange());
+ for (Entry<Key,Value> metadata : s) {
+ records.add(metadata);
+ log.debug("Meta: {} => {}", metadata.getKey().toStringNoTruncate(), metadata.getValue().toString());
+ }
+
+ Assert.assertEquals("Expected to find 2 records, but actually found " + records, 2, records.size());
+
+ for (Entry<Key,Value> metadata : records) {
+ Assert.assertTrue("Expected record to be in metadata but wasn't " + metadata.getKey().toStringNoTruncate() + ", tableIds remaining "
+ + tableIdsForMetadata, tableIdsForMetadata.remove(metadata.getKey().getColumnQualifier().toString()));
+ }
+
+ Assert.assertTrue("Expected that we had removed all metadata entries " + tableIdsForMetadata, tableIdsForMetadata.isEmpty());
+
+ // Should be creating these records in replication table from metadata table every second
+ Thread.sleep(5000);
+
+ // Verify that we found two replication records: one for table1 and one for table2
+ s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Iterator<Entry<Key,Value>> iter = s.iterator();
+ Assert.assertTrue("Found no records in replication table", iter.hasNext());
+ entry = iter.next();
+ Assert.assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
+ Assert.assertTrue("Expected to find two elements in replication table, only found one ", iter.hasNext());
+ entry = iter.next();
+ Assert.assertTrue("Expected to find element in replication table", tableIds.remove(entry.getKey().getColumnQualifier().toString()));
+ Assert.assertFalse("Expected to only find two elements in replication table", iter.hasNext());
+ }
+
+ private void writeSomeData(Connector conn, String table, int rows, int cols) throws Exception {
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ for (int row = 0; row < rows; row++) {
+ Mutation m = new Mutation(Integer.toString(row));
+ for (int col = 0; col < cols; col++) {
+ String value = Integer.toString(col);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+
+ @Test
+ public void replicationEntriesPrecludeWalDeletion() throws Exception {
+ final Connector conn = getConnector();
+ String table1 = "table1", table2 = "table2", table3 = "table3";
+ final Multimap<String,String> logs = HashMultimap.create();
+ final AtomicBoolean keepRunning = new AtomicBoolean(true);
+
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ // Should really be able to interrupt here, but the Scanner throws a fit to the logger
+ // when that happens
+ while (keepRunning.get()) {
+ try {
+ logs.putAll(getAllLogs(conn));
+ } catch (Exception e) {
+ log.error("Error getting logs", e);
+ }
+ }
+ }
+
+ });
+
+ t.start();
+
+ conn.tableOperations().create(table1);
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ Thread.sleep(2000);
+
+ // Write some data to table1
+ writeSomeData(conn, table1, 200, 500);
+
+ conn.tableOperations().create(table2);
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ Thread.sleep(2000);
+
+ writeSomeData(conn, table2, 200, 500);
+
+ conn.tableOperations().create(table3);
+ conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ Thread.sleep(2000);
+
+ writeSomeData(conn, table3, 200, 500);
+
+ // Force a write to metadata for the data written
+ for (String table : Arrays.asList(table1, table2, table3)) {
+ conn.tableOperations().flush(table, null, null, true);
+ }
+
+ keepRunning.set(false);
+ t.join(5000);
+
+ // The master is only running every second to create records in the replication table from the metadata table
+ // Sleep a sufficient amount of time to ensure that we get the straggling WALs that might have been created at the end
+ Thread.sleep(5000);
+
+ Set<String> replFiles = getReferencesToFilesToBeReplicated(conn);
+
+ // We might have a WAL that was use solely for the replication table
+ // We want to remove that from our list as it should not appear in the replication table
+ String replicationTableId = conn.tableOperations().tableIdMap().get(ReplicationTable.NAME);
+ Iterator<Entry<String,String>> observedLogs = logs.entries().iterator();
+ while (observedLogs.hasNext()) {
+ Entry<String,String> observedLog = observedLogs.next();
+ if (replicationTableId.equals(observedLog.getValue())) {
+ log.info("Removing {} because its tableId is for the replication table", observedLog);
+ observedLogs.remove();
+ }
+ }
+
+ // We should have *some* reference to each log that was seen in the metadata table
+ // They might not yet all be closed though (might be newfile)
+ Assert.assertTrue("Metadata log distribution: " + logs + "replFiles " + replFiles, logs.keySet().containsAll(replFiles));
+ Assert.assertTrue("Difference between replication entries and current logs is bigger than one", logs.keySet().size() - replFiles.size() <= 1);
+
+ final Configuration conf = new Configuration();
+ for (String replFile : replFiles) {
+ Path p = new Path(replFile);
+ FileSystem fs = p.getFileSystem(conf);
+ if (!fs.exists(p)) {
+ // double-check: the garbage collector can be fast
+ Set<String> currentSet = getReferencesToFilesToBeReplicated(conn);
+ log.info("Current references {}", currentSet);
+ log.info("Looking for reference to {}", replFile);
+ log.info("Contains? {}", currentSet.contains(replFile));
+ Assert.assertTrue("File does not exist anymore, it was likely incorrectly garbage collected: " + p, !currentSet.contains(replFile));
+ }
+ }
+ }
+
+ private Set<String> getReferencesToFilesToBeReplicated(final Connector conn) throws ReplicationTableOfflineException {
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Set<String> replFiles = new HashSet<>();
+ for (Entry<Key,Value> entry : s) {
+ replFiles.add(entry.getKey().getRow().toString());
+ }
+ return replFiles;
+ }
+
+ @Test
+ public void combinerWorksOnMetadata() throws Exception {
+ Connector conn = getConnector();
+
+ conn.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
+
+ ReplicationTableUtil.configureMetadataTable(conn, MetadataTable.NAME);
+
+ Status stat1 = StatusUtil.fileCreated(100);
+ Status stat2 = StatusUtil.fileClosed();
+
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ Mutation m = new Mutation(ReplicationSection.getRowPrefix() + "file:/accumulo/wals/tserver+port/uuid");
+ m.put(ReplicationSection.COLF, new Text("1"), ProtobufUtil.toValue(stat1));
+ bw.addMutation(m);
+ bw.close();
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(ReplicationSection.getRange());
+
+ Status actual = Status.parseFrom(Iterables.getOnlyElement(s).getValue().get());
+ Assert.assertEquals(stat1, actual);
+
+ bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ m = new Mutation(ReplicationSection.getRowPrefix() + "file:/accumulo/wals/tserver+port/uuid");
+ m.put(ReplicationSection.COLF, new Text("1"), ProtobufUtil.toValue(stat2));
+ bw.addMutation(m);
+ bw.close();
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(ReplicationSection.getRange());
+
+ actual = Status.parseFrom(Iterables.getOnlyElement(s).getValue().get());
+ Status expected = Status.newBuilder().setBegin(0).setEnd(0).setClosed(true).setInfiniteEnd(true).setCreatedTime(100).build();
+
+ Assert.assertEquals(expected, actual);
+ }
+
+ @Test
+ public void noDeadlock() throws Exception {
+ final Connector conn = getConnector();
+
+ ReplicationTable.setOnline(conn);
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE);
+ conn.tableOperations().deleteRows(ReplicationTable.NAME, null, null);
+
+ String table1 = "table1", table2 = "table2", table3 = "table3";
+ conn.tableOperations().create(table1);
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ conn.tableOperations().create(table2);
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ conn.tableOperations().create(table3);
+ conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+
+ writeSomeData(conn, table1, 200, 500);
+
+ writeSomeData(conn, table2, 200, 500);
+
+ writeSomeData(conn, table3, 200, 500);
+
+ // Flush everything to try to make the replication records
+ for (String table : Arrays.asList(table1, table2, table3)) {
+ conn.tableOperations().flush(table, null, null, true);
+ }
+
+ // Flush everything to try to make the replication records
+ for (String table : Arrays.asList(table1, table2, table3)) {
+ conn.tableOperations().flush(table, null, null, true);
+ }
+
+ for (String table : Arrays.asList(MetadataTable.NAME, table1, table2, table3)) {
+ Iterators.size(conn.createScanner(table, Authorizations.EMPTY).iterator());
+ }
+ }
+
+ @Test
+ public void filesClosedAfterUnused() throws Exception {
+ Connector conn = getConnector();
+
+ String table = "table";
+ conn.tableOperations().create(table);
+ String tableId = conn.tableOperations().tableIdMap().get(table);
+
+ Assert.assertNotNull(tableId);
+
+ conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ // just sleep
+ conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
+ ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
+
+ // Write a mutation to make a log file
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("one");
+ m.put("", "", "");
+ bw.addMutation(m);
+ bw.close();
+
+ // Write another to make sure the logger rolls itself?
+ bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ m = new Mutation("three");
+ m.put("", "", "");
+ bw.addMutation(m);
+ bw.close();
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(TabletsSection.LogColumnFamily.NAME);
+ s.setRange(TabletsSection.getRange(tableId));
+ Set<String> wals = new HashSet<>();
+ for (Entry<Key,Value> entry : s) {
+ LogEntry logEntry = LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
+ wals.add(new Path(logEntry.filename).toString());
+ }
+
+ log.warn("Found wals {}", wals);
+
+ bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ m = new Mutation("three");
+ byte[] bytes = new byte[1024 * 1024];
+ m.put("1".getBytes(), new byte[0], bytes);
+ m.put("2".getBytes(), new byte[0], bytes);
+ m.put("3".getBytes(), new byte[0], bytes);
+ m.put("4".getBytes(), new byte[0], bytes);
+ m.put("5".getBytes(), new byte[0], bytes);
+ bw.addMutation(m);
+ bw.close();
+
+ conn.tableOperations().flush(table, null, null, true);
+
+ while (!ReplicationTable.isOnline(conn)) {
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ }
+
+ for (int i = 0; i < 10; i++) {
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(LogColumnFamily.NAME);
+ s.setRange(TabletsSection.getRange(tableId));
+ for (Entry<Key,Value> entry : s) {
+ log.info(entry.getKey().toStringNoTruncate() + "=" + entry.getValue());
+ }
+
+ try {
+ s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Text buff = new Text();
+ boolean allReferencedLogsClosed = true;
+ int recordsFound = 0;
+ for (Entry<Key,Value> e : s) {
+ recordsFound++;
+ allReferencedLogsClosed = true;
+ StatusSection.getFile(e.getKey(), buff);
+ String file = buff.toString();
+ if (wals.contains(file)) {
+ Status stat = Status.parseFrom(e.getValue().get());
+ if (!stat.getClosed()) {
+ log.info("{} wasn't closed", file);
+ allReferencedLogsClosed = false;
+ }
+ }
+ }
+
+ if (recordsFound > 0 && allReferencedLogsClosed) {
+ return;
+ }
+ Thread.sleep(2000);
+ } catch (RuntimeException e) {
+ Throwable cause = e.getCause();
+ if (cause instanceof AccumuloSecurityException) {
+ AccumuloSecurityException ase = (AccumuloSecurityException) cause;
+ switch (ase.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ // We tried to read the replication table before the GRANT went through
+ Thread.sleep(2000);
+ break;
+ default:
+ throw e;
+ }
+ }
+ }
+ }
+
+ Assert.fail("We had a file that was referenced but didn't get closed");
+ }
+
+ @Test
+ public void singleTableWithSingleTarget() throws Exception {
+ // We want to kill the GC so it doesn't come along and close Status records and mess up the comparisons
+ // against expected Status messages.
+ getCluster().getClusterControl().stop(ServerType.GARBAGE_COLLECTOR);
+
+ Connector conn = getConnector();
+ String table1 = "table1";
+
+ // replication shouldn't be online when we begin
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ // Create a table
+ conn.tableOperations().create(table1);
+
+ int attempts = 10;
+
+ // Might think the table doesn't yet exist, retry
+ while (attempts > 0) {
+ try {
+ // Enable replication on table1
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+ // Replicate table1 to cluster1 in the table with id of '4'
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "4");
+ // Sleep for 100 seconds before saying something is replicated
+ conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
+ ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "100000"));
+ break;
+ } catch (Exception e) {
+ attempts--;
+ if (attempts <= 0) {
+ throw e;
+ }
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ }
+ }
+
+ // Write some data to table1
+ writeSomeData(conn, table1, 2000, 50);
+
+ // Make sure the replication table is online at this point
+ boolean online = ReplicationTable.isOnline(conn);
+ attempts = 10;
+ do {
+ if (!online) {
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ online = ReplicationTable.isOnline(conn);
+ attempts--;
+ }
+ } while (!online && attempts > 0);
+ Assert.assertTrue("Replication table was never created", online);
+
+ // ACCUMULO-2743 The Observer in the tserver has to be made aware of the change to get the combiner (made by the master)
+ for (int i = 0; i < 10 && !conn.tableOperations().listIterators(ReplicationTable.NAME).keySet().contains(ReplicationTable.COMBINER_NAME); i++) {
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ }
+
+ Assert.assertTrue("Combiner was never set on replication table",
+ conn.tableOperations().listIterators(ReplicationTable.NAME).keySet().contains(ReplicationTable.COMBINER_NAME));
+
+ // Trigger the minor compaction, waiting for it to finish.
+ // This should write the entry to metadata that the file has data
+ conn.tableOperations().flush(table1, null, null, true);
+
+ // Make sure that we have one status element, should be a new file
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Entry<Key,Value> entry = null;
+ Status expectedStatus = StatusUtil.openWithUnknownLength();
+ attempts = 10;
+ // This record will move from new to new with infinite length because of the minc (flush)
+ while (null == entry && attempts > 0) {
+ try {
+ entry = Iterables.getOnlyElement(s);
+ Status actual = Status.parseFrom(entry.getValue().get());
+ if (actual.getInfiniteEnd() != expectedStatus.getInfiniteEnd()) {
+ entry = null;
+ // the master process didn't yet fire and write the new mutation, wait for it to do
+ // so and try to read it again
+ Thread.sleep(1000);
+ }
+ } catch (NoSuchElementException e) {
+ entry = null;
+ Thread.sleep(500);
+ } catch (IllegalArgumentException e) {
+ // saw this contain 2 elements once
+ s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
+ }
+ throw e;
+ } finally {
+ attempts--;
+ }
+ }
+
+ Assert.assertNotNull("Could not find expected entry in replication table", entry);
+ Status actual = Status.parseFrom(entry.getValue().get());
+ Assert.assertTrue("Expected to find a replication entry that is open with infinite length: " + ProtobufUtil.toString(actual),
+ !actual.getClosed() && actual.getInfiniteEnd());
+
+ // Try a couple of times to watch for the work record to be created
+ boolean notFound = true;
+ for (int i = 0; i < 10 && notFound; i++) {
+ s = ReplicationTable.getScanner(conn);
+ WorkSection.limit(s);
+ int elementsFound = Iterables.size(s);
+ if (0 < elementsFound) {
+ Assert.assertEquals(1, elementsFound);
+ notFound = false;
+ }
+ Thread.sleep(500);
+ }
+
+ // If we didn't find the work record, print the contents of the table
+ if (notFound) {
+ s = ReplicationTable.getScanner(conn);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
+ }
+ Assert.assertFalse("Did not find the work entry for the status entry", notFound);
+ }
+
+ // Write some more data so that we over-run the single WAL
+ writeSomeData(conn, table1, 3000, 50);
+
+ log.info("Issued compaction for table");
+ conn.tableOperations().compact(table1, null, null, true, true);
+ log.info("Compaction completed");
+
+ // Master is creating entries in the replication table from the metadata table every second.
+ // Compaction should trigger the record to be written to metadata. Wait a bit to ensure
+ // that the master has time to work.
+ Thread.sleep(5000);
+
+ s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ int numRecords = 0;
+ for (Entry<Key,Value> e : s) {
+ numRecords++;
+ log.info("Found status record {}\t{}", e.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(e.getValue().get())));
+ }
+
+ Assert.assertEquals(2, numRecords);
+
+ // We should eventually get 2 work records recorded, need to account for a potential delay though
+ // might see: status1 -> work1 -> status2 -> (our scans) -> work2
+ notFound = true;
+ for (int i = 0; i < 10 && notFound; i++) {
+ s = ReplicationTable.getScanner(conn);
+ WorkSection.limit(s);
+ int elementsFound = Iterables.size(s);
+ if (2 == elementsFound) {
+ notFound = false;
+ }
+ Thread.sleep(500);
+ }
+
+ // If we didn't find the work record, print the contents of the table
+ if (notFound) {
+ s = ReplicationTable.getScanner(conn);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
+ }
+ Assert.assertFalse("Did not find the work entries for the status entries", notFound);
+ }
+ }
+
+ @Test
+ public void correctClusterNameInWorkEntry() throws Exception {
+ Connector conn = getConnector();
+ String table1 = "table1";
+
+ // replication shouldn't be online when we begin
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ // Create two tables
+ conn.tableOperations().create(table1);
+
+ int attempts = 5;
+ while (attempts > 0) {
+ try {
+ // Enable replication on table1
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+ // Replicate table1 to cluster1 in the table with id of '4'
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "4");
+ attempts = 0;
+ } catch (Exception e) {
+ attempts--;
+ if (attempts <= 0) {
+ throw e;
+ }
+ sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ // Write some data to table1
+ writeSomeData(conn, table1, 2000, 50);
+ conn.tableOperations().flush(table1, null, null, true);
+
+ String tableId = conn.tableOperations().tableIdMap().get(table1);
+ Assert.assertNotNull("Table ID was null", tableId);
+
+ // Make sure the replication table exists at this point
+ boolean online = ReplicationTable.isOnline(conn);
+ attempts = 5;
+ do {
+ if (!online) {
+ sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+ online = ReplicationTable.isOnline(conn);
+ attempts--;
+ }
+ } while (!online && attempts > 0);
+ Assert.assertTrue("Replication table did not exist", online);
+
+ for (int i = 0; i < 5 && !conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ); i++) {
+ Thread.sleep(1000);
+ }
+
+ Assert.assertTrue(conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
+
+ boolean notFound = true;
+ Scanner s;
+ for (int i = 0; i < 10 && notFound; i++) {
+ s = ReplicationTable.getScanner(conn);
+ WorkSection.limit(s);
+ try {
+ Entry<Key,Value> e = Iterables.getOnlyElement(s);
+ Text expectedColqual = new ReplicationTarget("cluster1", "4", tableId).toText();
+ Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier());
+ notFound = false;
+ } catch (NoSuchElementException e) {} catch (IllegalArgumentException e) {
+ s = ReplicationTable.getScanner(conn);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
+ }
+ Assert.fail("Found more than one work section entry");
+ }
+
+ Thread.sleep(500);
+ }
+
+ if (notFound) {
+ s = ReplicationTable.getScanner(conn);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
+ }
+ Assert.assertFalse("Did not find the work entry for the status entry", notFound);
+ }
+ }
+
+ @Test
+ public void replicationRecordsAreClosedAfterGarbageCollection() throws Exception {
+ getCluster().getClusterControl().stop(ServerType.GARBAGE_COLLECTOR);
+
+ final Connector conn = getConnector();
+
+ ReplicationTable.setOnline(conn);
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE);
+ conn.tableOperations().deleteRows(ReplicationTable.NAME, null, null);
+
+ final AtomicBoolean keepRunning = new AtomicBoolean(true);
+ final Set<String> metadataWals = new HashSet<>();
+
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ // Should really be able to interrupt here, but the Scanner throws a fit to the logger
+ // when that happens
+ while (keepRunning.get()) {
+ try {
+ metadataWals.addAll(getLogs(conn).keySet());
+ } catch (Exception e) {
+ log.error("Metadata table doesn't exist");
+ }
+ }
+ }
+
+ });
+
+ t.start();
+
+ String table1 = "table1", table2 = "table2", table3 = "table3";
+
+ try {
+ conn.tableOperations().create(table1);
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+ conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
+ ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, null));
+
+ // Write some data to table1
+ writeSomeData(conn, table1, 200, 500);
+
+ conn.tableOperations().create(table2);
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+
+ writeSomeData(conn, table2, 200, 500);
+
+ conn.tableOperations().create(table3);
+ conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION.getKey(), "true");
+ conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
+
+ writeSomeData(conn, table3, 200, 500);
+
+ // Flush everything to try to make the replication records
+ for (String table : Arrays.asList(table1, table2, table3)) {
+ conn.tableOperations().compact(table, null, null, true, true);
+ }
+ } finally {
+ keepRunning.set(false);
+ t.join(5000);
+ Assert.assertFalse(t.isAlive());
+ }
+
+ // Kill the tserver(s) and restart them
+ // to ensure that the WALs we previously observed all move to closed.
+ cluster.getClusterControl().stop(ServerType.TABLET_SERVER);
+ cluster.getClusterControl().start(ServerType.TABLET_SERVER);
+
+ // Make sure we can read all the tables (recovery complete)
+ for (String table : Arrays.asList(table1, table2, table3)) {
+ Iterators.size(conn.createScanner(table, Authorizations.EMPTY).iterator());
+ }
+
+ // Starting the gc will run CloseWriteAheadLogReferences which will first close Statuses
+ // in the metadata table, and then in the replication table
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+
+ waitForGCLock(conn);
+
+ Thread.sleep(1000);
+
+ log.info("GC is up and should have had time to run at least once by now");
+
+ try {
+ boolean allClosed = true;
+
+ // We should either find all closed records or no records
+ // After they're closed, they are candidates for deletion
+ for (int i = 0; i < 10; i++) {
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(Range.prefix(ReplicationSection.getRowPrefix()));
+ Iterator<Entry<Key,Value>> iter = s.iterator();
+
+ long recordsFound = 0l;
+ while (allClosed && iter.hasNext()) {
+ Entry<Key,Value> entry = iter.next();
+ String wal = entry.getKey().getRow().toString();
+ if (metadataWals.contains(wal)) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ log.info("{}={}", entry.getKey().toStringNoTruncate(), ProtobufUtil.toString(status));
+ allClosed &= status.getClosed();
+ recordsFound++;
+ }
+ }
+
+ log.info("Found {} records from the metadata table", recordsFound);
+ if (allClosed) {
+ break;
+ }
+
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+ }
+
+ if (!allClosed) {
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(Range.prefix(ReplicationSection.getRowPrefix()));
+ for (Entry<Key,Value> entry : s) {
+ log.info(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
+ }
+ Assert.fail("Expected all replication records in the metadata table to be closed");
+ }
+
+ for (int i = 0; i < 10; i++) {
+ allClosed = true;
+
+ Scanner s = ReplicationTable.getScanner(conn);
+ Iterator<Entry<Key,Value>> iter = s.iterator();
+
+ long recordsFound = 0l;
+ while (allClosed && iter.hasNext()) {
+ Entry<Key,Value> entry = iter.next();
+ String wal = entry.getKey().getRow().toString();
+ if (metadataWals.contains(wal)) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ log.info("{}={}", entry.getKey().toStringNoTruncate(), ProtobufUtil.toString(status));
+ allClosed &= status.getClosed();
+ recordsFound++;
+ }
+ }
+
+ log.info("Found {} records from the replication table", recordsFound);
+ if (allClosed) {
+ break;
+ }
+
+ sleepUninterruptibly(3, TimeUnit.SECONDS);
+ }
+
+ if (!allClosed) {
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ log.info(entry.getKey().toStringNoTruncate() + " " + TextFormat.shortDebugString(Status.parseFrom(entry.getValue().get())));
+ }
+ Assert.fail("Expected all replication records in the replication table to be closed");
+ }
+
+ } finally {
+ gc.destroy();
+ gc.waitFor();
+ }
+
+ }
+
+ @Test
+ public void replicatedStatusEntriesAreDeleted() throws Exception {
+ // Just stop it now, we'll restart it after we restart the tserver
+ getCluster().getClusterControl().stop(ServerType.GARBAGE_COLLECTOR);
+
+ final Connector conn = getConnector();
+ log.info("Got connector to MAC");
+ String table1 = "table1";
+
+ // replication shouldn't be online when we begin
+ Assert.assertFalse(ReplicationTable.isOnline(conn));
+
+ // Create two tables
+ conn.tableOperations().create(table1);
+
+ int attempts = 5;
+ while (attempts > 0) {
+ try {
+ // Enable replication on table1
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
+ // Replicate table1 to cluster1 in the table with id of '4'
+ conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "4");
+ // Use the MockReplicaSystem impl and sleep for 5seconds
+ conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1",
+ ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "1000"));
+ attempts = 0;
+ } catch (Exception e) {
+ attempts--;
+ if (attempts <= 0) {
+ throw e;
+ }
+ sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ String tableId = conn.tableOperations().tableIdMap().get(table1);
+ Assert.assertNotNull("Could not determine table id for " + table1, tableId);
+
+ // Write some data to table1
+ writeSomeData(conn, table1, 2000, 50);
+ conn.tableOperations().flush(table1, null, null, true);
+
+ // Make sure the replication table exists at this point
+ boolean online = ReplicationTable.isOnline(conn);
+ attempts = 10;
+ do {
+ if (!online) {
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ online = ReplicationTable.isOnline(conn);
+ attempts--;
+ }
+ } while (!online && attempts > 0);
+ Assert.assertTrue("Replication table did not exist", online);
+
+ // Grant ourselves the write permission for later
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE);
+
+ log.info("Checking for replication entries in replication");
+ // Then we need to get those records over to the replication table
+ Scanner s;
+ Set<String> entries = new HashSet<>();
+ for (int i = 0; i < 5; i++) {
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(ReplicationSection.getRange());
+ entries.clear();
+ for (Entry<Key,Value> entry : s) {
+ entries.add(entry.getKey().getRow().toString());
+ log.info("{}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
+ }
+ if (!entries.isEmpty()) {
+ log.info("Replication entries {}", entries);
+ break;
+ }
+ Thread.sleep(1000);
+ }
+
+ Assert.assertFalse("Did not find any replication entries in the replication table", entries.isEmpty());
+
+ // Find the WorkSection record that will be created for that data we ingested
+ boolean notFound = true;
+ for (int i = 0; i < 10 && notFound; i++) {
+ try {
+ s = ReplicationTable.getScanner(conn);
+ WorkSection.limit(s);
+ Entry<Key,Value> e = Iterables.getOnlyElement(s);
+ log.info("Found entry: " + e.getKey().toStringNoTruncate());
+ Text expectedColqual = new ReplicationTarget("cluster1", "4", tableId).toText();
+ Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier());
+ notFound = false;
+ } catch (NoSuchElementException e) {
+
+ } catch (IllegalArgumentException e) {
+ // Somehow we got more than one element. Log what they were
+ s = ReplicationTable.getScanner(conn);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
+ }
+ Assert.fail("Found more than one work section entry");
+ } catch (RuntimeException e) {
+ // Catch a propagation issue, fail if it's not what we expect
+ Throwable cause = e.getCause();
+ if (cause instanceof AccumuloSecurityException) {
+ AccumuloSecurityException sec = (AccumuloSecurityException) cause;
+ switch (sec.getSecurityErrorCode()) {
+ case PERMISSION_DENIED:
+ // retry -- the grant didn't happen yet
+ log.warn("Sleeping because permission was denied");
+ break;
+ default:
+ throw e;
+ }
+ } else {
+ throw e;
+ }
+ }
+
+ Thread.sleep(2000);
+ }
+
+ if (notFound) {
+ s = ReplicationTable.getScanner(conn);
+ for (Entry<Key,Value> content : s) {
+ log.info(content.getKey().toStringNoTruncate() + " => " + ProtobufUtil.toString(Status.parseFrom(content.getValue().get())));
+ }
+ Assert.assertFalse("Did not find the work entry for the status entry", notFound);
+ }
+
+ /**
+ * By this point, we should have data ingested into a table, with at least one WAL as a candidate for replication. Compacting the table should close all
+ * open WALs, which should ensure all records we're going to replicate have entries in the replication table, and nothing will exist in the metadata table
+ * anymore
+ */
+
+ log.info("Killing tserver");
+ // Kill the tserver(s) and restart them
+ // to ensure that the WALs we previously observed all move to closed.
+ cluster.getClusterControl().stop(ServerType.TABLET_SERVER);
+
+ log.info("Starting tserver");
+ cluster.getClusterControl().start(ServerType.TABLET_SERVER);
+
+ log.info("Waiting to read tables");
+ sleepUninterruptibly(2 * 3, TimeUnit.SECONDS);
+
+ // Make sure we can read all the tables (recovery complete)
+ for (String table : new String[] {MetadataTable.NAME, table1}) {
+ Iterators.size(conn.createScanner(table, Authorizations.EMPTY).iterator());
+ }
+
+ log.info("Recovered metadata:");
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : s) {
+ log.info("{}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
+ }
+
+ cluster.getClusterControl().start(ServerType.GARBAGE_COLLECTOR);
+
+ // Wait for a bit since the GC has to run (should be running after a one second delay)
+ waitForGCLock(conn);
+
+ Thread.sleep(1000);
+
+ log.info("After GC");
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ for (Entry<Key,Value> entry : s) {
+ log.info("{}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
+ }
+
+ // We expect no records in the metadata table after compaction. We have to poll
+ // because we have to wait for the StatusMaker's next iteration which will clean
+ // up the dangling *closed* records after we create the record in the replication table.
+ // We need the GC to close the file (CloseWriteAheadLogReferences) before we can remove the record
+ log.info("Checking metadata table for replication entries");
+ Set<String> remaining = new HashSet<>();
+ for (int i = 0; i < 10; i++) {
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(ReplicationSection.getRange());
+ remaining.clear();
+ for (Entry<Key,Value> e : s) {
+ remaining.add(e.getKey().getRow().toString());
+ }
+ remaining.retainAll(entries);
+ if (remaining.isEmpty()) {
+ break;
+ }
+ log.info("remaining {}", remaining);
+ Thread.sleep(2000);
+ log.info("");
+ }
+
+ Assert.assertTrue("Replication status messages were not cleaned up from metadata table", remaining.isEmpty());
+
+ /**
+ * After we close out and subsequently delete the metadata record, this will propagate to the replication table, which will cause those records to be
+ * deleted after replication occurs
+ */
+
+ int recordsFound = 0;
+ for (int i = 0; i < 30; i++) {
+ s = ReplicationTable.getScanner(conn);
+ recordsFound = 0;
+ for (Entry<Key,Value> entry : s) {
+ recordsFound++;
+ log.info("{} {}", entry.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
+ }
+
+ if (recordsFound <= 2) {
+ break;
+ } else {
+ Thread.sleep(1000);
+ log.info("");
+ }
+ }
+
+ Assert.assertTrue("Found unexpected replication records in the replication table", recordsFound <= 2);
+ }
+}
[03/13] accumulo git commit: ACCUMULO-4108 Increase zk timeout from
5s to 15s for ITs.
Posted by el...@apache.org.
ACCUMULO-4108 Increase zk timeout from 5s to 15s for ITs.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/29c6052d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/29c6052d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/29c6052d
Branch: refs/heads/master
Commit: 29c6052d64de0705729bf32136e44505e0c788cc
Parents: 46ad836
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 12 12:09:49 2016 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jan 12 12:09:49 2016 -0500
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java | 2 +-
test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 2 +-
.../test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/MasterFailoverIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/RestartIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/RestartStressIT.java | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java b/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
index eff7251..8a0086b 100644
--- a/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
+++ b/test/src/test/java/org/apache/accumulo/test/Accumulo3010IT.java
@@ -48,7 +48,7 @@ public class Accumulo3010IT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// file system supports recovery
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java b/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
index d55e427..323888a 100644
--- a/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
@@ -56,7 +56,7 @@ public class ExistingMacIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
index 72dabdf..f1372dc 100644
--- a/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
@@ -59,7 +59,7 @@ public class MasterRepairsDualAssignmentIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "5s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java b/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
index 3b25655..116092b 100644
--- a/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
@@ -45,7 +45,7 @@ public class MultiTableRecoveryIT extends ConfigurableMacIT {
@Override
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
+ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
index 7634f10..0c2631f 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
@@ -35,7 +35,7 @@ public class MasterFailoverIT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s"));
+ cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s"));
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index 4e55ab4..b498412 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@ -67,7 +67,7 @@ public class RestartIT extends AccumuloClusterIT {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
props.put(Property.GC_CYCLE_DELAY.getKey(), "1s");
props.put(Property.GC_CYCLE_START.getKey(), "1s");
cfg.setSiteConfig(props);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/29c6052d/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index b965420..c4b3afd 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@ -55,7 +55,7 @@ public class RestartStressIT extends AccumuloClusterIT {
opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
- opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
cfg.setSiteConfig(opts);
cfg.useMiniDFS(true);
[05/13] accumulo git commit: Merge branch '1.6' into 1.7
Posted by el...@apache.org.
Merge branch '1.6' into 1.7
Conflicts:
test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/94f4a19c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/94f4a19c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/94f4a19c
Branch: refs/heads/master
Commit: 94f4a19c0d776f3a57e855863c34e9e3164b615b
Parents: 6562828 29c6052
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 12 12:25:34 2016 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jan 12 12:25:34 2016 -0500
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/CleanWalIT.java | 2 +-
.../java/org/apache/accumulo/test/DetectDeadTabletServersIT.java | 2 +-
test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 2 +-
.../test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java | 2 +-
.../test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java | 2 +-
.../test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/BinaryStressIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/CleanTmpIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/CompactionIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/DurabilityIT.java | 2 +-
.../org/apache/accumulo/test/functional/GarbageCollectorIT.java | 2 +-
.../org/apache/accumulo/test/functional/KerberosRenewalIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/MasterFailoverIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/RestartIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/RestartStressIT.java | 2 +-
.../org/apache/accumulo/test/functional/SessionDurabilityIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/WriteAheadLogIT.java | 2 +-
.../org/apache/accumulo/test/functional/ZookeeperRestartIT.java | 2 +-
.../accumulo/test/replication/MultiInstanceReplicationIT.java | 2 +-
.../test/replication/UnorderedWorkAssignerReplicationIT.java | 2 +-
22 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
index d754a14,b2298f7..08e3c09
--- a/test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
@@@ -57,10 -54,9 +57,10 @@@ public class CleanWalIT extends Accumul
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setNumTservers(1);
- cfg.useMiniDFS(true);
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Before
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
index 04c781b,1e65601..f7ee089
--- a/test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
@@@ -42,7 -41,7 +42,7 @@@ public class DetectDeadTabletServersIT
@Override
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
index 1a3c92f,0000000..57c2c34
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
@@@ -1,101 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloClusterIT;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+// Accumulo3010
+public class RecoveryCompactionsAreFlushesIT extends AccumuloClusterIT {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ // file system supports recovery
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test
+ public void test() throws Exception {
+ // create a table
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "100");
+ c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "3");
+ // create 3 flush files
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", new Value("v".getBytes()));
+ for (int i = 0; i < 3; i++) {
+ bw.addMutation(m);
+ bw.flush();
+ c.tableOperations().flush(tableName, null, null, true);
+ }
+ // create an unsaved mutation
+ bw.addMutation(m);
+ bw.close();
+
+ ClusterControl control = cluster.getClusterControl();
+
+ // kill the tablet servers
+ control.stopAllServers(ServerType.TABLET_SERVER);
+
+ // recover
+ control.startAllServers(ServerType.TABLET_SERVER);
+
+ // ensure the table is readable
+ Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator());
+
+ // ensure that the recovery was not a merging minor compaction
+ Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : s) {
+ String filename = entry.getKey().getColumnQualifier().toString();
+ String parts[] = filename.split("/");
+ Assert.assertFalse(parts[parts.length - 1].startsWith("M"));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
index 081ee85,0000000..68bd07b
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
@@@ -1,75 -1,0 +1,75 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+// ACCUMULO-2480
+public class TabletServerGivesUpIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.useMiniDFS(true);
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_TOLERATED_CREATION_FAILURES, "15");
+ cfg.setProperty(Property.TSERV_WALOG_TOLERATED_MAXIMUM_WAIT_DURATION, "0s");
+ }
+
+ @Test(timeout = 30 * 1000)
+ public void test() throws Exception {
+ final Connector conn = this.getConnector();
+ // Yes, there's a tabletserver
+ assertEquals(1, conn.instanceOperations().getTabletServers().size());
+ final String tableName = getUniqueNames(1)[0];
+ conn.tableOperations().create(tableName);
+ // Kill dfs
+ cluster.getMiniDfs().shutdown();
+ // ask the tserver to do something
+ final AtomicReference<Exception> ex = new AtomicReference<>();
+ Thread splitter = new Thread() {
+ @Override
+ public void run() {
+ try {
+ TreeSet<Text> splits = new TreeSet<>();
+ splits.add(new Text("X"));
+ conn.tableOperations().addSplits(tableName, splits);
+ } catch (Exception e) {
+ ex.set(e);
+ }
+ }
+ };
+ splitter.start();
+ // wait for the tserver to give up on writing to the WAL
+ while (conn.instanceOperations().getTabletServers().size() == 1) {
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
index 8338021,0000000..c318075
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
@@@ -1,107 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class VerifySerialRecoveryIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "20");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testSerializedRecovery() throws Exception {
+ // make a table with many splits
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 200; i++) {
+ splits.add(new Text(AssignmentThreadsIT.randomHex(8)));
+ }
+ c.tableOperations().addSplits(tableName, splits);
+ // load data to give the recovery something to do
+ BatchWriter bw = c.createBatchWriter(tableName, null);
+ for (int i = 0; i < 50000; i++) {
+ Mutation m = new Mutation(AssignmentThreadsIT.randomHex(8));
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ // kill the tserver
+ for (ProcessReference ref : getCluster().getProcesses().get(ServerType.TABLET_SERVER))
+ getCluster().killProcess(ServerType.TABLET_SERVER, ref);
+ final Process ts = cluster.exec(TabletServer.class);
+
+ // wait for recovery
+ Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator());
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ ts.waitFor();
+ String result = FunctionalTestUtils.readAll(cluster, TabletServer.class, ts);
+ for (String line : result.split("\n")) {
+ System.out.println(line);
+ }
+ // walk through the output, verifying that only a single normal recovery was running at one time
+ boolean started = false;
+ int recoveries = 0;
+ for (String line : result.split("\n")) {
+ // ignore metadata tables
+ if (line.contains("!0") || line.contains("+r"))
+ continue;
+ if (line.contains("Starting Write-Ahead Log")) {
+ assertFalse(started);
+ started = true;
+ recoveries++;
+ }
+ if (line.contains("Write-Ahead Log recovery complete")) {
+ assertTrue(started);
+ started = false;
+ }
+ }
+ assertFalse(started);
+ assertTrue(recoveries > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
index 62d8738,fbe504e..f1bca1b
--- a/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@@ -50,9 -51,10 +50,9 @@@ public class BinaryStressIT extends Acc
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
- Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.TSERV_MAXMEM.getKey(), "50K");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
- cfg.setSiteConfig(siteConfig);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_MAXMEM, "50K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
}
private String majcDelay, maxMem;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
index 921d661,65422c9..d03007e
--- a/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
@@@ -51,11 -53,12 +51,11 @@@ public class CleanTmpIT extends Configu
private static final Logger log = LoggerFactory.getLogger(CleanTmpIT.class);
@Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
- cfg.setSiteConfig(props);
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setNumTservers(1);
- cfg.useMiniDFS(true);
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
index 2fe5470,907d17d..818dbc4
--- a/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
@@@ -58,12 -51,11 +58,12 @@@ public class CompactionIT extends Accum
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
- Map<String,String> map = new HashMap<String,String>();
- map.put(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), "4");
- map.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- map.put(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "1");
- cfg.setSiteConfig(map);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN, "4");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+ cfg.setProperty(Property.TSERV_MAJC_MAXCONCURRENT, "1");
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
index 9a262a1,0000000..819347e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@@ -1,222 -1,0 +1,222 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class DurabilityIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(DurabilityIT.class);
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ }
+
+ static final long N = 100000;
+
+ private String[] init() throws Exception {
+ String[] tableNames = getUniqueNames(4);
+ Connector c = getConnector();
+ TableOperations tableOps = c.tableOperations();
+ createTable(tableNames[0]);
+ createTable(tableNames[1]);
+ createTable(tableNames[2]);
+ createTable(tableNames[3]);
+ // default is sync
+ tableOps.setProperty(tableNames[1], Property.TABLE_DURABILITY.getKey(), "flush");
+ tableOps.setProperty(tableNames[2], Property.TABLE_DURABILITY.getKey(), "log");
+ tableOps.setProperty(tableNames[3], Property.TABLE_DURABILITY.getKey(), "none");
+ return tableNames;
+ }
+
+ private void cleanup(String[] tableNames) throws Exception {
+ Connector c = getConnector();
+ for (String tableName : tableNames) {
+ c.tableOperations().delete(tableName);
+ }
+ }
+
+ private void createTable(String tableName) throws Exception {
+ TableOperations tableOps = getConnector().tableOperations();
+ tableOps.create(tableName);
+ }
+
+ @Test(timeout = 2 * 60 * 1000)
+ public void testWriteSpeed() throws Exception {
+ TableOperations tableOps = getConnector().tableOperations();
+ String tableNames[] = init();
+ // write some gunk, delete the table to keep that table from messing with the performance numbers of successive calls
+ // sync
+ long t0 = writeSome(tableNames[0], N);
+ tableOps.delete(tableNames[0]);
+ // flush
+ long t1 = writeSome(tableNames[1], N);
+ tableOps.delete(tableNames[1]);
+ // log
+ long t2 = writeSome(tableNames[2], N);
+ tableOps.delete(tableNames[2]);
+ // none
+ long t3 = writeSome(tableNames[3], N);
+ tableOps.delete(tableNames[3]);
+ System.out.println(String.format("sync %d flush %d log %d none %d", t0, t1, t2, t3));
+ assertTrue("flush should be faster than sync", t0 > t1);
+ assertTrue("log should be faster than flush", t1 > t2);
+ assertTrue("no durability should be faster than log", t2 > t3);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testSync() throws Exception {
+ String tableNames[] = init();
+ // sync table should lose nothing
+ writeSome(tableNames[0], N);
+ restartTServer();
+ assertEquals(N, readSome(tableNames[0]));
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testFlush() throws Exception {
+ String tableNames[] = init();
+ // flush table won't lose anything since we're not losing power/dfs
+ writeSome(tableNames[1], N);
+ restartTServer();
+ assertEquals(N, readSome(tableNames[1]));
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testLog() throws Exception {
+ String tableNames[] = init();
+ // we're probably going to lose something the the log setting
+ writeSome(tableNames[2], N);
+ restartTServer();
+ long numResults = readSome(tableNames[2]);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testNone() throws Exception {
+ String tableNames[] = init();
+ // probably won't get any data back without logging
+ writeSome(tableNames[3], N);
+ restartTServer();
+ long numResults = readSome(tableNames[3]);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testIncreaseDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ writeSome(tableName, N);
+ restartTServer();
+ long numResults = readSome(tableName);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ writeSome(tableName, N);
+ restartTServer();
+ assertTrue(N == readSome(tableName));
+ }
+
+ private static Map<String,String> map(Iterable<Entry<String,String>> entries) {
+ Map<String,String> result = new HashMap<String,String>();
+ for (Entry<String,String> entry : entries) {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ return result;
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testMetaDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.instanceOperations().setProperty(Property.TABLE_DURABILITY.getKey(), "none");
+ Map<String,String> props = map(c.tableOperations().getProperties(MetadataTable.NAME));
+ assertEquals("sync", props.get(Property.TABLE_DURABILITY.getKey()));
+ c.tableOperations().create(tableName);
+ props = map(c.tableOperations().getProperties(tableName));
+ assertEquals("none", props.get(Property.TABLE_DURABILITY.getKey()));
+ restartTServer();
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ private long readSome(String table) throws Exception {
+ return Iterators.size(getConnector().createScanner(table, Authorizations.EMPTY).iterator());
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+ private long writeSome(String table, long count) throws Exception {
+ int iterations = 5;
+ long[] attempts = new long[iterations];
+ for (int attempt = 0; attempt < iterations; attempt++) {
+ long now = System.currentTimeMillis();
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(table, null);
+ for (int i = 1; i < count + 1; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put("", "", "");
+ bw.addMutation(m);
+ if (i % (Math.max(1, count / 100)) == 0) {
+ bw.flush();
+ }
+ }
+ bw.close();
+ attempts[attempt] = System.currentTimeMillis() - now;
+ }
+ Arrays.sort(attempts);
+ log.info("Attempt durations: {}", Arrays.toString(attempts));
+ // Return the median duration
+ return attempts[2];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 132cbcc,d5b92cf..6ab0541
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@@ -80,13 -80,14 +80,13 @@@ public class GarbageCollectorIT extend
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
- Map<String,String> settings = new HashMap<String,String>();
- settings.put(Property.INSTANCE_SECRET.getKey(), OUR_SECRET);
- settings.put(Property.GC_CYCLE_START.getKey(), "1");
- settings.put(Property.GC_CYCLE_DELAY.getKey(), "1");
- settings.put(Property.GC_PORT.getKey(), "0");
- settings.put(Property.TSERV_MAXMEM.getKey(), "5K");
- settings.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- cfg.setSiteConfig(settings);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.INSTANCE_SECRET, OUR_SECRET);
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_PORT, "0");
+ cfg.setProperty(Property.TSERV_MAXMEM, "5K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
index 19908f6,0000000..28c1dfc
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
@@@ -1,188 -1,0 +1,188 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.cluster.ClusterUser;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloIT;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
+import org.apache.accumulo.harness.MiniClusterHarness;
+import org.apache.accumulo.harness.TestingKdc;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * MAC test which uses {@link MiniKdc} to simulate ta secure environment. Can be used as a sanity check for Kerberos/SASL testing.
+ */
+public class KerberosRenewalIT extends AccumuloIT {
+ private static final Logger log = LoggerFactory.getLogger(KerberosRenewalIT.class);
+
+ private static TestingKdc kdc;
+ private static String krbEnabledForITs = null;
+ private static ClusterUser rootUser;
+
+ private static final long TICKET_LIFETIME = 6 * 60 * 1000; // Anything less seems to fail when generating the ticket
+ private static final long TICKET_TEST_LIFETIME = 8 * 60 * 1000; // Run a test for 8 mins
+ private static final long TEST_DURATION = 9 * 60 * 1000; // The test should finish within 9 mins
+
+ @BeforeClass
+ public static void startKdc() throws Exception {
+ // 30s renewal time window
+ kdc = new TestingKdc(TestingKdc.computeKdcDir(), TestingKdc.computeKeytabDir(), TICKET_LIFETIME);
+ kdc.start();
+ krbEnabledForITs = System.getProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION);
+ if (null == krbEnabledForITs || !Boolean.parseBoolean(krbEnabledForITs)) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, "true");
+ }
+ rootUser = kdc.getRootUser();
+ }
+
+ @AfterClass
+ public static void stopKdc() throws Exception {
+ if (null != kdc) {
+ kdc.stop();
+ }
+ if (null != krbEnabledForITs) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, krbEnabledForITs);
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return (int) TEST_DURATION / 1000;
+ }
+
+ private MiniAccumuloClusterImpl mac;
+
+ @Before
+ public void startMac() throws Exception {
+ MiniClusterHarness harness = new MiniClusterHarness();
+ mac = harness.create(this, new PasswordToken("unused"), kdc, new MiniClusterConfigurationCallback() {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ Map<String,String> site = cfg.getSiteConfig();
- site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "10s");
++ site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ // Reduce the period just to make sure we trigger renewal fast
+ site.put(Property.GENERAL_KERBEROS_RENEWAL_PERIOD.getKey(), "5s");
+ cfg.setSiteConfig(site);
+ }
+
+ });
+
+ mac.getConfig().setNumTservers(1);
+ mac.start();
+ // Enabled kerberos auth
+ Configuration conf = new Configuration(false);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ @After
+ public void stopMac() throws Exception {
+ if (null != mac) {
+ mac.stop();
+ }
+ }
+
+ // Intentially setting the Test annotation timeout. We do not want to scale the timeout.
+ @Test(timeout = TEST_DURATION)
+ public void testReadAndWriteThroughTicketLifetime() throws Exception {
+ // Attempt to use Accumulo for a duration of time that exceeds the Kerberos ticket lifetime.
+ // This is a functional test to verify that Accumulo services renew their ticket.
+ // If the test doesn't finish on its own, this signifies that Accumulo services failed
+ // and the test should fail. If Accumulo services renew their ticket, the test case
+ // should exit gracefully on its own.
+
+ // Login as the "root" user
+ UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ long duration = 0;
+ long last = System.currentTimeMillis();
+ // Make sure we have a couple renewals happen
+ while (duration < TICKET_TEST_LIFETIME) {
+ // Create a table, write a record, compact, read the record, drop the table.
+ createReadWriteDrop(conn);
+ // Wait a bit after
+ Thread.sleep(5000);
+
+ // Update the duration
+ long now = System.currentTimeMillis();
+ duration += now - last;
+ last = now;
+ }
+ }
+
+ /**
+ * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure that the system user exists (since the master does an RPC to
+ * the tserver which will create the system user if it doesn't already exist).
+ */
+ private void createReadWriteDrop(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException {
+ final String table = testName.getMethodName() + "_table";
+ conn.tableOperations().create(table);
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", "d");
+ bw.addMutation(m);
+ bw.close();
+ conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true));
+ Scanner s = conn.createScanner(table, Authorizations.EMPTY);
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ assertEquals("Did not find the expected key", 0, new Key("a", "b", "c").compareTo(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL));
+ assertEquals("d", entry.getValue().toString());
+ conn.tableOperations().delete(table);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
index dd83574,0c2631f..49160aa
--- a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
@@@ -37,9 -35,7 +37,9 @@@ public class MasterFailoverIT extends A
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s"));
+ Map<String,String> siteConfig = cfg.getSiteConfig();
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
++ siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ cfg.setSiteConfig(siteConfig);
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index 2ba6d31,b498412..d1fb9f9
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@@ -70,10 -66,12 +70,10 @@@ public class RestartIT extends Accumulo
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
- Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
- props.put(Property.GC_CYCLE_DELAY.getKey(), "1s");
- props.put(Property.GC_CYCLE_START.getKey(), "1s");
- cfg.setSiteConfig(props);
- cfg.useMiniDFS(true);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
private static final ScannerOpts SOPTS = new ScannerOpts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index 3f7d67d,c4b3afd..68448eb
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@@ -58,10 -55,10 +58,10 @@@ public class RestartStressIT extends Ac
opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
- opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
cfg.setSiteConfig(opts);
- cfg.useMiniDFS(true);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
index aec6bae,0000000..ca45382
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
@@@ -1,153 -1,0 +1,153 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriter.Status;
+import org.apache.accumulo.core.client.ConditionalWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Durability;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Condition;
+import org.apache.accumulo.core.data.ConditionalMutation;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class SessionDurabilityIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void nondurableTableHasDurableWrites() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default has no durability
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ // send durable writes
+ BatchWriterConfig cfg = new BatchWriterConfig();
+ cfg.setDurability(Durability.SYNC);
+ writeSome(tableName, 10, cfg);
+ assertEquals(10, count(tableName));
+ // verify writes servive restart
+ restartTServer();
+ assertEquals(10, count(tableName));
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void durableTableLosesNonDurableWrites() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ // write with no durability
+ BatchWriterConfig cfg = new BatchWriterConfig();
+ cfg.setDurability(Durability.NONE);
+ writeSome(tableName, 10, cfg);
+ // verify writes are lost on restart
+ restartTServer();
+ assertTrue(10 > count(tableName));
+ }
+
+ private int count(String tableName) throws Exception {
+ return Iterators.size(getConnector().createScanner(tableName, Authorizations.EMPTY).iterator());
+ }
+
+ private void writeSome(String tableName, int n, BatchWriterConfig cfg) throws Exception {
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(tableName, cfg);
+ for (int i = 0; i < n; i++) {
+ Mutation m = new Mutation(i + "");
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void testConditionDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ // write without durability
+ ConditionalWriterConfig cfg = new ConditionalWriterConfig();
+ cfg.setDurability(Durability.NONE);
+ conditionWriteSome(tableName, 10, cfg);
+ // everything in there?
+ assertEquals(10, count(tableName));
+ // restart the server and verify the updates are lost
+ restartTServer();
+ assertEquals(0, count(tableName));
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void testConditionDurability2() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ // write with durability
+ ConditionalWriterConfig cfg = new ConditionalWriterConfig();
+ cfg.setDurability(Durability.SYNC);
+ conditionWriteSome(tableName, 10, cfg);
+ // everything in there?
+ assertEquals(10, count(tableName));
+ // restart the server and verify the updates are still there
+ restartTServer();
+ assertEquals(10, count(tableName));
+ }
+
+ private void conditionWriteSome(String tableName, int n, ConditionalWriterConfig cfg) throws Exception {
+ Connector c = getConnector();
+ ConditionalWriter cw = c.createConditionalWriter(tableName, cfg);
+ for (int i = 0; i < n; i++) {
+ ConditionalMutation m = new ConditionalMutation((CharSequence) (i + ""), new Condition("", ""));
+ m.put("", "", "X");
+ assertEquals(Status.ACCEPTED, cw.write(m).getStatus());
+ }
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
index 44473b0,bfca75b..bc36257
--- a/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
@@@ -35,13 -35,15 +35,13 @@@ public class WriteAheadLogIT extends Ac
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "2M");
- siteConfig.put(Property.GC_CYCLE_DELAY.getKey(), "1");
- siteConfig.put(Property.GC_CYCLE_START.getKey(), "1");
- siteConfig.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "4s");
- cfg.setSiteConfig(siteConfig);
- cfg.useMiniDFS(true);
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "1s");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "4s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
index f852901,f852901..fefb9a6
--- a/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
@@@ -45,7 -45,7 +45,7 @@@ public class ZookeeperRestartIT extend
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> siteConfig = new HashMap<String,String>();
-- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
++ siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
cfg.setSiteConfig(siteConfig);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
index bd10f90,0000000..35bc0fe
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
@@@ -1,733 -1,0 +1,733 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.master.replication.SequentialWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+/**
+ * Replication tests which start at least two MAC instances and replicate data between them
+ */
+public class MultiInstanceReplicationIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(MultiInstanceReplicationIT.class);
+
+ private ExecutorService executor;
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 10 * 60;
+ }
+
+ @Before
+ public void createExecutor() {
+ executor = Executors.newSingleThreadExecutor();
+ }
+
+ @After
+ public void stopExecutor() {
+ if (null != executor) {
+ executor.shutdownNow();
+ }
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, SequentialWorkAssigner.class.getName());
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+ */
+ private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+ // Set the same SSL information from the primary when present
+ Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+ if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+ Map<String,String> peerSiteConfig = new HashMap<String,String>();
+ peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+ String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+ Assert.assertNotNull("Keystore Path was null", keystorePath);
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+ String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+ Assert.assertNotNull("Truststore Path was null", truststorePath);
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+ // Passwords might be stored in CredentialProvider
+ String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+ if (null != keystorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+ }
+ String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+ if (null != truststorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+ }
+
+ System.out.println("Setting site configuration for peer " + peerSiteConfig);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+
+ // Use the CredentialProvider if the primary also uses one
+ String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+ if (null != credProvider) {
+ Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+ peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeer() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ try {
+ final Connector connMaster = getConnector();
+ final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ ReplicationTable.setOnline(connMaster);
+
+ String peerUserName = "peer", peerPassword = "foo";
+
+ String peerClusterName = "peer";
+
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ final String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ log.info("Files to replicate: " + filesNeedingReplication);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("TabletServer restarted");
+ Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+ log.info("TabletServer is online");
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+ @Override
+ public Boolean call() throws Exception {
+ long then = System.currentTimeMillis();
+ connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+ long now = System.currentTimeMillis();
+ log.info("Drain completed in " + (now - then) + "ms");
+ return true;
+ }
+
+ });
+
+ try {
+ future.get(60, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ Assert.fail("Drain did not finish within 60 seconds");
+ } finally {
+ executor.shutdownNow();
+ }
+
+ log.info("drain completed");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Entry<Key,Value> masterEntry = null, peerEntry = null;
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ masterEntry = masterIter.next();
+ peerEntry = peerIter.next();
+ Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ log.info("Last master entry: " + masterEntry);
+ log.info("Last peer entry: " + peerEntry);
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+ } finally {
+ peerCluster.stop();
+ }
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTable() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+ String peerUserName = "peer", peerPassword = "foo";
+
+ // Create local user
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ // Create tables
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Grant write permission
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ long masterTable1Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable1Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ long masterTable2Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable2Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+ masterTable2);
+
+ log.info("Files to replicate for table1: " + filesFor1);
+ log.info("Files to replicate for table2: " + filesFor2);
+
+ // Restart the tserver to force a close on the WAL
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("Restarted the tserver");
+
+ // Read the data -- the tserver is back up and running
+ Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ // Wait for both tables to be replicated
+ log.info("Waiting for {} for {}", filesFor1, masterTable1);
+ connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+ log.info("Waiting for {} for {}", filesFor2, masterTable2);
+ connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+ long countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+ Assert.assertEquals(masterTable1Records, countTable);
+
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+ Assert.assertEquals(masterTable2Records, countTable);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Give our replication user the ability to write to the table
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ log.info("Files to replicate:" + files);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ connMaster.replicationOperations().drain(masterTable, files);
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+ Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+ peerCluster.stop();
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Give our replication user the ability to write to the tables
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ UtilWaitThread.sleep(2000);
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
+ }
+ }
+
+ Assert.assertNotEquals(0, fullyReplicated);
+
+ // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
+ // Be cautious in how quickly we assert that the data is present on the peer
+ long countTable = 0l;
+ for (int i = 0; i < 10; i++) {
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+
+ if (0l == countTable) {
+ Thread.sleep(5000);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertTrue("Found no records in " + peerTable1 + " in the peer cluster", countTable > 0);
+
+ // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
+ // Be cautious in how quickly we assert that the data is present on the peer
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+
+ if (0l == countTable) {
+ Thread.sleep(5000);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertTrue("Found no records in " + peerTable2 + " in the peer cluster", countTable > 0);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+}
[04/13] accumulo git commit: Merge branch '1.6' into 1.7
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
index 761d92c,0000000..090ac83
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/UnorderedWorkAssignerReplicationIT.java
@@@ -1,731 -1,0 +1,731 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.master.replication.UnorderedWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class UnorderedWorkAssignerReplicationIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(UnorderedWorkAssignerReplicationIT.class);
+
+ private ExecutorService executor;
+ private int timeoutFactor = 1;
+
+ @Before
+ public void createExecutor() {
+ executor = Executors.newSingleThreadExecutor();
+
+ try {
+ timeoutFactor = Integer.parseInt(System.getProperty("timeout.factor"));
+ } catch (NumberFormatException exception) {
+ log.warn("Could not parse timeout.factor, not increasing timeout.");
+ }
+
+ Assert.assertTrue("The timeout factor must be a positive, non-zero value", timeoutFactor > 0);
+ }
+
+ @After
+ public void stopExecutor() {
+ if (null != executor) {
+ executor.shutdownNow();
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60 * 5;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, UnorderedWorkAssigner.class.getName());
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+ */
+ private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+ // Set the same SSL information from the primary when present
+ Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+ if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+ Map<String,String> peerSiteConfig = new HashMap<String,String>();
+ peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+ String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+ Assert.assertNotNull("Keystore Path was null", keystorePath);
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+ String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+ Assert.assertNotNull("Truststore Path was null", truststorePath);
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+ // Passwords might be stored in CredentialProvider
+ String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+ if (null != keystorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+ }
+ String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+ if (null != truststorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+ }
+
+ System.out.println("Setting site configuration for peer " + peerSiteConfig);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+
+ // Use the CredentialProvider if the primary also uses one
+ String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+ if (null != credProvider) {
+ Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+ peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeer() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ try {
+ final Connector connMaster = getConnector();
+ final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ ReplicationTable.setOnline(connMaster);
+
+ String peerUserName = "peer", peerPassword = "foo";
+
+ String peerClusterName = "peer";
+
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ final String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Wait for zookeeper updates (configuration) to propagate
+ UtilWaitThread.sleep(3 * 1000);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("TabletServer restarted");
+ Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+ log.info("TabletServer is online");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+ @Override
+ public Boolean call() throws Exception {
+ connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+ log.info("Drain completed");
+ return true;
+ }
+
+ });
+
+ long timeoutSeconds = timeoutFactor * 30;
+ try {
+ future.get(timeoutSeconds, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ Assert.fail("Drain did not finish within " + timeoutSeconds + " seconds");
+ }
+
+ log.info("drain completed");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Entry<Key,Value> masterEntry = null, peerEntry = null;
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ masterEntry = masterIter.next();
+ peerEntry = peerIter.next();
+ Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ log.info("Last master entry: " + masterEntry);
+ log.info("Last peer entry: " + peerEntry);
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+ } finally {
+ peerCluster.stop();
+ }
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTable() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+ String peerUserName = "peer", peerPassword = "foo";
+
+ // Create local user
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ // Create tables
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Grant write permission
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Wait for zookeeper updates (configuration) to propogate
+ UtilWaitThread.sleep(3 * 1000);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ long masterTable1Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable1Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ long masterTable2Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable2Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+ masterTable2);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ Thread.sleep(500);
+ }
+
+ // Restart the tserver to force a close on the WAL
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("Restarted the tserver");
+
+ // Read the data -- the tserver is back up and running
+ Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+ // Wait for both tables to be replicated
+ log.info("Waiting for {} for {}", filesFor1, masterTable1);
+ connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+ log.info("Waiting for {} for {}", filesFor2, masterTable2);
+ connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+ long countTable = 0l;
+ for (int i = 0; i < 5; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+
+ if (masterTable1Records != countTable) {
+ log.warn("Did not find {} expected records in {}, only found {}", masterTable1Records, peerTable1, countTable);
+ }
+ }
+
+ Assert.assertEquals(masterTable1Records, countTable);
+
+ for (int i = 0; i < 5; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+
+ if (masterTable2Records != countTable) {
+ log.warn("Did not find {} expected records in {}, only found {}", masterTable2Records, peerTable2, countTable);
+ }
+ }
+
+ Assert.assertEquals(masterTable2Records, countTable);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Give our replication user the ability to write to the table
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+ for (String s : files) {
+ log.info("Found referenced file for " + masterTable + ": " + s);
+ }
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+ for (Entry<Key,Value> kv : connMaster.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
+ log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ connMaster.replicationOperations().drain(masterTable, files);
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Assert.assertTrue("No data in master table", masterIter.hasNext());
+ Assert.assertTrue("No data in peer table", peerIter.hasNext());
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+ Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+ peerCluster.stop();
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Give our replication user the ability to write to the tables
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Wait for zookeeper updates (configuration) to propagate
+ UtilWaitThread.sleep(3 * 1000);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ Thread.sleep(500);
+ }
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ UtilWaitThread.sleep(timeoutFactor * 2000);
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
+ }
+ }
+
+ Assert.assertNotEquals(0, fullyReplicated);
+
+ long countTable = 0l;
+
+ // Check a few times
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+ log.info("Found {} records in {}", countTable, peerTable1);
+ if (0 < countTable) {
+ break;
+ }
+ Thread.sleep(2000);
+ }
+
+ Assert.assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
+
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+ if (0 < countTable) {
+ break;
+ }
+ Thread.sleep(2000);
+ }
+ Assert.assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+}
[10/13] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/replication/GarbageCollectorCommunicatesWithTServersIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/replication/GarbageCollectorCommunicatesWithTServersIT.java
index 6a14de3,0000000..2852387
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/GarbageCollectorCommunicatesWithTServersIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/GarbageCollectorCommunicatesWithTServersIT.java
@@@ -1,415 -1,0 +1,415 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.Credentials;
+import org.apache.accumulo.core.client.impl.MasterClient;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.master.thrift.MasterClientService;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.rpc.ThriftUtil;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
+import org.apache.accumulo.core.trace.Tracer;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.log.WalStateManager;
+import org.apache.accumulo.server.log.WalStateManager.WalState;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ * ACCUMULO-3302 series of tests which ensure that a WAL is prematurely closed when a TServer may still continue to use it. Checking that no tablet references a
+ * WAL is insufficient to determine if a WAL will never be used in the future.
+ */
+public class GarbageCollectorCommunicatesWithTServersIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(GarbageCollectorCommunicatesWithTServersIT.class);
+
+ private final int GC_PERIOD_SECONDS = 1;
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, GC_PERIOD_SECONDS + "s");
+ // Wait longer to try to let the replication table come online before a cycle runs
+ cfg.setProperty(Property.GC_CYCLE_START, "10s");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ // Set really long delays for the master to do stuff for replication. We don't need
+ // it to be doing anything, so just let it sleep
+ cfg.setProperty(Property.REPLICATION_WORK_PROCESSOR_DELAY, "240s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "240s");
+ cfg.setProperty(Property.REPLICATION_DRIVER_DELAY, "240s");
+ // Pull down the maximum size of the wal so we can test close()'ing it.
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1M");
+ coreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Fetch all of the WALs referenced by tablets in the metadata table for this table
+ */
+ private Set<String> getWalsForTable(String tableName) throws Exception {
+ final Connector conn = getConnector();
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+
+ Assert.assertNotNull("Could not determine table ID for " + tableName, tableId);
+
+ Instance i = conn.getInstance();
+ ZooReaderWriter zk = new ZooReaderWriter(i.getZooKeepers(), i.getZooKeepersSessionTimeOut(), "");
+ WalStateManager wals = new WalStateManager(conn.getInstance(), zk);
+
+ Set<String> result = new HashSet<String>();
+ for (Entry<Path,WalState> entry : wals.getAllState().entrySet()) {
+ log.debug("Reading WALs: {}={}", entry.getKey(), entry.getValue());
+ result.add(entry.getKey().toString());
+ }
+ return result;
+ }
+
+ /**
+ * Fetch all of the rfiles referenced by tablets in the metadata table for this table
+ */
+ private Set<String> getFilesForTable(String tableName) throws Exception {
+ final Connector conn = getConnector();
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+
+ Assert.assertNotNull("Could not determine table ID for " + tableName, tableId);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ Range r = MetadataSchema.TabletsSection.getRange(tableId);
+ s.setRange(r);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+
+ Set<String> rfiles = new HashSet<String>();
+ for (Entry<Key,Value> entry : s) {
+ log.debug("Reading RFiles: {}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
+ // uri://path/to/wal
+ String cq = entry.getKey().getColumnQualifier().toString();
+ String path = new Path(cq).toString();
+ log.debug("Normalize path to rfile: {}", path);
+ rfiles.add(path);
+ }
+
+ return rfiles;
+ }
+
+ /**
+ * Get the replication status messages for the given table that exist in the metadata table (~repl entries)
+ */
+ private Map<String,Status> getMetadataStatusForTable(String tableName) throws Exception {
+ final Connector conn = getConnector();
+ final String tableId = conn.tableOperations().tableIdMap().get(tableName);
+
+ Assert.assertNotNull("Could not determine table ID for " + tableName, tableId);
+
+ Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ Range r = MetadataSchema.ReplicationSection.getRange();
+ s.setRange(r);
+ s.fetchColumn(MetadataSchema.ReplicationSection.COLF, new Text(tableId));
+
+ Map<String,Status> fileToStatus = new HashMap<String,Status>();
+ for (Entry<Key,Value> entry : s) {
+ Text file = new Text();
+ MetadataSchema.ReplicationSection.getFile(entry.getKey(), file);
+ Status status = Status.parseFrom(entry.getValue().get());
+ log.info("Got status for {}: {}", file, ProtobufUtil.toString(status));
+ fileToStatus.put(file.toString(), status);
+ }
+
+ return fileToStatus;
+ }
+
+ @Test
+ public void testActiveWalPrecludesClosing() throws Exception {
+ final String table = getUniqueNames(1)[0];
+ final Connector conn = getConnector();
+
+ // Bring the replication table online first and foremost
+ ReplicationTable.setOnline(conn);
+
+ log.info("Creating {}", table);
+ conn.tableOperations().create(table);
+
+ conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
+
+ log.info("Writing a few mutations to the table");
+
+ BatchWriter bw = conn.createBatchWriter(table, null);
+
+ byte[] empty = new byte[0];
+ for (int i = 0; i < 5; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put(empty, empty, empty);
+ bw.addMutation(m);
+ }
+
+ log.info("Flushing mutations to the server");
+ bw.flush();
+
+ log.info("Checking that metadata only has two WALs recorded for this table (inUse, and opened)");
+
+ Set<String> wals = getWalsForTable(table);
+ Assert.assertEquals("Expected to only find two WALs for the table", 2, wals.size());
+
+ // Flush our test table to remove the WAL references in it
+ conn.tableOperations().flush(table, null, null, true);
+ // Flush the metadata table too because it will have a reference to the WAL
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+
+ log.info("Waiting for replication table to come online");
+
+ log.info("Fetching replication statuses from metadata table");
+
+ Map<String,Status> fileToStatus = getMetadataStatusForTable(table);
+
+ Assert.assertEquals("Expected to only find one replication status message", 1, fileToStatus.size());
+
+ String walName = fileToStatus.keySet().iterator().next();
+ wals.retainAll(fileToStatus.keySet());
+ Assert.assertEquals(1, wals.size());
+
+ Status status = fileToStatus.get(walName);
+
+ Assert.assertEquals("Expected Status for file to not be closed", false, status.getClosed());
+
+ Set<String> filesForTable = getFilesForTable(table);
+ Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTable.size());
+ log.info("Files for table before MajC: {}", filesForTable);
+
+ // Issue a MajC to roll a new file in HDFS
+ conn.tableOperations().compact(table, null, null, false, true);
+
+ Set<String> filesForTableAfterCompaction = getFilesForTable(table);
+
+ log.info("Files for table after MajC: {}", filesForTableAfterCompaction);
+
+ Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTableAfterCompaction.size());
+ Assert.assertNotEquals("Expected the files before and after compaction to differ", filesForTableAfterCompaction, filesForTable);
+
+ // Use the rfile which was just replaced by the MajC to determine when the GC has ran
+ Path fileToBeDeleted = new Path(filesForTable.iterator().next());
+ FileSystem fs = getCluster().getFileSystem();
+
+ boolean fileExists = fs.exists(fileToBeDeleted);
+ while (fileExists) {
+ log.info("File which should get deleted still exists: {}", fileToBeDeleted);
+ Thread.sleep(2000);
+ fileExists = fs.exists(fileToBeDeleted);
+ }
+
+ Map<String,Status> fileToStatusAfterMinc = getMetadataStatusForTable(table);
+ Assert.assertEquals("Expected to still find only one replication status message: " + fileToStatusAfterMinc, 1, fileToStatusAfterMinc.size());
+
+ Assert.assertEquals("Status before and after MinC should be identical", fileToStatus, fileToStatusAfterMinc);
+ }
+
+ @Test(timeout = 2 * 60 * 1000)
+ public void testUnreferencedWalInTserverIsClosed() throws Exception {
+ final String[] names = getUniqueNames(2);
+ // `table` will be replicated, `otherTable` is only used to roll the WAL on the tserver
+ final String table = names[0], otherTable = names[1];
+ final Connector conn = getConnector();
+
+ // Bring the replication table online first and foremost
+ ReplicationTable.setOnline(conn);
+
+ log.info("Creating {}", table);
+ conn.tableOperations().create(table);
+
+ conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
+
+ log.info("Writing a few mutations to the table");
+
+ BatchWriter bw = conn.createBatchWriter(table, null);
+
+ byte[] empty = new byte[0];
+ for (int i = 0; i < 5; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put(empty, empty, empty);
+ bw.addMutation(m);
+ }
+
+ log.info("Flushing mutations to the server");
+ bw.close();
+
+ log.info("Checking that metadata only has one WAL recorded for this table");
+
+ Set<String> wals = getWalsForTable(table);
+ Assert.assertEquals("Expected to only find two WAL for the table", 2, wals.size());
+
+ log.info("Compacting the table which will remove all WALs from the tablets");
+
+ // Flush our test table to remove the WAL references in it
+ conn.tableOperations().flush(table, null, null, true);
+ // Flush the metadata table too because it will have a reference to the WAL
+ conn.tableOperations().flush(MetadataTable.NAME, null, null, true);
+
+ log.info("Fetching replication statuses from metadata table");
+
+ Map<String,Status> fileToStatus = getMetadataStatusForTable(table);
+
+ Assert.assertEquals("Expected to only find one replication status message", 1, fileToStatus.size());
+
+ String walName = fileToStatus.keySet().iterator().next();
+ Assert.assertTrue("Expected log file name from tablet to equal replication entry", wals.contains(walName));
+
+ Status status = fileToStatus.get(walName);
+
+ Assert.assertEquals("Expected Status for file to not be closed", false, status.getClosed());
+
+ Set<String> filesForTable = getFilesForTable(table);
+ Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTable.size());
+ log.info("Files for table before MajC: {}", filesForTable);
+
+ // Issue a MajC to roll a new file in HDFS
+ conn.tableOperations().compact(table, null, null, false, true);
+
+ Set<String> filesForTableAfterCompaction = getFilesForTable(table);
+
+ log.info("Files for table after MajC: {}", filesForTableAfterCompaction);
+
+ Assert.assertEquals("Expected to only find one rfile for table", 1, filesForTableAfterCompaction.size());
+ Assert.assertNotEquals("Expected the files before and after compaction to differ", filesForTableAfterCompaction, filesForTable);
+
+ // Use the rfile which was just replaced by the MajC to determine when the GC has ran
+ Path fileToBeDeleted = new Path(filesForTable.iterator().next());
+ FileSystem fs = getCluster().getFileSystem();
+
+ boolean fileExists = fs.exists(fileToBeDeleted);
+ while (fileExists) {
+ log.info("File which should get deleted still exists: {}", fileToBeDeleted);
+ Thread.sleep(2000);
+ fileExists = fs.exists(fileToBeDeleted);
+ }
+
+ // At this point in time, we *know* that the GarbageCollector has run which means that the Status
+ // for our WAL should not be altered.
+
+ Map<String,Status> fileToStatusAfterMinc = getMetadataStatusForTable(table);
+ Assert.assertEquals("Expected to still find only one replication status message: " + fileToStatusAfterMinc, 1, fileToStatusAfterMinc.size());
+
+ /*
+ * To verify that the WALs is still getting closed, we have to force the tserver to close the existing WAL and open a new one instead. The easiest way to do
+ * this is to write a load of data that will exceed the 1.33% full threshold that the logger keeps track of
+ */
+
+ conn.tableOperations().create(otherTable);
+ bw = conn.createBatchWriter(otherTable, null);
+ // 500k
+ byte[] bigValue = new byte[1024 * 500];
+ Arrays.fill(bigValue, (byte) 1);
+ // 500k * 50
+ for (int i = 0; i < 50; i++) {
+ Mutation m = new Mutation(Integer.toString(i));
+ m.put(empty, empty, bigValue);
+ bw.addMutation(m);
+ if (i % 10 == 0) {
+ bw.flush();
+ }
+ }
+
+ bw.close();
+
+ conn.tableOperations().flush(otherTable, null, null, true);
+
+ // Get the tservers which the master deems as active
+ final ClientContext context = new ClientContext(conn.getInstance(), new Credentials("root", new PasswordToken(ConfigurableMacBase.ROOT_PASSWORD)),
+ getClientConfig());
+ List<String> tservers = MasterClient.execute(context, new ClientExecReturn<List<String>,MasterClientService.Client>() {
+ @Override
+ public List<String> execute(MasterClientService.Client client) throws Exception {
+ return client.getActiveTservers(Tracer.traceInfo(), context.rpcCreds());
+ }
+ });
+
+ Assert.assertEquals("Expected only one active tservers", 1, tservers.size());
+
+ HostAndPort tserver = HostAndPort.fromString(tservers.get(0));
+
+ // Get the active WALs from that server
+ log.info("Fetching active WALs from {}", tserver);
+
+ Client client = ThriftUtil.getTServerClient(tserver, context);
+ List<String> activeWalsForTserver = client.getActiveLogs(Tracer.traceInfo(), context.rpcCreds());
+
+ log.info("Active wals: {}", activeWalsForTserver);
+
+ Assert.assertEquals("Expected to find only one active WAL", 1, activeWalsForTserver.size());
+
+ String activeWal = new Path(activeWalsForTserver.get(0)).toString();
+
+ Assert.assertNotEquals("Current active WAL on tserver should not be the original WAL we saw", walName, activeWal);
+
+ log.info("Ensuring that replication status does get closed after WAL is no longer in use by Tserver");
+
+ do {
+ Map<String,Status> replicationStatuses = getMetadataStatusForTable(table);
+
+ log.info("Got replication status messages {}", replicationStatuses);
+ Assert.assertEquals("Did not expect to find additional status records", 1, replicationStatuses.size());
+
+ status = replicationStatuses.values().iterator().next();
+ log.info("Current status: {}", ProtobufUtil.toString(status));
+
+ if (status.getClosed()) {
+ return;
+ }
+
+ log.info("Status is not yet closed, waiting for garbage collector to close it");
+
+ Thread.sleep(2000);
+ } while (true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
index cb3df68,0000000..0c68af6
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
@@@ -1,733 -1,0 +1,733 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.master.replication.SequentialWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+/**
+ * Replication tests which start at least two MAC instances and replicate data between them
+ */
+public class MultiInstanceReplicationIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(MultiInstanceReplicationIT.class);
+
+ private ExecutorService executor;
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 10 * 60;
+ }
+
+ @Before
+ public void createExecutor() {
+ executor = Executors.newSingleThreadExecutor();
+ }
+
+ @After
+ public void stopExecutor() {
+ if (null != executor) {
+ executor.shutdownNow();
+ }
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, SequentialWorkAssigner.class.getName());
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+ */
+ private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+ // Set the same SSL information from the primary when present
+ Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+ if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+ Map<String,String> peerSiteConfig = new HashMap<String,String>();
+ peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+ String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+ Assert.assertNotNull("Keystore Path was null", keystorePath);
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+ String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+ Assert.assertNotNull("Truststore Path was null", truststorePath);
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+ // Passwords might be stored in CredentialProvider
+ String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+ if (null != keystorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+ }
+ String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+ if (null != truststorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+ }
+
+ System.out.println("Setting site configuration for peer " + peerSiteConfig);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+
+ // Use the CredentialProvider if the primary also uses one
+ String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+ if (null != credProvider) {
+ Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+ peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+ }
+
+ @Test(timeout = 10 * 60 * 1000)
+ public void dataWasReplicatedToThePeer() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ try {
+ final Connector connMaster = getConnector();
+ final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ ReplicationTable.setOnline(connMaster);
+
+ String peerUserName = "peer", peerPassword = "foo";
+
+ String peerClusterName = "peer";
+
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ final String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ log.info("Files to replicate: " + filesNeedingReplication);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("TabletServer restarted");
+ Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+ log.info("TabletServer is online");
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+ @Override
+ public Boolean call() throws Exception {
+ long then = System.currentTimeMillis();
+ connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+ long now = System.currentTimeMillis();
+ log.info("Drain completed in " + (now - then) + "ms");
+ return true;
+ }
+
+ });
+
+ try {
+ future.get(60, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ Assert.fail("Drain did not finish within 60 seconds");
+ } finally {
+ executor.shutdownNow();
+ }
+
+ log.info("drain completed");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Entry<Key,Value> masterEntry = null, peerEntry = null;
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ masterEntry = masterIter.next();
+ peerEntry = peerIter.next();
+ Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ log.info("Last master entry: " + masterEntry);
+ log.info("Last peer entry: " + peerEntry);
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+ } finally {
+ peerCluster.stop();
+ }
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTable() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+ String peerUserName = "peer", peerPassword = "foo";
+
+ // Create local user
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ // Create tables
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Grant write permission
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ long masterTable1Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable1Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ long masterTable2Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable2Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+ masterTable2);
+
+ log.info("Files to replicate for table1: " + filesFor1);
+ log.info("Files to replicate for table2: " + filesFor2);
+
+ // Restart the tserver to force a close on the WAL
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("Restarted the tserver");
+
+ // Read the data -- the tserver is back up and running
+ Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ // Wait for both tables to be replicated
+ log.info("Waiting for {} for {}", filesFor1, masterTable1);
+ connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+ log.info("Waiting for {} for {}", filesFor2, masterTable2);
+ connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+ long countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+ Assert.assertEquals(masterTable1Records, countTable);
+
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+ Assert.assertEquals(masterTable2Records, countTable);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Give our replication user the ability to write to the table
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ log.info("Files to replicate:" + files);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ connMaster.replicationOperations().drain(masterTable, files);
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+ Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+ peerCluster.stop();
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Give our replication user the ability to write to the tables
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ sleepUninterruptibly(2, TimeUnit.SECONDS);
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
+ }
+ }
+
+ Assert.assertNotEquals(0, fullyReplicated);
+
+ // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
+ // Be cautious in how quickly we assert that the data is present on the peer
+ long countTable = 0l;
+ for (int i = 0; i < 10; i++) {
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+
+ if (0l == countTable) {
+ Thread.sleep(5000);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertTrue("Found no records in " + peerTable1 + " in the peer cluster", countTable > 0);
+
+ // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
+ // Be cautious in how quickly we assert that the data is present on the peer
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+
+ if (0l == countTable) {
+ Thread.sleep(5000);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertTrue("Found no records in " + peerTable2 + " in the peer cluster", countTable > 0);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+}
[07/13] accumulo git commit: Merge branch '1.6' into 1.7
Posted by el...@apache.org.
Merge branch '1.6' into 1.7
Conflicts:
test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/94f4a19c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/94f4a19c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/94f4a19c
Branch: refs/heads/1.7
Commit: 94f4a19c0d776f3a57e855863c34e9e3164b615b
Parents: 6562828 29c6052
Author: Josh Elser <el...@apache.org>
Authored: Tue Jan 12 12:25:34 2016 -0500
Committer: Josh Elser <el...@apache.org>
Committed: Tue Jan 12 12:25:34 2016 -0500
----------------------------------------------------------------------
test/src/test/java/org/apache/accumulo/test/CleanWalIT.java | 2 +-
.../java/org/apache/accumulo/test/DetectDeadTabletServersIT.java | 2 +-
test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java | 2 +-
.../org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java | 2 +-
.../test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 +-
.../org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java | 2 +-
.../test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java | 2 +-
.../test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/BinaryStressIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/CleanTmpIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/CompactionIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/DurabilityIT.java | 2 +-
.../org/apache/accumulo/test/functional/GarbageCollectorIT.java | 2 +-
.../org/apache/accumulo/test/functional/KerberosRenewalIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/MasterFailoverIT.java | 2 +-
.../test/java/org/apache/accumulo/test/functional/RestartIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/RestartStressIT.java | 2 +-
.../org/apache/accumulo/test/functional/SessionDurabilityIT.java | 2 +-
.../java/org/apache/accumulo/test/functional/WriteAheadLogIT.java | 2 +-
.../org/apache/accumulo/test/functional/ZookeeperRestartIT.java | 2 +-
.../accumulo/test/replication/MultiInstanceReplicationIT.java | 2 +-
.../test/replication/UnorderedWorkAssignerReplicationIT.java | 2 +-
22 files changed, 22 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
index d754a14,b2298f7..08e3c09
--- a/test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/CleanWalIT.java
@@@ -57,10 -54,9 +57,10 @@@ public class CleanWalIT extends Accumul
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setNumTservers(1);
- cfg.useMiniDFS(true);
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Before
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
index 04c781b,1e65601..f7ee089
--- a/test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/DetectDeadTabletServersIT.java
@@@ -42,7 -41,7 +42,7 @@@ public class DetectDeadTabletServersIT
@Override
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
-- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
}
@Test
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/ExistingMacIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/MasterRepairsDualAssignmentIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/MultiTableRecoveryIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
index 1a3c92f,0000000..57c2c34
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java
@@@ -1,101 -1,0 +1,101 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloClusterIT;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+// Accumulo3010
+public class RecoveryCompactionsAreFlushesIT extends AccumuloClusterIT {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60;
+ }
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ // file system supports recovery
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test
+ public void test() throws Exception {
+ // create a table
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "100");
+ c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "3");
+ // create 3 flush files
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", new Value("v".getBytes()));
+ for (int i = 0; i < 3; i++) {
+ bw.addMutation(m);
+ bw.flush();
+ c.tableOperations().flush(tableName, null, null, true);
+ }
+ // create an unsaved mutation
+ bw.addMutation(m);
+ bw.close();
+
+ ClusterControl control = cluster.getClusterControl();
+
+ // kill the tablet servers
+ control.stopAllServers(ServerType.TABLET_SERVER);
+
+ // recover
+ control.startAllServers(ServerType.TABLET_SERVER);
+
+ // ensure the table is readable
+ Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator());
+
+ // ensure that the recovery was not a merging minor compaction
+ Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ for (Entry<Key,Value> entry : s) {
+ String filename = entry.getKey().getColumnQualifier().toString();
+ String parts[] = filename.split("/");
+ Assert.assertFalse(parts[parts.length - 1].startsWith("M"));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
index 081ee85,0000000..68bd07b
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/TabletServerGivesUpIT.java
@@@ -1,75 -1,0 +1,75 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+// ACCUMULO-2480
+public class TabletServerGivesUpIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.useMiniDFS(true);
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_TOLERATED_CREATION_FAILURES, "15");
+ cfg.setProperty(Property.TSERV_WALOG_TOLERATED_MAXIMUM_WAIT_DURATION, "0s");
+ }
+
+ @Test(timeout = 30 * 1000)
+ public void test() throws Exception {
+ final Connector conn = this.getConnector();
+ // Yes, there's a tabletserver
+ assertEquals(1, conn.instanceOperations().getTabletServers().size());
+ final String tableName = getUniqueNames(1)[0];
+ conn.tableOperations().create(tableName);
+ // Kill dfs
+ cluster.getMiniDfs().shutdown();
+ // ask the tserver to do something
+ final AtomicReference<Exception> ex = new AtomicReference<>();
+ Thread splitter = new Thread() {
+ @Override
+ public void run() {
+ try {
+ TreeSet<Text> splits = new TreeSet<>();
+ splits.add(new Text("X"));
+ conn.tableOperations().addSplits(tableName, splits);
+ } catch (Exception e) {
+ ex.set(e);
+ }
+ }
+ };
+ splitter.start();
+ // wait for the tserver to give up on writing to the WAL
+ while (conn.instanceOperations().getTabletServers().size() == 1) {
+ UtilWaitThread.sleep(1000);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
index 8338021,0000000..c318075
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java
@@@ -1,107 -1,0 +1,107 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.test.functional.FunctionalTestUtils;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class VerifySerialRecoveryIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT, "20");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testSerializedRecovery() throws Exception {
+ // make a table with many splits
+ String tableName = getUniqueNames(1)[0];
+ Connector c = getConnector();
+ c.tableOperations().create(tableName);
+ SortedSet<Text> splits = new TreeSet<Text>();
+ for (int i = 0; i < 200; i++) {
+ splits.add(new Text(AssignmentThreadsIT.randomHex(8)));
+ }
+ c.tableOperations().addSplits(tableName, splits);
+ // load data to give the recovery something to do
+ BatchWriter bw = c.createBatchWriter(tableName, null);
+ for (int i = 0; i < 50000; i++) {
+ Mutation m = new Mutation(AssignmentThreadsIT.randomHex(8));
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ // kill the tserver
+ for (ProcessReference ref : getCluster().getProcesses().get(ServerType.TABLET_SERVER))
+ getCluster().killProcess(ServerType.TABLET_SERVER, ref);
+ final Process ts = cluster.exec(TabletServer.class);
+
+ // wait for recovery
+ Iterators.size(c.createScanner(tableName, Authorizations.EMPTY).iterator());
+ assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
+ ts.waitFor();
+ String result = FunctionalTestUtils.readAll(cluster, TabletServer.class, ts);
+ for (String line : result.split("\n")) {
+ System.out.println(line);
+ }
+ // walk through the output, verifying that only a single normal recovery was running at one time
+ boolean started = false;
+ int recoveries = 0;
+ for (String line : result.split("\n")) {
+ // ignore metadata tables
+ if (line.contains("!0") || line.contains("+r"))
+ continue;
+ if (line.contains("Starting Write-Ahead Log")) {
+ assertFalse(started);
+ started = true;
+ recoveries++;
+ }
+ if (line.contains("Write-Ahead Log recovery complete")) {
+ assertTrue(started);
+ started = false;
+ }
+ }
+ assertFalse(started);
+ assertTrue(recoveries > 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
index 62d8738,fbe504e..f1bca1b
--- a/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/BinaryStressIT.java
@@@ -50,9 -51,10 +50,9 @@@ public class BinaryStressIT extends Acc
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
- Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.TSERV_MAXMEM.getKey(), "50K");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "0");
- cfg.setSiteConfig(siteConfig);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_MAXMEM, "50K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "0");
}
private String majcDelay, maxMem;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
index 921d661,65422c9..d03007e
--- a/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CleanTmpIT.java
@@@ -51,11 -53,12 +51,11 @@@ public class CleanTmpIT extends Configu
private static final Logger log = LoggerFactory.getLogger(CleanTmpIT.class);
@Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
- cfg.setSiteConfig(props);
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
cfg.setNumTservers(1);
- cfg.useMiniDFS(true);
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
index 2fe5470,907d17d..818dbc4
--- a/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/CompactionIT.java
@@@ -58,12 -51,11 +58,12 @@@ public class CompactionIT extends Accum
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
- Map<String,String> map = new HashMap<String,String>();
- map.put(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), "4");
- map.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- map.put(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "1");
- cfg.setSiteConfig(map);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN, "4");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+ cfg.setProperty(Property.TSERV_MAJC_MAXCONCURRENT, "1");
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
index 9a262a1,0000000..819347e
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@@ -1,222 -1,0 +1,222 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class DurabilityIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(DurabilityIT.class);
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ }
+
+ static final long N = 100000;
+
+ private String[] init() throws Exception {
+ String[] tableNames = getUniqueNames(4);
+ Connector c = getConnector();
+ TableOperations tableOps = c.tableOperations();
+ createTable(tableNames[0]);
+ createTable(tableNames[1]);
+ createTable(tableNames[2]);
+ createTable(tableNames[3]);
+ // default is sync
+ tableOps.setProperty(tableNames[1], Property.TABLE_DURABILITY.getKey(), "flush");
+ tableOps.setProperty(tableNames[2], Property.TABLE_DURABILITY.getKey(), "log");
+ tableOps.setProperty(tableNames[3], Property.TABLE_DURABILITY.getKey(), "none");
+ return tableNames;
+ }
+
+ private void cleanup(String[] tableNames) throws Exception {
+ Connector c = getConnector();
+ for (String tableName : tableNames) {
+ c.tableOperations().delete(tableName);
+ }
+ }
+
+ private void createTable(String tableName) throws Exception {
+ TableOperations tableOps = getConnector().tableOperations();
+ tableOps.create(tableName);
+ }
+
+ @Test(timeout = 2 * 60 * 1000)
+ public void testWriteSpeed() throws Exception {
+ TableOperations tableOps = getConnector().tableOperations();
+ String tableNames[] = init();
+ // write some gunk, delete the table to keep that table from messing with the performance numbers of successive calls
+ // sync
+ long t0 = writeSome(tableNames[0], N);
+ tableOps.delete(tableNames[0]);
+ // flush
+ long t1 = writeSome(tableNames[1], N);
+ tableOps.delete(tableNames[1]);
+ // log
+ long t2 = writeSome(tableNames[2], N);
+ tableOps.delete(tableNames[2]);
+ // none
+ long t3 = writeSome(tableNames[3], N);
+ tableOps.delete(tableNames[3]);
+ System.out.println(String.format("sync %d flush %d log %d none %d", t0, t1, t2, t3));
+ assertTrue("flush should be faster than sync", t0 > t1);
+ assertTrue("log should be faster than flush", t1 > t2);
+ assertTrue("no durability should be faster than log", t2 > t3);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testSync() throws Exception {
+ String tableNames[] = init();
+ // sync table should lose nothing
+ writeSome(tableNames[0], N);
+ restartTServer();
+ assertEquals(N, readSome(tableNames[0]));
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testFlush() throws Exception {
+ String tableNames[] = init();
+ // flush table won't lose anything since we're not losing power/dfs
+ writeSome(tableNames[1], N);
+ restartTServer();
+ assertEquals(N, readSome(tableNames[1]));
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testLog() throws Exception {
+ String tableNames[] = init();
+ // we're probably going to lose something the the log setting
+ writeSome(tableNames[2], N);
+ restartTServer();
+ long numResults = readSome(tableNames[2]);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testNone() throws Exception {
+ String tableNames[] = init();
+ // probably won't get any data back without logging
+ writeSome(tableNames[3], N);
+ restartTServer();
+ long numResults = readSome(tableNames[3]);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testIncreaseDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ writeSome(tableName, N);
+ restartTServer();
+ long numResults = readSome(tableName);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ writeSome(tableName, N);
+ restartTServer();
+ assertTrue(N == readSome(tableName));
+ }
+
+ private static Map<String,String> map(Iterable<Entry<String,String>> entries) {
+ Map<String,String> result = new HashMap<String,String>();
+ for (Entry<String,String> entry : entries) {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ return result;
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testMetaDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.instanceOperations().setProperty(Property.TABLE_DURABILITY.getKey(), "none");
+ Map<String,String> props = map(c.tableOperations().getProperties(MetadataTable.NAME));
+ assertEquals("sync", props.get(Property.TABLE_DURABILITY.getKey()));
+ c.tableOperations().create(tableName);
+ props = map(c.tableOperations().getProperties(tableName));
+ assertEquals("none", props.get(Property.TABLE_DURABILITY.getKey()));
+ restartTServer();
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ private long readSome(String table) throws Exception {
+ return Iterators.size(getConnector().createScanner(table, Authorizations.EMPTY).iterator());
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+ private long writeSome(String table, long count) throws Exception {
+ int iterations = 5;
+ long[] attempts = new long[iterations];
+ for (int attempt = 0; attempt < iterations; attempt++) {
+ long now = System.currentTimeMillis();
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(table, null);
+ for (int i = 1; i < count + 1; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put("", "", "");
+ bw.addMutation(m);
+ if (i % (Math.max(1, count / 100)) == 0) {
+ bw.flush();
+ }
+ }
+ bw.close();
+ attempts[attempt] = System.currentTimeMillis() - now;
+ }
+ Arrays.sort(attempts);
+ log.info("Attempt durations: {}", Arrays.toString(attempts));
+ // Return the median duration
+ return attempts[2];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 132cbcc,d5b92cf..6ab0541
--- a/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@@ -80,13 -80,14 +80,13 @@@ public class GarbageCollectorIT extend
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
- Map<String,String> settings = new HashMap<String,String>();
- settings.put(Property.INSTANCE_SECRET.getKey(), OUR_SECRET);
- settings.put(Property.GC_CYCLE_START.getKey(), "1");
- settings.put(Property.GC_CYCLE_DELAY.getKey(), "1");
- settings.put(Property.GC_PORT.getKey(), "0");
- settings.put(Property.TSERV_MAXMEM.getKey(), "5K");
- settings.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- cfg.setSiteConfig(settings);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.INSTANCE_SECRET, OUR_SECRET);
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_PORT, "0");
+ cfg.setProperty(Property.TSERV_MAXMEM, "5K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
// use raw local file system so walogs sync and flush will work
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
index 19908f6,0000000..28c1dfc
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
@@@ -1,188 -1,0 +1,188 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.cluster.ClusterUser;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloIT;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
+import org.apache.accumulo.harness.MiniClusterHarness;
+import org.apache.accumulo.harness.TestingKdc;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * MAC test which uses {@link MiniKdc} to simulate ta secure environment. Can be used as a sanity check for Kerberos/SASL testing.
+ */
+public class KerberosRenewalIT extends AccumuloIT {
+ private static final Logger log = LoggerFactory.getLogger(KerberosRenewalIT.class);
+
+ private static TestingKdc kdc;
+ private static String krbEnabledForITs = null;
+ private static ClusterUser rootUser;
+
+ private static final long TICKET_LIFETIME = 6 * 60 * 1000; // Anything less seems to fail when generating the ticket
+ private static final long TICKET_TEST_LIFETIME = 8 * 60 * 1000; // Run a test for 8 mins
+ private static final long TEST_DURATION = 9 * 60 * 1000; // The test should finish within 9 mins
+
+ @BeforeClass
+ public static void startKdc() throws Exception {
+ // 30s renewal time window
+ kdc = new TestingKdc(TestingKdc.computeKdcDir(), TestingKdc.computeKeytabDir(), TICKET_LIFETIME);
+ kdc.start();
+ krbEnabledForITs = System.getProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION);
+ if (null == krbEnabledForITs || !Boolean.parseBoolean(krbEnabledForITs)) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, "true");
+ }
+ rootUser = kdc.getRootUser();
+ }
+
+ @AfterClass
+ public static void stopKdc() throws Exception {
+ if (null != kdc) {
+ kdc.stop();
+ }
+ if (null != krbEnabledForITs) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, krbEnabledForITs);
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return (int) TEST_DURATION / 1000;
+ }
+
+ private MiniAccumuloClusterImpl mac;
+
+ @Before
+ public void startMac() throws Exception {
+ MiniClusterHarness harness = new MiniClusterHarness();
+ mac = harness.create(this, new PasswordToken("unused"), kdc, new MiniClusterConfigurationCallback() {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ Map<String,String> site = cfg.getSiteConfig();
- site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "10s");
++ site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ // Reduce the period just to make sure we trigger renewal fast
+ site.put(Property.GENERAL_KERBEROS_RENEWAL_PERIOD.getKey(), "5s");
+ cfg.setSiteConfig(site);
+ }
+
+ });
+
+ mac.getConfig().setNumTservers(1);
+ mac.start();
+ // Enabled kerberos auth
+ Configuration conf = new Configuration(false);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ @After
+ public void stopMac() throws Exception {
+ if (null != mac) {
+ mac.stop();
+ }
+ }
+
+ // Intentially setting the Test annotation timeout. We do not want to scale the timeout.
+ @Test(timeout = TEST_DURATION)
+ public void testReadAndWriteThroughTicketLifetime() throws Exception {
+ // Attempt to use Accumulo for a duration of time that exceeds the Kerberos ticket lifetime.
+ // This is a functional test to verify that Accumulo services renew their ticket.
+ // If the test doesn't finish on its own, this signifies that Accumulo services failed
+ // and the test should fail. If Accumulo services renew their ticket, the test case
+ // should exit gracefully on its own.
+
+ // Login as the "root" user
+ UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ long duration = 0;
+ long last = System.currentTimeMillis();
+ // Make sure we have a couple renewals happen
+ while (duration < TICKET_TEST_LIFETIME) {
+ // Create a table, write a record, compact, read the record, drop the table.
+ createReadWriteDrop(conn);
+ // Wait a bit after
+ Thread.sleep(5000);
+
+ // Update the duration
+ long now = System.currentTimeMillis();
+ duration += now - last;
+ last = now;
+ }
+ }
+
+ /**
+ * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure that the system user exists (since the master does an RPC to
+ * the tserver which will create the system user if it doesn't already exist).
+ */
+ private void createReadWriteDrop(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException {
+ final String table = testName.getMethodName() + "_table";
+ conn.tableOperations().create(table);
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", "d");
+ bw.addMutation(m);
+ bw.close();
+ conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true));
+ Scanner s = conn.createScanner(table, Authorizations.EMPTY);
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ assertEquals("Did not find the expected key", 0, new Key("a", "b", "c").compareTo(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL));
+ assertEquals("d", entry.getValue().toString());
+ conn.tableOperations().delete(table);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
index dd83574,0c2631f..49160aa
--- a/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
@@@ -37,9 -35,7 +37,9 @@@ public class MasterFailoverIT extends A
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setSiteConfig(Collections.singletonMap(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s"));
+ Map<String,String> siteConfig = cfg.getSiteConfig();
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
++ siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ cfg.setSiteConfig(siteConfig);
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
index 2ba6d31,b498412..d1fb9f9
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartIT.java
@@@ -70,10 -66,12 +70,10 @@@ public class RestartIT extends Accumulo
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
- Map<String,String> props = new HashMap<String,String>();
- props.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
- props.put(Property.GC_CYCLE_DELAY.getKey(), "1s");
- props.put(Property.GC_CYCLE_START.getKey(), "1s");
- cfg.setSiteConfig(props);
- cfg.useMiniDFS(true);
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
private static final ScannerOpts SOPTS = new ScannerOpts();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
index 3f7d67d,c4b3afd..68448eb
--- a/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/RestartStressIT.java
@@@ -58,10 -55,10 +58,10 @@@ public class RestartStressIT extends Ac
opts.put(Property.TSERV_MAXMEM.getKey(), "100K");
opts.put(Property.TSERV_MAJC_DELAY.getKey(), "100ms");
opts.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "1M");
- opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
+ opts.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
opts.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
cfg.setSiteConfig(opts);
- cfg.useMiniDFS(true);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
index aec6bae,0000000..ca45382
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/SessionDurabilityIT.java
@@@ -1,153 -1,0 +1,153 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.ConditionalWriter;
+import org.apache.accumulo.core.client.ConditionalWriter.Status;
+import org.apache.accumulo.core.client.ConditionalWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Durability;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Condition;
+import org.apache.accumulo.core.data.ConditionalMutation;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+
+public class SessionDurabilityIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void nondurableTableHasDurableWrites() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default has no durability
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ // send durable writes
+ BatchWriterConfig cfg = new BatchWriterConfig();
+ cfg.setDurability(Durability.SYNC);
+ writeSome(tableName, 10, cfg);
+ assertEquals(10, count(tableName));
+ // verify writes servive restart
+ restartTServer();
+ assertEquals(10, count(tableName));
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void durableTableLosesNonDurableWrites() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ // write with no durability
+ BatchWriterConfig cfg = new BatchWriterConfig();
+ cfg.setDurability(Durability.NONE);
+ writeSome(tableName, 10, cfg);
+ // verify writes are lost on restart
+ restartTServer();
+ assertTrue(10 > count(tableName));
+ }
+
+ private int count(String tableName) throws Exception {
+ return Iterators.size(getConnector().createScanner(tableName, Authorizations.EMPTY).iterator());
+ }
+
+ private void writeSome(String tableName, int n, BatchWriterConfig cfg) throws Exception {
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(tableName, cfg);
+ for (int i = 0; i < n; i++) {
+ Mutation m = new Mutation(i + "");
+ m.put("", "", "");
+ bw.addMutation(m);
+ }
+ bw.close();
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void testConditionDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ // write without durability
+ ConditionalWriterConfig cfg = new ConditionalWriterConfig();
+ cfg.setDurability(Durability.NONE);
+ conditionWriteSome(tableName, 10, cfg);
+ // everything in there?
+ assertEquals(10, count(tableName));
+ // restart the server and verify the updates are lost
+ restartTServer();
+ assertEquals(0, count(tableName));
+ }
+
+ @Test(timeout = 3 * 60 * 1000)
+ public void testConditionDurability2() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ // table default is durable writes
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ // write with durability
+ ConditionalWriterConfig cfg = new ConditionalWriterConfig();
+ cfg.setDurability(Durability.SYNC);
+ conditionWriteSome(tableName, 10, cfg);
+ // everything in there?
+ assertEquals(10, count(tableName));
+ // restart the server and verify the updates are still there
+ restartTServer();
+ assertEquals(10, count(tableName));
+ }
+
+ private void conditionWriteSome(String tableName, int n, ConditionalWriterConfig cfg) throws Exception {
+ Connector c = getConnector();
+ ConditionalWriter cw = c.createConditionalWriter(tableName, cfg);
+ for (int i = 0; i < n; i++) {
+ ConditionalMutation m = new ConditionalMutation((CharSequence) (i + ""), new Condition("", ""));
+ m.put("", "", "X");
+ assertEquals(Status.ACCEPTED, cw.write(m).getStatus());
+ }
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
index 44473b0,bfca75b..bc36257
--- a/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/WriteAheadLogIT.java
@@@ -35,13 -35,15 +35,13 @@@ public class WriteAheadLogIT extends Ac
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- Map<String,String> siteConfig = new HashMap<String,String>();
- siteConfig.put(Property.TSERV_WALOG_MAX_SIZE.getKey(), "2M");
- siteConfig.put(Property.GC_CYCLE_DELAY.getKey(), "1");
- siteConfig.put(Property.GC_CYCLE_START.getKey(), "1");
- siteConfig.put(Property.MASTER_RECOVERY_DELAY.getKey(), "1s");
- siteConfig.put(Property.TSERV_MAJC_DELAY.getKey(), "1");
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "4s");
- cfg.setSiteConfig(siteConfig);
- cfg.useMiniDFS(true);
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.MASTER_RECOVERY_DELAY, "1s");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "4s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
index f852901,f852901..fefb9a6
--- a/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ZookeeperRestartIT.java
@@@ -45,7 -45,7 +45,7 @@@ public class ZookeeperRestartIT extend
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
Map<String,String> siteConfig = new HashMap<String,String>();
-- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "3s");
++ siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
cfg.setSiteConfig(siteConfig);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/94f4a19c/test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
----------------------------------------------------------------------
diff --cc test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
index bd10f90,0000000..35bc0fe
mode 100644,000000..100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/MultiInstanceReplicationIT.java
@@@ -1,733 -1,0 +1,733 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.master.replication.SequentialWorkAssigner;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.replication.ReplicaSystemFactory;
+import org.apache.accumulo.server.replication.StatusUtil;
+import org.apache.accumulo.server.replication.proto.Replication.Status;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.tserver.TabletServer;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+/**
+ * Replication tests which start at least two MAC instances and replicate data between them
+ */
+public class MultiInstanceReplicationIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(MultiInstanceReplicationIT.class);
+
+ private ExecutorService executor;
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 10 * 60;
+ }
+
+ @Before
+ public void createExecutor() {
+ executor = Executors.newSingleThreadExecutor();
+ }
+
+ @After
+ public void stopExecutor() {
+ if (null != executor) {
+ executor.shutdownNow();
+ }
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
+ cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "1s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
+ cfg.setProperty(Property.REPLICATION_NAME, "master");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNER, SequentialWorkAssigner.class.getName());
+ cfg.setProperty(Property.TSERV_TOTAL_MUTATION_QUEUE_MAX, "1M");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ /**
+ * Use the same SSL and credential provider configuration that is set up by AbstractMacIT for the other MAC used for replication
+ */
+ private void updatePeerConfigFromPrimary(MiniAccumuloConfigImpl primaryCfg, MiniAccumuloConfigImpl peerCfg) {
+ // Set the same SSL information from the primary when present
+ Map<String,String> primarySiteConfig = primaryCfg.getSiteConfig();
+ if ("true".equals(primarySiteConfig.get(Property.INSTANCE_RPC_SSL_ENABLED.getKey()))) {
+ Map<String,String> peerSiteConfig = new HashMap<String,String>();
+ peerSiteConfig.put(Property.INSTANCE_RPC_SSL_ENABLED.getKey(), "true");
+ String keystorePath = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PATH.getKey());
+ Assert.assertNotNull("Keystore Path was null", keystorePath);
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PATH.getKey(), keystorePath);
+ String truststorePath = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PATH.getKey());
+ Assert.assertNotNull("Truststore Path was null", truststorePath);
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PATH.getKey(), truststorePath);
+
+ // Passwords might be stored in CredentialProvider
+ String keystorePassword = primarySiteConfig.get(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey());
+ if (null != keystorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_KEYSTORE_PASSWORD.getKey(), keystorePassword);
+ }
+ String truststorePassword = primarySiteConfig.get(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey());
+ if (null != truststorePassword) {
+ peerSiteConfig.put(Property.RPC_SSL_TRUSTSTORE_PASSWORD.getKey(), truststorePassword);
+ }
+
+ System.out.println("Setting site configuration for peer " + peerSiteConfig);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+
+ // Use the CredentialProvider if the primary also uses one
+ String credProvider = primarySiteConfig.get(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey());
+ if (null != credProvider) {
+ Map<String,String> peerSiteConfig = peerCfg.getSiteConfig();
+ peerSiteConfig.put(Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS.getKey(), credProvider);
+ peerCfg.setSiteConfig(peerSiteConfig);
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeer() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ try {
+ final Connector connMaster = getConnector();
+ final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ ReplicationTable.setOnline(connMaster);
+
+ String peerUserName = "peer", peerPassword = "foo";
+
+ String peerClusterName = "peer";
+
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ final String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ final Set<String> filesNeedingReplication = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ log.info("Files to replicate: " + filesNeedingReplication);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("TabletServer restarted");
+ Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
+ log.info("TabletServer is online");
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Future<Boolean> future = executor.submit(new Callable<Boolean>() {
+
+ @Override
+ public Boolean call() throws Exception {
+ long then = System.currentTimeMillis();
+ connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
+ long now = System.currentTimeMillis();
+ log.info("Drain completed in " + (now - then) + "ms");
+ return true;
+ }
+
+ });
+
+ try {
+ future.get(60, TimeUnit.SECONDS);
+ } catch (TimeoutException e) {
+ future.cancel(true);
+ Assert.fail("Drain did not finish within 60 seconds");
+ } finally {
+ executor.shutdownNow();
+ }
+
+ log.info("drain completed");
+
+ log.info("");
+ log.info("Fetching metadata records:");
+ for (Entry<Key,Value> kv : connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ } else {
+ log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
+ }
+ }
+
+ log.info("");
+ log.info("Fetching replication records:");
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.info(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ Entry<Key,Value> masterEntry = null, peerEntry = null;
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ masterEntry = masterIter.next();
+ peerEntry = peerIter.next();
+ Assert.assertEquals(masterEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ log.info("Last master entry: " + masterEntry);
+ log.info("Last peer entry: " + peerEntry);
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+ } finally {
+ peerCluster.stop();
+ }
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTable() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+ String peerUserName = "peer", peerPassword = "foo";
+
+ // Create local user
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ // Create tables
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Grant write permission
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ long masterTable1Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable1Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ long masterTable2Records = 0l;
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ masterTable2Records++;
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> filesFor1 = connMaster.replicationOperations().referencedFiles(masterTable1), filesFor2 = connMaster.replicationOperations().referencedFiles(
+ masterTable2);
+
+ log.info("Files to replicate for table1: " + filesFor1);
+ log.info("Files to replicate for table2: " + filesFor2);
+
+ // Restart the tserver to force a close on the WAL
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.exec(TabletServer.class);
+
+ log.info("Restarted the tserver");
+
+ // Read the data -- the tserver is back up and running
+ Iterators.size(connMaster.createScanner(masterTable1, Authorizations.EMPTY).iterator());
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ // Wait for both tables to be replicated
+ log.info("Waiting for {} for {}", filesFor1, masterTable1);
+ connMaster.replicationOperations().drain(masterTable1, filesFor1);
+
+ log.info("Waiting for {} for {}", filesFor2, masterTable2);
+ connMaster.replicationOperations().drain(masterTable2, filesFor2);
+
+ long countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+ Assert.assertEquals(masterTable1Records, countTable);
+
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+ Assert.assertEquals(masterTable2Records, countTable);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+
+ @Test
+ public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Give our replication user the ability to write to the table
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
+
+ log.info("Files to replicate:" + files);
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
+
+ for (Entry<Key,Value> kv : ReplicationTable.getScanner(connMaster)) {
+ log.debug(kv.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
+ }
+
+ connMaster.replicationOperations().drain(masterTable, files);
+
+ Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY), peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> masterIter = master.iterator(), peerIter = peer.iterator();
+ while (masterIter.hasNext() && peerIter.hasNext()) {
+ Entry<Key,Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
+ Assert.assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0,
+ masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
+ Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
+ }
+
+ Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
+ Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
+
+ peerCluster.stop();
+ }
+
+ @Test
+ public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
+
+ updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
+
+ MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
+
+ peer1Cluster.start();
+
+ try {
+ Connector connMaster = getConnector();
+ Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
+
+ String peerClusterName = "peer";
+
+ String peerUserName = "repl";
+ String peerPassword = "passwd";
+
+ // Create a user on the peer for replication to use
+ connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
+
+ // Configure the credentials we should use to authenticate ourselves to the peer for replication
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(
+ Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class,
+ AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
+
+ String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
+
+ connMaster.tableOperations().create(masterTable1);
+ String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
+ Assert.assertNotNull(masterTableId1);
+
+ connMaster.tableOperations().create(masterTable2);
+ String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
+ Assert.assertNotNull(masterTableId2);
+
+ connPeer.tableOperations().create(peerTable1);
+ String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
+ Assert.assertNotNull(peerTableId1);
+
+ connPeer.tableOperations().create(peerTable2);
+ String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
+ Assert.assertNotNull(peerTableId2);
+
+ // Give our replication user the ability to write to the tables
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
+ connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
+
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable1 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ // Write some data to table2
+ bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
+ for (int rows = 0; rows < 2500; rows++) {
+ Mutation m = new Mutation(masterTable2 + rows);
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ log.info("Wrote all data to master cluster");
+
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+
+ cluster.exec(TabletServer.class);
+
+ while (!ReplicationTable.isOnline(connMaster)) {
+ log.info("Replication table still offline, waiting");
+ Thread.sleep(5000);
+ }
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ UtilWaitThread.sleep(2000);
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
+ }
+ }
+
+ Assert.assertNotEquals(0, fullyReplicated);
+
+ // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
+ // Be cautious in how quickly we assert that the data is present on the peer
+ long countTable = 0l;
+ for (int i = 0; i < 10; i++) {
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable1));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable1);
+
+ if (0l == countTable) {
+ Thread.sleep(5000);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertTrue("Found no records in " + peerTable1 + " in the peer cluster", countTable > 0);
+
+ // We have to wait for the master to assign the replication work, a local tserver to process it, and then the remote tserver to replay it
+ // Be cautious in how quickly we assert that the data is present on the peer
+ for (int i = 0; i < 10; i++) {
+ countTable = 0l;
+ for (Entry<Key,Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
+ countTable++;
+ Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString()
+ .startsWith(masterTable2));
+ }
+
+ log.info("Found {} records in {}", countTable, peerTable2);
+
+ if (0l == countTable) {
+ Thread.sleep(5000);
+ } else {
+ break;
+ }
+ }
+
+ Assert.assertTrue("Found no records in " + peerTable2 + " in the peer cluster", countTable > 0);
+
+ } finally {
+ peer1Cluster.stop();
+ }
+ }
+}
[12/13] accumulo git commit: Merge branch '1.7'
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/CleanTmpIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CleanTmpIT.java
index 779b407,0000000..751e827
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CleanTmpIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CleanTmpIT.java
@@@ -1,112 -1,0 +1,112 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+
+public class CleanTmpIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(CleanTmpIT.class);
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "3s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ // make a table
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ // write to it
+ BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
+ Mutation m = new Mutation("row");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.flush();
+
+ // Compact memory to make a file
+ c.tableOperations().compact(tableName, null, null, true, true);
+
+ // Make sure that we'll have a WAL
+ m = new Mutation("row2");
+ m.put("cf", "cq", "value");
+ bw.addMutation(m);
+ bw.close();
+
+ // create a fake _tmp file in its directory
+ String id = c.tableOperations().tableIdMap().get(tableName);
+ Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(Range.prefix(id));
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ Path file = new Path(entry.getKey().getColumnQualifier().toString());
+
+ FileSystem fs = getCluster().getFileSystem();
+ assertTrue("Could not find file: " + file, fs.exists(file));
+ Path tabletDir = file.getParent();
+ assertNotNull("Tablet dir should not be null", tabletDir);
+ Path tmp = new Path(tabletDir, "junk.rf_tmp");
+ // Make the file
+ fs.create(tmp).close();
+ log.info("Created tmp file {}", tmp.toString());
+ getCluster().stop();
+ getCluster().start();
+
+ Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
+ assertEquals(2, Iterators.size(scanner.iterator()));
+ // If we performed log recovery, we should have cleaned up any stray files
+ assertFalse("File still exists: " + tmp, fs.exists(tmp));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index 003d66f,0000000..ee181df
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@@ -1,185 -1,0 +1,185 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.accumulo.core.cli.ClientOpts.Password;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+public class CompactionIT extends AccumuloClusterHarness {
+ private static final Logger log = LoggerFactory.getLogger(CompactionIT.class);
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN, "4");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+ cfg.setProperty(Property.TSERV_MAJC_MAXCONCURRENT, "1");
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 4 * 60;
+ }
+
+ private String majcThreadMaxOpen, majcDelay, majcMaxConcurrent;
+
+ @Before
+ public void alterConfig() throws Exception {
+ if (ClusterType.STANDALONE == getClusterType()) {
+ InstanceOperations iops = getConnector().instanceOperations();
+ Map<String,String> config = iops.getSystemConfiguration();
+ majcThreadMaxOpen = config.get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey());
+ majcDelay = config.get(Property.TSERV_MAJC_DELAY.getKey());
+ majcMaxConcurrent = config.get(Property.TSERV_MAJC_MAXCONCURRENT.getKey());
+
+ iops.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), "4");
+ iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), "1");
+ iops.setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), "1");
+
+ getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+ }
+ }
+
+ @After
+ public void resetConfig() throws Exception {
+ // We set the values..
+ if (null != majcThreadMaxOpen) {
+ InstanceOperations iops = getConnector().instanceOperations();
+
+ iops.setProperty(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey(), majcThreadMaxOpen);
+ iops.setProperty(Property.TSERV_MAJC_DELAY.getKey(), majcDelay);
+ iops.setProperty(Property.TSERV_MAJC_MAXCONCURRENT.getKey(), majcMaxConcurrent);
+
+ getClusterControl().stopAllServers(ServerType.TABLET_SERVER);
+ getClusterControl().startAllServers(ServerType.TABLET_SERVER);
+ }
+ }
+
+ @Test
+ public void test() throws Exception {
+ final Connector c = getConnector();
+ final String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1.0");
+ FileSystem fs = getFileSystem();
+ Path root = new Path(cluster.getTemporaryPath(), getClass().getName());
+ Path testrf = new Path(root, "testrf");
+ FunctionalTestUtils.createRFiles(c, fs, testrf.toString(), 500000, 59, 4);
+
+ FunctionalTestUtils.bulkImport(c, fs, tableName, testrf.toString());
+ int beforeCount = countFiles(c);
+
+ final AtomicBoolean fail = new AtomicBoolean(false);
+ final ClientConfiguration clientConf = cluster.getClientConfig();
+ final int THREADS = 5;
+ for (int count = 0; count < THREADS; count++) {
+ ExecutorService executor = Executors.newFixedThreadPool(THREADS);
+ final int span = 500000 / 59;
+ for (int i = 0; i < 500000; i += 500000 / 59) {
+ final int finalI = i;
+ Runnable r = new Runnable() {
+ @Override
+ public void run() {
+ try {
+ VerifyIngest.Opts opts = new VerifyIngest.Opts();
+ opts.startRow = finalI;
+ opts.rows = span;
+ opts.random = 56;
+ opts.dataSize = 50;
+ opts.cols = 1;
+ opts.setTableName(tableName);
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConf);
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ PasswordToken passwordToken = (PasswordToken) getAdminToken();
+ opts.setPassword(new Password(new String(passwordToken.getPassword(), UTF_8)));
+ }
+ VerifyIngest.verifyIngest(c, opts, new ScannerOpts());
+ } catch (Exception ex) {
+ log.warn("Got exception verifying data", ex);
+ fail.set(true);
+ }
+ }
+ };
+ executor.execute(r);
+ }
+ executor.shutdown();
+ executor.awaitTermination(defaultTimeoutSeconds(), TimeUnit.SECONDS);
+ assertFalse("Failed to successfully run all threads, Check the test output for error", fail.get());
+ }
+
+ int finalCount = countFiles(c);
+ assertTrue(finalCount < beforeCount);
+ try {
+ getClusterControl().adminStopAll();
+ } finally {
+ // Make sure the internal state in the cluster is reset (e.g. processes in MAC)
+ getCluster().stop();
+ if (ClusterType.STANDALONE == getClusterType()) {
+ // Then restart things for the next test if it's a standalone
+ getCluster().start();
+ }
+ }
+ }
+
+ private int countFiles(Connector c) throws Exception {
+ Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.TabletColumnFamily.NAME);
+ s.fetchColumnFamily(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME);
+ return Iterators.size(s.iterator());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
index ce9ad85,0000000..4078e69
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/DurabilityIT.java
@@@ -1,233 -1,0 +1,233 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeFalse;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.test.mrit.IntegrationTestMapReduce;
+import org.apache.accumulo.test.PerformanceTest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterators;
+
+@Category(PerformanceTest.class)
+public class DurabilityIT extends ConfigurableMacBase {
+ private static final Logger log = LoggerFactory.getLogger(DurabilityIT.class);
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setNumTservers(1);
+ }
+
+ @BeforeClass
+ static public void checkMR() {
+ assumeFalse(IntegrationTestMapReduce.isMapReduce());
+ }
+
+ static final long N = 100000;
+
+ private String[] init() throws Exception {
+ String[] tableNames = getUniqueNames(4);
+ Connector c = getConnector();
+ TableOperations tableOps = c.tableOperations();
+ createTable(tableNames[0]);
+ createTable(tableNames[1]);
+ createTable(tableNames[2]);
+ createTable(tableNames[3]);
+ // default is sync
+ tableOps.setProperty(tableNames[1], Property.TABLE_DURABILITY.getKey(), "flush");
+ tableOps.setProperty(tableNames[2], Property.TABLE_DURABILITY.getKey(), "log");
+ tableOps.setProperty(tableNames[3], Property.TABLE_DURABILITY.getKey(), "none");
+ return tableNames;
+ }
+
+ private void cleanup(String[] tableNames) throws Exception {
+ Connector c = getConnector();
+ for (String tableName : tableNames) {
+ c.tableOperations().delete(tableName);
+ }
+ }
+
+ private void createTable(String tableName) throws Exception {
+ TableOperations tableOps = getConnector().tableOperations();
+ tableOps.create(tableName);
+ }
+
+ @Test(timeout = 2 * 60 * 1000)
+ public void testWriteSpeed() throws Exception {
+ TableOperations tableOps = getConnector().tableOperations();
+ String tableNames[] = init();
+ // write some gunk, delete the table to keep that table from messing with the performance numbers of successive calls
+ // sync
+ long t0 = writeSome(tableNames[0], N);
+ tableOps.delete(tableNames[0]);
+ // flush
+ long t1 = writeSome(tableNames[1], N);
+ tableOps.delete(tableNames[1]);
+ // log
+ long t2 = writeSome(tableNames[2], N);
+ tableOps.delete(tableNames[2]);
+ // none
+ long t3 = writeSome(tableNames[3], N);
+ tableOps.delete(tableNames[3]);
+ System.out.println(String.format("sync %d flush %d log %d none %d", t0, t1, t2, t3));
+ assertTrue("flush should be faster than sync", t0 > t1);
+ assertTrue("log should be faster than flush", t1 > t2);
+ assertTrue("no durability should be faster than log", t2 > t3);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testSync() throws Exception {
+ String tableNames[] = init();
+ // sync table should lose nothing
+ writeSome(tableNames[0], N);
+ restartTServer();
+ assertEquals(N, readSome(tableNames[0]));
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testFlush() throws Exception {
+ String tableNames[] = init();
+ // flush table won't lose anything since we're not losing power/dfs
+ writeSome(tableNames[1], N);
+ restartTServer();
+ assertEquals(N, readSome(tableNames[1]));
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testLog() throws Exception {
+ String tableNames[] = init();
+ // we're probably going to lose something the the log setting
+ writeSome(tableNames[2], N);
+ restartTServer();
+ long numResults = readSome(tableNames[2]);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testNone() throws Exception {
+ String tableNames[] = init();
+ // probably won't get any data back without logging
+ writeSome(tableNames[3], N);
+ restartTServer();
+ long numResults = readSome(tableNames[3]);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ cleanup(tableNames);
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testIncreaseDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "none");
+ writeSome(tableName, N);
+ restartTServer();
+ long numResults = readSome(tableName);
+ assertTrue("Expected " + N + " >= " + numResults, N >= numResults);
+ c.tableOperations().setProperty(tableName, Property.TABLE_DURABILITY.getKey(), "sync");
+ writeSome(tableName, N);
+ restartTServer();
+ assertTrue(N == readSome(tableName));
+ }
+
+ private static Map<String,String> map(Iterable<Entry<String,String>> entries) {
+ Map<String,String> result = new HashMap<String,String>();
+ for (Entry<String,String> entry : entries) {
+ result.put(entry.getKey(), entry.getValue());
+ }
+ return result;
+ }
+
+ @Test(timeout = 4 * 60 * 1000)
+ public void testMetaDurability() throws Exception {
+ Connector c = getConnector();
+ String tableName = getUniqueNames(1)[0];
+ c.instanceOperations().setProperty(Property.TABLE_DURABILITY.getKey(), "none");
+ Map<String,String> props = map(c.tableOperations().getProperties(MetadataTable.NAME));
+ assertEquals("sync", props.get(Property.TABLE_DURABILITY.getKey()));
+ c.tableOperations().create(tableName);
+ props = map(c.tableOperations().getProperties(tableName));
+ assertEquals("none", props.get(Property.TABLE_DURABILITY.getKey()));
+ restartTServer();
+ assertTrue(c.tableOperations().exists(tableName));
+ }
+
+ private long readSome(String table) throws Exception {
+ return Iterators.size(getConnector().createScanner(table, Authorizations.EMPTY).iterator());
+ }
+
+ private void restartTServer() throws Exception {
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, proc);
+ }
+ cluster.start();
+ }
+
+ private long writeSome(String table, long count) throws Exception {
+ int iterations = 5;
+ long[] attempts = new long[iterations];
+ for (int attempt = 0; attempt < iterations; attempt++) {
+ long now = System.currentTimeMillis();
+ Connector c = getConnector();
+ BatchWriter bw = c.createBatchWriter(table, null);
+ for (int i = 1; i < count + 1; i++) {
+ Mutation m = new Mutation("" + i);
+ m.put("", "", "");
+ bw.addMutation(m);
+ if (i % (Math.max(1, count / 100)) == 0) {
+ bw.flush();
+ }
+ }
+ bw.close();
+ attempts[attempt] = System.currentTimeMillis() - now;
+ }
+ Arrays.sort(attempts);
+ log.info("Attempt durations: {}", Arrays.toString(attempts));
+ // Return the median duration
+ return attempts[2];
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 7f11851,0000000..25e541b
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@@ -1,309 -1,0 +1,309 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.core.util.ServerServices;
+import org.apache.accumulo.core.util.ServerServices.Service;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooLock;
+import org.apache.accumulo.gc.SimpleGarbageCollector;
+import org.apache.accumulo.minicluster.MemoryUnit;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessNotFoundException;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
+import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.Iterators;
+import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
+
+public class GarbageCollectorIT extends ConfigurableMacBase {
+ private static final String OUR_SECRET = "itsreallysecret";
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 5 * 60;
+ }
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "5s");
++ cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s");
+ cfg.setProperty(Property.INSTANCE_SECRET, OUR_SECRET);
+ cfg.setProperty(Property.GC_CYCLE_START, "1");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1");
+ cfg.setProperty(Property.GC_PORT, "0");
+ cfg.setProperty(Property.TSERV_MAXMEM, "5K");
+ cfg.setProperty(Property.TSERV_MAJC_DELAY, "1");
+
+ // use raw local file system so walogs sync and flush will work
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ private void killMacGc() throws ProcessNotFoundException, InterruptedException, KeeperException {
+ // kill gc started by MAC
+ getCluster().killProcess(ServerType.GARBAGE_COLLECTOR, getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR).iterator().next());
+ // delete lock in zookeeper if there, this will allow next GC to start quickly
+ String path = ZooUtil.getRoot(new ZooKeeperInstance(getCluster().getClientConfig())) + Constants.ZGC_LOCK;
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ try {
+ ZooLock.deleteLock(zk, path);
+ } catch (IllegalStateException e) {
+
+ }
+
+ assertNull(getCluster().getProcesses().get(ServerType.GARBAGE_COLLECTOR));
+ }
+
+ @Test
+ public void gcTest() throws Exception {
+ killMacGc();
+ Connector c = getConnector();
+ c.tableOperations().create("test_ingest");
+ c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "5K");
+ TestIngest.Opts opts = new TestIngest.Opts();
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.rows = opts.rows = 10000;
+ vopts.cols = opts.cols = 1;
+ opts.setPrincipal("root");
+ vopts.setPrincipal("root");
+ TestIngest.ingest(c, cluster.getFileSystem(), opts, new BatchWriterOpts());
+ c.tableOperations().compact("test_ingest", null, null, true, true);
+ int before = countFiles();
+ while (true) {
+ sleepUninterruptibly(1, TimeUnit.SECONDS);
+ int more = countFiles();
+ if (more <= before)
+ break;
+ before = more;
+ }
+
+ // restart GC
+ getCluster().start();
+ sleepUninterruptibly(15, TimeUnit.SECONDS);
+ int after = countFiles();
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ assertTrue(after < before);
+ }
+
+ @Test
+ public void gcLotsOfCandidatesIT() throws Exception {
+ killMacGc();
+
+ log.info("Filling metadata table with bogus delete flags");
+ Connector c = getConnector();
+ addEntries(c, new BatchWriterOpts());
+ cluster.getConfig().setDefaultMemory(10, MemoryUnit.MEGABYTE);
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ sleepUninterruptibly(20, TimeUnit.SECONDS);
+ String output = "";
+ while (!output.contains("delete candidates has exceeded")) {
+ byte buffer[] = new byte[10 * 1024];
+ try {
+ int n = gc.getInputStream().read(buffer);
+ output = new String(buffer, 0, n, UTF_8);
+ } catch (IOException ex) {
+ break;
+ }
+ }
+ gc.destroy();
+ assertTrue(output.contains("delete candidates has exceeded"));
+ }
+
+ @Test
+ public void dontGCRootLog() throws Exception {
+ killMacGc();
+ // dirty metadata
+ Connector c = getConnector();
+ String table = getUniqueNames(1)[0];
+ c.tableOperations().create(table);
+ // let gc run for a bit
+ cluster.start();
+ sleepUninterruptibly(20, TimeUnit.SECONDS);
+ killMacGc();
+ // kill tservers
+ for (ProcessReference ref : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
+ cluster.killProcess(ServerType.TABLET_SERVER, ref);
+ }
+ // run recovery
+ cluster.start();
+ // did it recover?
+ Scanner scanner = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ Iterators.size(scanner.iterator());
+ }
+
+ private Mutation createDelMutation(String path, String cf, String cq, String val) {
+ Text row = new Text(MetadataSchema.DeletesSection.getRowPrefix() + path);
+ Mutation delFlag = new Mutation(row);
+ delFlag.put(cf, cq, val);
+ return delFlag;
+ }
+
+ @Test
+ public void testInvalidDelete() throws Exception {
+ killMacGc();
+
+ String table = getUniqueNames(1)[0];
+ getConnector().tableOperations().create(table);
+
+ BatchWriter bw2 = getConnector().createBatchWriter(table, new BatchWriterConfig());
+ Mutation m1 = new Mutation("r1");
+ m1.put("cf1", "cq1", "v1");
+ bw2.addMutation(m1);
+ bw2.close();
+
+ getConnector().tableOperations().flush(table, null, null, true);
+
+ // ensure an invalid delete entry does not cause GC to go berserk ACCUMULO-2520
+ getConnector().securityOperations().grantTablePermission(getConnector().whoami(), MetadataTable.NAME, TablePermission.WRITE);
+ BatchWriter bw3 = getConnector().createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ bw3.addMutation(createDelMutation("", "", "", ""));
+ bw3.addMutation(createDelMutation("", "testDel", "test", "valueTest"));
+ bw3.addMutation(createDelMutation("/", "", "", ""));
+ bw3.close();
+
+ Process gc = cluster.exec(SimpleGarbageCollector.class);
+ try {
+ String output = "";
+ while (!output.contains("Ingoring invalid deletion candidate")) {
+ sleepUninterruptibly(250, TimeUnit.MILLISECONDS);
+ try {
+ output = FunctionalTestUtils.readAll(cluster, SimpleGarbageCollector.class, gc);
+ } catch (IOException ioe) {
+ log.error("Could not read all from cluster.", ioe);
+ }
+ }
+ } finally {
+ gc.destroy();
+ }
+
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+ Iterator<Entry<Key,Value>> iter = scanner.iterator();
+ assertTrue(iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ Assert.assertEquals("r1", entry.getKey().getRow().toString());
+ Assert.assertEquals("cf1", entry.getKey().getColumnFamily().toString());
+ Assert.assertEquals("cq1", entry.getKey().getColumnQualifier().toString());
+ Assert.assertEquals("v1", entry.getValue().toString());
+ Assert.assertFalse(iter.hasNext());
+ }
+
+ @Test
+ public void testProperPortAdvertisement() throws Exception {
+
+ Connector conn = getConnector();
+ Instance instance = conn.getInstance();
+
+ ZooReaderWriter zk = new ZooReaderWriter(cluster.getZooKeepers(), 30000, OUR_SECRET);
+ String path = ZooUtil.getRoot(instance) + Constants.ZGC_LOCK;
+ for (int i = 0; i < 5; i++) {
+ List<String> locks;
+ try {
+ locks = zk.getChildren(path, null);
+ } catch (NoNodeException e) {
+ Thread.sleep(5000);
+ continue;
+ }
+
+ if (locks != null && locks.size() > 0) {
+ Collections.sort(locks);
+
+ String lockPath = path + "/" + locks.get(0);
+
+ String gcLoc = new String(zk.getData(lockPath, null));
+
+ Assert.assertTrue("Found unexpected data in zookeeper for GC location: " + gcLoc, gcLoc.startsWith(Service.GC_CLIENT.name()));
+ int loc = gcLoc.indexOf(ServerServices.SEPARATOR_CHAR);
+ Assert.assertNotEquals("Could not find split point of GC location for: " + gcLoc, -1, loc);
+ String addr = gcLoc.substring(loc + 1);
+
+ int addrSplit = addr.indexOf(':');
+ Assert.assertNotEquals("Could not find split of GC host:port for: " + addr, -1, addrSplit);
+
+ String host = addr.substring(0, addrSplit), port = addr.substring(addrSplit + 1);
+ // We shouldn't have the "bindall" address in zk
+ Assert.assertNotEquals("0.0.0.0", host);
+ // Nor should we have the "random port" in zk
+ Assert.assertNotEquals(0, Integer.parseInt(port));
+ return;
+ }
+
+ Thread.sleep(5000);
+ }
+
+ Assert.fail("Could not find advertised GC address");
+ }
+
+ private int countFiles() throws Exception {
+ Path path = new Path(cluster.getConfig().getDir() + "/accumulo/tables/1/*/*.rf");
+ return Iterators.size(Arrays.asList(cluster.getFileSystem().globStatus(path)).iterator());
+ }
+
+ public static void addEntries(Connector conn, BatchWriterOpts bwOpts) throws Exception {
+ conn.securityOperations().grantTablePermission(conn.whoami(), MetadataTable.NAME, TablePermission.WRITE);
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, bwOpts.getBatchWriterConfig());
+
+ for (int i = 0; i < 100000; ++i) {
+ final Text emptyText = new Text("");
+ Text row = new Text(String.format("%s/%020d/%s", MetadataSchema.DeletesSection.getRowPrefix(), i,
+ "aaaaaaaaaabbbbbbbbbbccccccccccddddddddddeeeeeeeeeeffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj"));
+ Mutation delFlag = new Mutation(row);
+ delFlag.put(emptyText, emptyText, new Value(new byte[] {}));
+ bw.addMutation(delFlag);
+ }
+ bw.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/KerberosIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/KerberosIT.java
index 6b6108a,0000000..f7a151e
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/KerberosIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/KerberosIT.java
@@@ -1,644 -1,0 +1,644 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.security.PrivilegedExceptionAction;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.accumulo.cluster.ClusterUser;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
+import org.apache.accumulo.core.client.admin.DelegationTokenConfig;
+import org.apache.accumulo.core.client.impl.AuthenticationTokenIdentifier;
+import org.apache.accumulo.core.client.impl.DelegationTokenImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.ColumnVisibility;
+import org.apache.accumulo.core.security.SystemPermission;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.harness.AccumuloITBase;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
+import org.apache.accumulo.harness.MiniClusterHarness;
+import org.apache.accumulo.harness.TestingKdc;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+/**
+ * MAC test which uses {@link MiniKdc} to simulate ta secure environment. Can be used as a sanity check for Kerberos/SASL testing.
+ */
+public class KerberosIT extends AccumuloITBase {
+ private static final Logger log = LoggerFactory.getLogger(KerberosIT.class);
+
+ private static TestingKdc kdc;
+ private static String krbEnabledForITs = null;
+ private static ClusterUser rootUser;
+
+ @BeforeClass
+ public static void startKdc() throws Exception {
+ kdc = new TestingKdc();
+ kdc.start();
+ krbEnabledForITs = System.getProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION);
+ if (null == krbEnabledForITs || !Boolean.parseBoolean(krbEnabledForITs)) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, "true");
+ }
+ rootUser = kdc.getRootUser();
+ }
+
+ @AfterClass
+ public static void stopKdc() throws Exception {
+ if (null != kdc) {
+ kdc.stop();
+ }
+ if (null != krbEnabledForITs) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, krbEnabledForITs);
+ }
+ UserGroupInformation.setConfiguration(new Configuration(false));
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 60 * 5;
+ }
+
+ private MiniAccumuloClusterImpl mac;
+
+ @Before
+ public void startMac() throws Exception {
+ MiniClusterHarness harness = new MiniClusterHarness();
+ mac = harness.create(this, new PasswordToken("unused"), kdc, new MiniClusterConfigurationCallback() {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ Map<String,String> site = cfg.getSiteConfig();
- site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "10s");
++ site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ cfg.setSiteConfig(site);
+ }
+
+ });
+
+ mac.getConfig().setNumTservers(1);
+ mac.start();
+ // Enabled kerberos auth
+ Configuration conf = new Configuration(false);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ @After
+ public void stopMac() throws Exception {
+ if (null != mac) {
+ mac.stop();
+ }
+ }
+
+ @Test
+ public void testAdminUser() throws Exception {
+ // Login as the client (provided to `accumulo init` as the "root" user)
+ UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ final Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+
+ // The "root" user should have all system permissions
+ for (SystemPermission perm : SystemPermission.values()) {
+ assertTrue("Expected user to have permission: " + perm, conn.securityOperations().hasSystemPermission(conn.whoami(), perm));
+ }
+
+ // and the ability to modify the root and metadata tables
+ for (String table : Arrays.asList(RootTable.NAME, MetadataTable.NAME)) {
+ assertTrue(conn.securityOperations().hasTablePermission(conn.whoami(), table, TablePermission.ALTER_TABLE));
+ }
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testNewUser() throws Exception {
+ String newUser = testName.getMethodName();
+ final File newUserKeytab = new File(kdc.getKeytabDir(), newUser + ".keytab");
+ if (newUserKeytab.exists() && !newUserKeytab.delete()) {
+ log.warn("Unable to delete {}", newUserKeytab);
+ }
+
+ // Create a new user
+ kdc.createPrincipal(newUserKeytab, newUser);
+
+ final String newQualifiedUser = kdc.qualifyUser(newUser);
+ final HashSet<String> users = Sets.newHashSet(rootUser.getPrincipal());
+
+ // Login as the "root" user
+ UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ // Make sure the system user doesn't exist -- this will force some RPC to happen server-side
+ createTableWithDataAndCompact(conn);
+
+ assertEquals(users, conn.securityOperations().listLocalUsers());
+
+ return null;
+ }
+ });
+ // Switch to a new user
+ ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(newQualifiedUser, newUserKeytab.getAbsolutePath());
+ log.info("Logged in as {}", newQualifiedUser);
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(newQualifiedUser, new KerberosToken());
+ log.info("Created connector as {}", newQualifiedUser);
+ assertEquals(newQualifiedUser, conn.whoami());
+
+ // The new user should have no system permissions
+ for (SystemPermission perm : SystemPermission.values()) {
+ assertFalse(conn.securityOperations().hasSystemPermission(newQualifiedUser, perm));
+ }
+
+ users.add(newQualifiedUser);
+
+ // Same users as before, plus the new user we just created
+ assertEquals(users, conn.securityOperations().listLocalUsers());
+ return null;
+ }
+
+ });
+ }
+
+ @Test
+ public void testUserPrivilegesThroughGrant() throws Exception {
+ String user1 = testName.getMethodName();
+ final File user1Keytab = new File(kdc.getKeytabDir(), user1 + ".keytab");
+ if (user1Keytab.exists() && !user1Keytab.delete()) {
+ log.warn("Unable to delete {}", user1Keytab);
+ }
+
+ // Create some new users
+ kdc.createPrincipal(user1Keytab, user1);
+
+ final String qualifiedUser1 = kdc.qualifyUser(user1);
+
+ // Log in as user1
+ UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(user1, user1Keytab.getAbsolutePath());
+ log.info("Logged in as {}", user1);
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Indirectly creates this user when we use it
+ Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken());
+ log.info("Created connector as {}", qualifiedUser1);
+
+ // The new user should have no system permissions
+ for (SystemPermission perm : SystemPermission.values()) {
+ assertFalse(conn.securityOperations().hasSystemPermission(qualifiedUser1, perm));
+ }
+
+ return null;
+ }
+ });
+
+ ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ conn.securityOperations().grantSystemPermission(qualifiedUser1, SystemPermission.CREATE_TABLE);
+ return null;
+ }
+ });
+
+ // Switch back to the original user
+ ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(user1, user1Keytab.getAbsolutePath());
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken());
+
+ // Shouldn't throw an exception since we granted the create table permission
+ final String table = testName.getMethodName() + "_user_table";
+ conn.tableOperations().create(table);
+
+ // Make sure we can actually use the table we made
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", "d");
+ bw.addMutation(m);
+ bw.close();
+
+ conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testUserPrivilegesForTable() throws Exception {
+ String user1 = testName.getMethodName();
+ final File user1Keytab = new File(kdc.getKeytabDir(), user1 + ".keytab");
+ if (user1Keytab.exists() && !user1Keytab.delete()) {
+ log.warn("Unable to delete {}", user1Keytab);
+ }
+
+ // Create some new users -- cannot contain realm
+ kdc.createPrincipal(user1Keytab, user1);
+
+ final String qualifiedUser1 = kdc.qualifyUser(user1);
+
+ // Log in as user1
+ UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(qualifiedUser1, user1Keytab.getAbsolutePath());
+ log.info("Logged in as {}", user1);
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // Indirectly creates this user when we use it
+ Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken());
+ log.info("Created connector as {}", qualifiedUser1);
+
+ // The new user should have no system permissions
+ for (SystemPermission perm : SystemPermission.values()) {
+ assertFalse(conn.securityOperations().hasSystemPermission(qualifiedUser1, perm));
+ }
+ return null;
+ }
+
+ });
+
+ final String table = testName.getMethodName() + "_user_table";
+ final String viz = "viz";
+
+ ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ conn.tableOperations().create(table);
+ // Give our unprivileged user permission on the table we made for them
+ conn.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.READ);
+ conn.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.WRITE);
+ conn.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.ALTER_TABLE);
+ conn.securityOperations().grantTablePermission(qualifiedUser1, table, TablePermission.DROP_TABLE);
+ conn.securityOperations().changeUserAuthorizations(qualifiedUser1, new Authorizations(viz));
+ return null;
+ }
+ });
+
+ // Switch back to the original user
+ ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(qualifiedUser1, user1Keytab.getAbsolutePath());
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(qualifiedUser1, new KerberosToken());
+
+ // Make sure we can actually use the table we made
+
+ // Write data
+ final long ts = 1000l;
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", new ColumnVisibility(viz.getBytes()), ts, "d");
+ bw.addMutation(m);
+ bw.close();
+
+ // Compact
+ conn.tableOperations().compact(table, new CompactionConfig().setWait(true).setFlush(true));
+
+ // Alter
+ conn.tableOperations().setProperty(table, Property.TABLE_BLOOM_ENABLED.getKey(), "true");
+
+ // Read (and proper authorizations)
+ Scanner s = conn.createScanner(table, new Authorizations(viz));
+ Iterator<Entry<Key,Value>> iter = s.iterator();
+ assertTrue("No results from iterator", iter.hasNext());
+ Entry<Key,Value> entry = iter.next();
+ assertEquals(new Key("a", "b", "c", viz, ts), entry.getKey());
+ assertEquals(new Value("d".getBytes()), entry.getValue());
+ assertFalse("Had more results from iterator", iter.hasNext());
+ return null;
+ }
+ });
+ }
+
+ @Test
+ public void testDelegationToken() throws Exception {
+ final String tableName = getUniqueNames(1)[0];
+
+ // Login as the "root" user
+ UserGroupInformation root = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ final int numRows = 100, numColumns = 10;
+
+ // As the "root" user, open up the connection and get a delegation token
+ final AuthenticationToken delegationToken = root.doAs(new PrivilegedExceptionAction<AuthenticationToken>() {
+ @Override
+ public AuthenticationToken run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ conn.tableOperations().create(tableName);
+ BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
+ for (int r = 0; r < numRows; r++) {
+ Mutation m = new Mutation(Integer.toString(r));
+ for (int c = 0; c < numColumns; c++) {
+ String col = Integer.toString(c);
+ m.put(col, col, col);
+ }
+ bw.addMutation(m);
+ }
+ bw.close();
+
+ return conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+ }
+ });
+
+ // The above login with keytab doesn't have a way to logout, so make a fake user that won't have krb credentials
+ UserGroupInformation userWithoutPrivs = UserGroupInformation.createUserForTesting("fake_user", new String[0]);
+ int recordsSeen = userWithoutPrivs.doAs(new PrivilegedExceptionAction<Integer>() {
+ @Override
+ public Integer run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), delegationToken);
+
+ BatchScanner bs = conn.createBatchScanner(tableName, Authorizations.EMPTY, 2);
+ bs.setRanges(Collections.singleton(new Range()));
+ int recordsSeen = Iterables.size(bs);
+ bs.close();
+ return recordsSeen;
+ }
+ });
+
+ assertEquals(numRows * numColumns, recordsSeen);
+ }
+
+ @Test
+ public void testDelegationTokenAsDifferentUser() throws Exception {
+ // Login as the "root" user
+ UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ final AuthenticationToken delegationToken;
+ try {
+ delegationToken = ugi.doAs(new PrivilegedExceptionAction<AuthenticationToken>() {
+ @Override
+ public AuthenticationToken run() throws Exception {
+ // As the "root" user, open up the connection and get a delegation token
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+ return conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+ }
+ });
+ } catch (UndeclaredThrowableException ex) {
+ throw ex;
+ }
+
+ // make a fake user that won't have krb credentials
+ UserGroupInformation userWithoutPrivs = UserGroupInformation.createUserForTesting("fake_user", new String[0]);
+ try {
+ // Use the delegation token to try to log in as a different user
+ userWithoutPrivs.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ mac.getConnector("some_other_user", delegationToken);
+ return null;
+ }
+ });
+ fail("Using a delegation token as a different user should throw an exception");
+ } catch (UndeclaredThrowableException e) {
+ Throwable cause = e.getCause();
+ assertNotNull(cause);
+ // We should get an AccumuloSecurityException from trying to use a delegation token for the wrong user
+ assertTrue("Expected cause to be AccumuloSecurityException, but was " + cause.getClass(), cause instanceof AccumuloSecurityException);
+ }
+ }
+
+ @Test
+ public void testGetDelegationTokenDenied() throws Exception {
+ String newUser = testName.getMethodName();
+ final File newUserKeytab = new File(kdc.getKeytabDir(), newUser + ".keytab");
+ if (newUserKeytab.exists() && !newUserKeytab.delete()) {
+ log.warn("Unable to delete {}", newUserKeytab);
+ }
+
+ // Create a new user
+ kdc.createPrincipal(newUserKeytab, newUser);
+
+ final String qualifiedNewUser = kdc.qualifyUser(newUser);
+
+ // Login as a normal user
+ UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(qualifiedNewUser, newUserKeytab.getAbsolutePath());
+ try {
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ // As the "root" user, open up the connection and get a delegation token
+ Connector conn = mac.getConnector(qualifiedNewUser, new KerberosToken());
+ log.info("Created connector as {}", qualifiedNewUser);
+ assertEquals(qualifiedNewUser, conn.whoami());
+
+ conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+ return null;
+ }
+ });
+ } catch (UndeclaredThrowableException ex) {
+ assertTrue(ex.getCause() instanceof AccumuloSecurityException);
+ }
+ }
+
+ @Test
+ public void testRestartedMasterReusesSecretKey() throws Exception {
+ // Login as the "root" user
+ UserGroupInformation root = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ // As the "root" user, open up the connection and get a delegation token
+ final AuthenticationToken delegationToken1 = root.doAs(new PrivilegedExceptionAction<AuthenticationToken>() {
+ @Override
+ public AuthenticationToken run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ AuthenticationToken token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+
+ assertTrue("Could not get tables with delegation token", mac.getConnector(rootUser.getPrincipal(), token).tableOperations().list().size() > 0);
+
+ return token;
+ }
+ });
+
+ log.info("Stopping master");
+ mac.getClusterControl().stop(ServerType.MASTER);
+ Thread.sleep(5000);
+ log.info("Restarting master");
+ mac.getClusterControl().start(ServerType.MASTER);
+
+ // Make sure our original token is still good
+ root.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), delegationToken1);
+
+ assertTrue("Could not get tables with delegation token", conn.tableOperations().list().size() > 0);
+
+ return null;
+ }
+ });
+
+ // Get a new token, so we can compare the keyId on the second to the first
+ final AuthenticationToken delegationToken2 = root.doAs(new PrivilegedExceptionAction<AuthenticationToken>() {
+ @Override
+ public AuthenticationToken run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ AuthenticationToken token = conn.securityOperations().getDelegationToken(new DelegationTokenConfig());
+
+ assertTrue("Could not get tables with delegation token", mac.getConnector(rootUser.getPrincipal(), token).tableOperations().list().size() > 0);
+
+ return token;
+ }
+ });
+
+ // A restarted master should reuse the same secret key after a restart if the secret key hasn't expired (1day by default)
+ DelegationTokenImpl dt1 = (DelegationTokenImpl) delegationToken1;
+ DelegationTokenImpl dt2 = (DelegationTokenImpl) delegationToken2;
+ assertEquals(dt1.getIdentifier().getKeyId(), dt2.getIdentifier().getKeyId());
+ }
+
+ @Test(expected = AccumuloException.class)
+ public void testDelegationTokenWithInvalidLifetime() throws Throwable {
+ // Login as the "root" user
+ UserGroupInformation root = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ // As the "root" user, open up the connection and get a delegation token
+ try {
+ root.doAs(new PrivilegedExceptionAction<AuthenticationToken>() {
+ @Override
+ public AuthenticationToken run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ // Should fail
+ return conn.securityOperations().getDelegationToken(new DelegationTokenConfig().setTokenLifetime(Long.MAX_VALUE, TimeUnit.MILLISECONDS));
+ }
+ });
+ } catch (UndeclaredThrowableException e) {
+ Throwable cause = e.getCause();
+ if (null != cause) {
+ throw cause;
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ @Test
+ public void testDelegationTokenWithReducedLifetime() throws Throwable {
+ // Login as the "root" user
+ UserGroupInformation root = UserGroupInformation.loginUserFromKeytabAndReturnUGI(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ // As the "root" user, open up the connection and get a delegation token
+ final AuthenticationToken dt = root.doAs(new PrivilegedExceptionAction<AuthenticationToken>() {
+ @Override
+ public AuthenticationToken run() throws Exception {
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ return conn.securityOperations().getDelegationToken(new DelegationTokenConfig().setTokenLifetime(5, TimeUnit.MINUTES));
+ }
+ });
+
+ AuthenticationTokenIdentifier identifier = ((DelegationTokenImpl) dt).getIdentifier();
+ assertTrue("Expected identifier to expire in no more than 5 minutes: " + identifier,
+ identifier.getExpirationDate() - identifier.getIssueDate() <= (5 * 60 * 1000));
+ }
+
+ /**
+ * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure that the system user exists (since the master does an RPC to
+ * the tserver which will create the system user if it doesn't already exist).
+ */
+ private void createTableWithDataAndCompact(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException {
+ final String table = testName.getMethodName() + "_table";
+ conn.tableOperations().create(table);
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", "d");
+ bw.addMutation(m);
+ bw.close();
+ conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true));
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
index cf55683,0000000..142a8bb
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/KerberosRenewalIT.java
@@@ -1,188 -1,0 +1,188 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.cluster.ClusterUser;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
+import org.apache.accumulo.core.client.security.tokens.KerberosToken;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.harness.AccumuloITBase;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
+import org.apache.accumulo.harness.MiniClusterHarness;
+import org.apache.accumulo.harness.TestingKdc;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+
+/**
+ * MAC test which uses {@link MiniKdc} to simulate ta secure environment. Can be used as a sanity check for Kerberos/SASL testing.
+ */
+public class KerberosRenewalIT extends AccumuloITBase {
+ private static final Logger log = LoggerFactory.getLogger(KerberosRenewalIT.class);
+
+ private static TestingKdc kdc;
+ private static String krbEnabledForITs = null;
+ private static ClusterUser rootUser;
+
+ private static final long TICKET_LIFETIME = 6 * 60 * 1000; // Anything less seems to fail when generating the ticket
+ private static final long TICKET_TEST_LIFETIME = 8 * 60 * 1000; // Run a test for 8 mins
+ private static final long TEST_DURATION = 9 * 60 * 1000; // The test should finish within 9 mins
+
+ @BeforeClass
+ public static void startKdc() throws Exception {
+ // 30s renewal time window
+ kdc = new TestingKdc(TestingKdc.computeKdcDir(), TestingKdc.computeKeytabDir(), TICKET_LIFETIME);
+ kdc.start();
+ krbEnabledForITs = System.getProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION);
+ if (null == krbEnabledForITs || !Boolean.parseBoolean(krbEnabledForITs)) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, "true");
+ }
+ rootUser = kdc.getRootUser();
+ }
+
+ @AfterClass
+ public static void stopKdc() throws Exception {
+ if (null != kdc) {
+ kdc.stop();
+ }
+ if (null != krbEnabledForITs) {
+ System.setProperty(MiniClusterHarness.USE_KERBEROS_FOR_IT_OPTION, krbEnabledForITs);
+ }
+ }
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return (int) TEST_DURATION / 1000;
+ }
+
+ private MiniAccumuloClusterImpl mac;
+
+ @Before
+ public void startMac() throws Exception {
+ MiniClusterHarness harness = new MiniClusterHarness();
+ mac = harness.create(this, new PasswordToken("unused"), kdc, new MiniClusterConfigurationCallback() {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
+ Map<String,String> site = cfg.getSiteConfig();
- site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "10s");
++ site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ // Reduce the period just to make sure we trigger renewal fast
+ site.put(Property.GENERAL_KERBEROS_RENEWAL_PERIOD.getKey(), "5s");
+ cfg.setSiteConfig(site);
+ }
+
+ });
+
+ mac.getConfig().setNumTservers(1);
+ mac.start();
+ // Enabled kerberos auth
+ Configuration conf = new Configuration(false);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ @After
+ public void stopMac() throws Exception {
+ if (null != mac) {
+ mac.stop();
+ }
+ }
+
+ // Intentially setting the Test annotation timeout. We do not want to scale the timeout.
+ @Test(timeout = TEST_DURATION)
+ public void testReadAndWriteThroughTicketLifetime() throws Exception {
+ // Attempt to use Accumulo for a duration of time that exceeds the Kerberos ticket lifetime.
+ // This is a functional test to verify that Accumulo services renew their ticket.
+ // If the test doesn't finish on its own, this signifies that Accumulo services failed
+ // and the test should fail. If Accumulo services renew their ticket, the test case
+ // should exit gracefully on its own.
+
+ // Login as the "root" user
+ UserGroupInformation.loginUserFromKeytab(rootUser.getPrincipal(), rootUser.getKeytab().getAbsolutePath());
+ log.info("Logged in as {}", rootUser.getPrincipal());
+
+ Connector conn = mac.getConnector(rootUser.getPrincipal(), new KerberosToken());
+ log.info("Created connector as {}", rootUser.getPrincipal());
+ assertEquals(rootUser.getPrincipal(), conn.whoami());
+
+ long duration = 0;
+ long last = System.currentTimeMillis();
+ // Make sure we have a couple renewals happen
+ while (duration < TICKET_TEST_LIFETIME) {
+ // Create a table, write a record, compact, read the record, drop the table.
+ createReadWriteDrop(conn);
+ // Wait a bit after
+ Thread.sleep(5000);
+
+ // Update the duration
+ long now = System.currentTimeMillis();
+ duration += now - last;
+ last = now;
+ }
+ }
+
+ /**
+ * Creates a table, adds a record to it, and then compacts the table. A simple way to make sure that the system user exists (since the master does an RPC to
+ * the tserver which will create the system user if it doesn't already exist).
+ */
+ private void createReadWriteDrop(Connector conn) throws TableNotFoundException, AccumuloSecurityException, AccumuloException, TableExistsException {
+ final String table = testName.getMethodName() + "_table";
+ conn.tableOperations().create(table);
+ BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
+ Mutation m = new Mutation("a");
+ m.put("b", "c", "d");
+ bw.addMutation(m);
+ bw.close();
+ conn.tableOperations().compact(table, new CompactionConfig().setFlush(true).setWait(true));
+ Scanner s = conn.createScanner(table, Authorizations.EMPTY);
+ Entry<Key,Value> entry = Iterables.getOnlyElement(s);
+ assertEquals("Did not find the expected key", 0, new Key("a", "b", "c").compareTo(entry.getKey(), PartialKey.ROW_COLFAM_COLQUAL));
+ assertEquals("d", entry.getValue().toString());
+ conn.tableOperations().delete(table);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/081eb1fa/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
index 3489c26,0000000..8ac67d9
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MasterFailoverIT.java
@@@ -1,80 -1,0 +1,80 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import java.util.Map;
+
+import org.apache.accumulo.cluster.ClusterControl;
+import org.apache.accumulo.core.cli.BatchWriterOpts;
+import org.apache.accumulo.core.cli.ScannerOpts;
+import org.apache.accumulo.core.client.ClientConfiguration;
+import org.apache.accumulo.core.client.ClientConfiguration.ClientProperty;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+public class MasterFailoverIT extends AccumuloClusterHarness {
+
+ @Override
+ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ Map<String,String> siteConfig = cfg.getSiteConfig();
- siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "5s");
++ siteConfig.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
+ cfg.setSiteConfig(siteConfig);
+ }
+
+ @Override
+ protected int defaultTimeoutSeconds() {
+ return 90;
+ }
+
+ @Test
+ public void test() throws Exception {
+ Connector c = getConnector();
+ String[] names = getUniqueNames(2);
+ c.tableOperations().create(names[0]);
+ TestIngest.Opts opts = new TestIngest.Opts();
+ opts.setTableName(names[0]);
+ ClientConfiguration clientConf = cluster.getClientConfig();
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ opts.updateKerberosCredentials(clientConf);
+ } else {
+ opts.setPrincipal(getAdminPrincipal());
+ }
+ TestIngest.ingest(c, opts, new BatchWriterOpts());
+
+ ClusterControl control = cluster.getClusterControl();
+ control.stopAllServers(ServerType.MASTER);
+ // start up a new one
+ control.startAllServers(ServerType.MASTER);
+ // talk to it
+ c.tableOperations().rename(names[0], names[1]);
+ VerifyIngest.Opts vopts = new VerifyIngest.Opts();
+ vopts.setTableName(names[1]);
+ if (clientConf.getBoolean(ClientProperty.INSTANCE_RPC_SASL_ENABLED.getKey(), false)) {
+ vopts.updateKerberosCredentials(clientConf);
+ } else {
+ vopts.setPrincipal(getAdminPrincipal());
+ }
+ VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
+ }
+}