You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by el...@apache.org on 2014/05/09 17:28:39 UTC
[01/50] [abbrv] git commit: Merge branch '1.6.0-SNAPSHOT'
Repository: accumulo
Updated Branches:
refs/heads/ACCUMULO-378 5fd07ec03 -> f2c0c16db
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/43a44ba8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/43a44ba8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/43a44ba8
Branch: refs/heads/ACCUMULO-378
Commit: 43a44ba82c2d9564cdb8ee5054f16398ebd2836a
Parents: 2881631 15e4f00
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed Apr 30 09:18:30 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Wed Apr 30 09:18:30 2014 -0400
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/43a44ba8/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index df7a5ac,135d15d..78af9c5
--- a/pom.xml
+++ b/pom.xml
@@@ -908,9 -896,12 +908,12 @@@
<artifactId>maven-pmd-plugin</artifactId>
<version>3.1</version>
<configuration>
+ <excludes>
+ <exclude>**/thrift/*.java</exclude>
+ </excludes>
<format>html</format>
<includeTests>true</includeTests>
- <targetJdk>1.6</targetJdk>
+ <targetJdk>1.7</targetJdk>
</configuration>
</plugin>
<plugin>
[16/50] [abbrv] git commit: ACCUMULO-378 Fix test failure on
permission propagation from jenkins
Posted by el...@apache.org.
ACCUMULO-378 Fix test failure on permission propagation from jenkins
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f4f7c948
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f4f7c948
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f4f7c948
Branch: refs/heads/ACCUMULO-378
Commit: f4f7c94840e288ec6ee59adb02d1d6c02d3af3ec
Parents: 05de21f
Author: Josh Elser <el...@apache.org>
Authored: Sat May 3 11:32:34 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sat May 3 11:32:34 2014 -0400
----------------------------------------------------------------------
.../accumulo/test/replication/ReplicationWithMakerTest.java | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f4f7c948/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
index 1c056ed..613a01b 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
@@ -32,6 +32,7 @@ import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
import org.apache.accumulo.core.replication.ReplicationTarget;
import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
@@ -282,13 +283,19 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
attempts = 5;
do {
if (!exists) {
- UtilWaitThread.sleep(200);
+ UtilWaitThread.sleep(500);
exists = conn.tableOperations().exists(ReplicationTable.NAME);
attempts--;
}
} while (!exists && attempts > 0);
Assert.assertTrue("Replication table did not exist", exists);
+ for (int i = 0; i < 5 && !conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ); i++) {
+ Thread.sleep(1000);
+ }
+
+ Assert.assertTrue(conn.securityOperations().hasTablePermission("root", ReplicationTable.NAME, TablePermission.READ));
+
boolean notFound = true;
Scanner s;
for (int i = 0; i < 10 && notFound; i++) {
[10/50] [abbrv] git commit: ACCUMULO-378 Adding in missing verify
calls
Posted by el...@apache.org.
ACCUMULO-378 Adding in missing verify calls
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/32479af6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/32479af6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/32479af6
Branch: refs/heads/ACCUMULO-378
Commit: 32479af61b01a7ba888d62e2dbf41394d46e356b
Parents: cc9a72a
Author: Josh Elser <el...@apache.org>
Authored: Thu May 1 21:20:33 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 1 21:20:33 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/util/ReplicationTableUtilTest.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/32479af6/server/base/src/test/java/org/apache/accumulo/server/util/ReplicationTableUtilTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/util/ReplicationTableUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/util/ReplicationTableUtilTest.java
index 2fa4814..d1a43c3 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/util/ReplicationTableUtilTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/util/ReplicationTableUtilTest.java
@@ -20,6 +20,7 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
import java.util.ArrayList;
import java.util.Collections;
@@ -87,6 +88,8 @@ public class ReplicationTableUtilTest {
ReplicationTableUtil.updateFiles(creds, new KeyExtent(new Text("1"), null, null), Collections.singleton(myFile), StatusUtil.newFile());
+ verify(writer);
+
Assert.assertEquals(1, mutations.size());
Mutation m = mutations.get(0);
@@ -141,5 +144,7 @@ public class ReplicationTableUtilTest {
replay(conn, tops);
ReplicationTableUtil.configureMetadataTable(conn, myMetadataTable);
+
+ verify(conn, tops);
}
}
[11/50] [abbrv] git commit: ACCUMULO-378 Stabilize the test some more.
Posted by el...@apache.org.
ACCUMULO-378 Stabilize the test some more.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/274c185a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/274c185a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/274c185a
Branch: refs/heads/ACCUMULO-378
Commit: 274c185a0f960ad818ebfd69e048aed15172da7c
Parents: 32479af
Author: Josh Elser <el...@apache.org>
Authored: Fri May 2 16:06:22 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri May 2 16:06:22 2014 -0400
----------------------------------------------------------------------
.../test/replication/ReplicationWithGCIT.java | 33 +++++++++++++++++++-
1 file changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/274c185a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
index 449827b..33da8a0 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
@@ -388,7 +388,9 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
Text expectedColqual = ReplicationTarget.toText(new ReplicationTarget("cluster1", "4"));
Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier());
notFound = false;
- } catch (NoSuchElementException e) {} catch (IllegalArgumentException e) {
+ } catch (NoSuchElementException e) {
+
+ } catch (IllegalArgumentException e) {
s = ReplicationTable.getScanner(conn);
for (Entry<Key,Value> content : s) {
log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
@@ -441,6 +443,34 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
for (@SuppressWarnings("unused")
Entry<Key,Value> entry : s) {}
+ // Wait for a bit since the GC has to run (should be running after a one second delay)
+ Thread.sleep(5000);
+
+ // Need to make sure we get the entries in metadata
+ boolean foundResults = false;
+ for (int i = 0; i < 5 && !foundResults; i++) {
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(ReplicationSection.getRange());
+ if (Iterables.size(s) > 0) {
+ foundResults = true;
+ }
+ Thread.sleep(1000);
+ }
+
+ Assert.assertTrue("Did not find any replication entries in the metadata table", foundResults);
+
+ // Then we need to get those records over to the replication table
+ foundResults = false;
+ for (int i = 0; i < 5 && !foundResults; i++) {
+ s = ReplicationTable.getScanner(conn);
+ if (Iterables.size(s) > 0) {
+ foundResults = true;
+ }
+ Thread.sleep(1000);
+ }
+
+ Assert.assertTrue("Did nto find any replication entries in the replication table", foundResults);
+
/**
* After recovery completes, we should have unreplicated, closed Status messages. The close happens at the beginning of log recovery.
*/
@@ -458,6 +488,7 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
builder.setBegin(Long.MAX_VALUE).setEnd(status.getEnd()).setClosed(status.getClosed()).setInfiniteEnd(status.getInfiniteEnd());
+ log.info("Writing update to replication to " + k);
Mutation m = new Mutation(k.getRow());
m.put(k.getColumnFamily(), k.getColumnQualifier(), ProtobufUtil.toValue(builder.build()));
bw.addMutation(m);
[45/50] [abbrv] git commit: ACCUMULO-378 Add some javadoc for the
general key structure of each table
Posted by el...@apache.org.
ACCUMULO-378 Add some javadoc for the general key structure of each table
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b18e0eeb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b18e0eeb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b18e0eeb
Branch: refs/heads/ACCUMULO-378
Commit: b18e0eeb7f379458454d169ab21608b65a1345db
Parents: 2372c71
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 16:14:30 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 16:14:30 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/metadata/schema/MetadataSchema.java | 2 ++
.../apache/accumulo/core/replication/ReplicationSchema.java | 7 +++++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b18e0eeb/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index 32772c1..e246e45 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -229,6 +229,8 @@ public class MetadataSchema {
/**
* Holds references to files that need replication
+ * <p>
+ * <code>~replhdfs://localhost:8020/accumulo/wal/tserver+port/WAL stat:local_table_id [] -> protobuf</code>
*/
public static class ReplicationSection {
public static final Text COLF = new Text("stat");
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b18e0eeb/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
index 1ea9ea4..57be449 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationSchema.java
@@ -31,6 +31,11 @@ import com.google.common.base.Preconditions;
*/
public class ReplicationSchema {
+ /**
+ * Portion of a file that must be replication to the given target: peer and some identifying location on that peer, e.g. remote table ID
+ * <p>
+ * <code>hdfs://localhost:8020/accumulo/wal/tserver+port/WAL work:serialized_ReplicationTarget [] -> Protobuf</code>
+ */
public static class WorkSection {
public static final Text NAME = new Text("work");
private static final ByteSequence BYTE_SEQ_NAME = new ArrayByteSequence("work");
@@ -69,6 +74,8 @@ public class ReplicationSchema {
/**
* Holds replication markers tracking status for files
+ * <p>
+ * <code>hdfs://localhost:8020/accumulo/wal/tserver+port/WAL repl:local_table_id [] -> Protobuf</code>
*/
public static class StatusSection {
public static final Text NAME = new Text("repl");
[06/50] [abbrv] git commit: ACCUMULO-378 Fix up some tests that
failed on jenkins and remove now unnecessary test
Posted by el...@apache.org.
ACCUMULO-378 Fix up some tests that failed on jenkins and remove now unnecessary test
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f247c8e2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f247c8e2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f247c8e2
Branch: refs/heads/ACCUMULO-378
Commit: f247c8e251abb99765ed28b98873f22152d52f58
Parents: 91396e5
Author: Josh Elser <el...@apache.org>
Authored: Wed Apr 30 17:30:44 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed Apr 30 17:30:44 2014 -0400
----------------------------------------------------------------------
.../replication/ReplicationDeadlockTest.java | 3 -
.../ReplicationTableTimestampIT.java | 245 -------------------
.../test/replication/ReplicationWithGCIT.java | 2 +-
.../replication/ReplicationWithMakerTest.java | 40 ++-
4 files changed, 32 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f247c8e2/test/src/test/java/org/apache/accumulo/test/replication/ReplicationDeadlockTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationDeadlockTest.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationDeadlockTest.java
index 9713c8c..d43aa32 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationDeadlockTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationDeadlockTest.java
@@ -17,7 +17,6 @@
package org.apache.accumulo.test.replication;
import java.util.Arrays;
-import java.util.Collection;
import java.util.HashSet;
import java.util.Map.Entry;
import java.util.Set;
@@ -36,9 +35,7 @@ import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.Lo
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.minicluster.impl.ProcessReference;
import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f247c8e2/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTableTimestampIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTableTimestampIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTableTimestampIT.java
deleted file mode 100644
index 3111164..0000000
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationTableTimestampIT.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.test.replication;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.accumulo.core.client.BatchWriter;
-import org.apache.accumulo.core.client.BatchWriterConfig;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Scanner;
-import org.apache.accumulo.core.client.TableNotFoundException;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.Mutation;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.LogColumnFamily;
-import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
-import org.apache.accumulo.core.replication.proto.Replication.Status;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.tabletserver.log.LogEntry;
-import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
-import org.apache.accumulo.server.replication.ReplicationTable;
-import org.apache.accumulo.test.functional.ConfigurableMacIT;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RawLocalFileSystem;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.LinkedListMultimap;
-import com.google.common.collect.Multimap;
-
-/**
- * Integration Tests that attempt to evaluate the accuracy of the internal bookkeeping performed on the accumulo "master" instance. Does not send data to any
- * remote instance, merely tracks what is stored locally.
- */
-public class ReplicationTableTimestampIT extends ConfigurableMacIT {
- @Override
- public int defaultTimeoutSeconds() {
- return 300;
- }
-
- @Override
- public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
- cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "0s");
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1M");
- cfg.setProperty(Property.GC_CYCLE_START, "1s");
- cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
- cfg.setNumTservers(1);
- hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
- }
-
- private Multimap<String,String> getLogs(Connector conn) throws TableNotFoundException {
- Multimap<String,String> logs = HashMultimap.create();
- Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- scanner.fetchColumnFamily(LogColumnFamily.NAME);
- scanner.setRange(new Range());
- for (Entry<Key,Value> entry : scanner) {
- if (Thread.interrupted()) {
- return logs;
- }
-
- LogEntry logEntry = LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
-
- for (String log : logEntry.logSet) {
- logs.put(new Path(log).toString(), logEntry.extent.getTableId().toString());
- }
- }
- return logs;
- }
-
- @Test
- public void closedReplicationStatusStayClosed() throws Exception {
- final Connector conn = getConnector();
- String table1 = "table1", table2 = "table2", table3 = "table3";
- final Multimap<String,String> metadataTableWals = HashMultimap.create();
- final AtomicBoolean keepRunning = new AtomicBoolean(true);
-
- Thread t = new Thread(new Runnable() {
- @Override
- public void run() {
- // Should really be able to interrupt here, but the Scanner throws a fit to the logger
- // when that happens
- while (keepRunning.get()) {
- try {
- metadataTableWals.putAll(getLogs(conn));
- } catch (TableNotFoundException e) {
- log.error("Metadata table doesn't exist");
- }
- }
- }
-
- });
-
- t.start();
-
- conn.tableOperations().create(table1);
- conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
- conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGETS.getKey() + "cluster1", "1");
- Thread.sleep(1000);
-
- // Write some data to table1
- BatchWriter bw = conn.createBatchWriter(table1, new BatchWriterConfig());
- for (int rows = 0; rows < 200; rows++) {
- Mutation m = new Mutation(Integer.toString(rows));
- for (int cols = 0; cols < 500; cols++) {
- String value = Integer.toString(cols);
- m.put(value, "", value);
- }
- bw.addMutation(m);
- }
-
- bw.close();
-
- conn.tableOperations().create(table2);
- conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION.getKey(), "true");
- conn.tableOperations().setProperty(table2, Property.TABLE_REPLICATION_TARGETS.getKey() + "cluster1", "1");
- Thread.sleep(1000);
-
- // Write some data to table2
- bw = conn.createBatchWriter(table2, new BatchWriterConfig());
- for (int rows = 0; rows < 200; rows++) {
- Mutation m = new Mutation(Integer.toString(rows));
- for (int cols = 0; cols < 500; cols++) {
- String value = Integer.toString(cols);
- m.put(value, "", value);
- }
- bw.addMutation(m);
- }
-
- bw.close();
-
- conn.tableOperations().create(table3);
- conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION.getKey(), "true");
- conn.tableOperations().setProperty(table3, Property.TABLE_REPLICATION_TARGETS.getKey() + "cluster1", "1");
- Thread.sleep(1000);
-
- // Write some data to table3
- bw = conn.createBatchWriter(table3, new BatchWriterConfig());
- for (int rows = 0; rows < 200; rows++) {
- Mutation m = new Mutation(Integer.toString(rows));
- for (int cols = 0; cols < 500; cols++) {
- String value = Integer.toString(cols);
- m.put(value, "", value);
- }
- bw.addMutation(m);
- }
-
- bw.close();
-
- // Stop the thread which is constantly scanning metadata to track all WALs seen
- keepRunning.set(false);
- t.join(5000);
-
- // See which files we have for replication in the replication table
- Scanner s = ReplicationTable.getScanner(conn);
- StatusSection.limit(s);
- Set<String> replFiles = new HashSet<>();
- for (Entry<Key,Value> entry : s) {
- replFiles.add(entry.getKey().getRow().toString());
- }
-
- // We might have a WAL that was used solely for the replication table
- // We want to remove that from our list as it should not appear in the replication table
- String replicationTableId = conn.tableOperations().tableIdMap().get(ReplicationTable.NAME);
- Iterator<Entry<String,String>> observedLogs = metadataTableWals.entries().iterator();
- while (observedLogs.hasNext()) {
- Entry<String,String> observedLog = observedLogs.next();
- if (replicationTableId.equals(observedLog.getValue())) {
- observedLogs.remove();
- }
- }
-
- // We should have *some* reference to each log that was seen in the metadata table
- // They might not yet all be closed though (might be newfile)
- Assert.assertEquals("Metadata log distribution: " + metadataTableWals, metadataTableWals.keySet(), replFiles);
-
- LinkedListMultimap<String,Entry<Key,Value>> kvByRow = LinkedListMultimap.create();
- s = ReplicationTable.getScanner(conn);
- StatusSection.limit(s);
- for (Entry<Key,Value> entry : s) {
- kvByRow.put(entry.getKey().getRow().toString(), entry);
- }
-
- for (String row : kvByRow.keySet()) {
- ArrayList<Entry<Key,Value>> kvs = new ArrayList<>(kvByRow.get(row));
- Collections.sort(kvs, new Comparator<Entry<Key,Value>>() {
- @Override
- public int compare(Entry<Key,Value> o1, Entry<Key,Value> o2) {
- return (new Long(o1.getKey().getTimestamp())).compareTo(new Long(o2.getKey().getTimestamp()));
- }
- });
-
- Key closedKey = null;
- boolean observedClosed = false;
- for (Entry<Key,Value> kv : kvs) {
- Status status = Status.parseFrom(kv.getValue().get());
-
- // Once we get a closed record, every subsequent record should *also* be closed
- // A file cannot be "re-opened"
- if (!observedClosed) {
- if (status.getClosed()) {
- closedKey = kv.getKey();
- observedClosed = true;
- }
- } else {
- Assert.assertTrue("Found a non-closed Status (" + kv.getKey().toStringNoTruncate() + ") after a closed Status (" + closedKey.toStringNoTruncate() + ") was observed", status.getClosed());
- }
- }
-
- }
-
- for (String replFile : replFiles) {
- Path p = new Path(replFile);
- FileSystem fs = p.getFileSystem(new Configuration());
- Assert.assertTrue("File does not exist anymore, it was likely incorrectly garbage collected: " + p, fs.exists(p));
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f247c8e2/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
index 3f6b8dc..449827b 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
@@ -496,7 +496,7 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
s.setRange(ReplicationSection.getRange());
recordsFound = 0;
- for (@SuppressWarnings("unused") Entry<Key,Value> entry : s) {
+ for (Entry<Key,Value> entry : s) {
recordsFound++;
log.info(entry.getKey().toStringNoTruncate() + " " + Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/f247c8e2/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
index 1ffb2a2..1c056ed 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
@@ -33,7 +33,9 @@ import org.apache.accumulo.core.replication.ReplicationTarget;
import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.minicluster.impl.ProcessReference;
import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.hadoop.conf.Configuration;
@@ -58,12 +60,20 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1M");
+ // Run the process in the master which writes replication records from metadata to replication
+ // repeatedly without pause
cfg.setProperty(Property.MASTER_REPLICATION_SCAN_INTERVAL, "0");
cfg.setNumTservers(1);
}
@Test
public void singleTableSingleTarget() throws Exception {
+ // We want to kill the GC so it doesn't come along and close Status records and mess up the comparisons
+ // against expected Status messages.
+ for (ProcessReference proc : cluster.getProcesses().get(ServerType.GARBAGE_COLLECTOR)) {
+ cluster.killProcess(ServerType.GARBAGE_COLLECTOR, proc);
+ }
+
Connector conn = getConnector();
String table1 = "table1";
@@ -80,7 +90,7 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION.getKey(), "true");
// Replicate table1 to cluster1 in the table with id of '4'
conn.tableOperations().setProperty(table1, Property.TABLE_REPLICATION_TARGETS.getKey() + "cluster1", "4");
- attempts = 0;
+ break;
} catch (Exception e) {
attempts--;
if (attempts <= 0) {
@@ -113,26 +123,39 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
attempts--;
}
} while (!exists && attempts > 0);
- Assert.assertTrue("Replication table did not exist", exists);
+ Assert.assertTrue("Replication table was never created", exists);
// ACCUMULO-2743 The Observer in the tserver has to be made aware of the change to get the combiner (made by the master)
for (int i = 0; i < 5 && !conn.tableOperations().listIterators(ReplicationTable.NAME).keySet().contains(ReplicationTable.COMBINER_NAME); i++) {
UtilWaitThread.sleep(1000);
}
- Assert.assertTrue("Did not find expected combiner", conn.tableOperations().listIterators(ReplicationTable.NAME).keySet().contains(ReplicationTable.COMBINER_NAME));
+ Assert.assertTrue("Combiner was never set on replication table",
+ conn.tableOperations().listIterators(ReplicationTable.NAME).keySet().contains(ReplicationTable.COMBINER_NAME));
+
+ // Trigger the minor compaction, waiting for it to finish.
+ // This should write the entry to metadata that the file has data
+ conn.tableOperations().flush(ReplicationTable.NAME, null, null, true);
// Make sure that we have one status element, should be a new file
Scanner s = ReplicationTable.getScanner(conn);
StatusSection.limit(s);
Entry<Key,Value> entry = null;
attempts = 5;
+ // This record will move from new to new with infinite length because of the minc (flush)
+ Status expectedStatus = StatusUtil.openWithUnknownLength();
while (null == entry && attempts > 0) {
- try{
+ try {
entry = Iterables.getOnlyElement(s);
+ if (!expectedStatus.equals(Status.parseFrom(entry.getValue().get()))) {
+ entry = null;
+ // the master process didn't yet fire and write the new mutation, wait for it to do
+ // so and try to read it again
+ Thread.sleep(1000);
+ }
} catch (NoSuchElementException e) {
entry = null;
- Thread.sleep(200);
+ Thread.sleep(500);
} catch (IllegalArgumentException e) {
// saw this contain 2 elements once
s = ReplicationTable.getScanner(conn);
@@ -146,8 +169,8 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
}
}
- Assert.assertNotNull(entry);
- Assert.assertEquals(StatusUtil.openWithUnknownLength(), Status.parseFrom(entry.getValue().get()));
+ Assert.assertNotNull("Could not find expected entry in replication table", entry);
+ Assert.assertEquals("Expected to find a replication entry that is open with infinite length", expectedStatus, Status.parseFrom(entry.getValue().get()));
// Try a couple of times to watch for the work record to be created
boolean notFound = true;
@@ -276,8 +299,7 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
Text expectedColqual = ReplicationTarget.toText(new ReplicationTarget("cluster1", "4"));
Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier());
notFound = false;
- } catch (NoSuchElementException e) {
- } catch (IllegalArgumentException e) {
+ } catch (NoSuchElementException e) {} catch (IllegalArgumentException e) {
s = ReplicationTable.getScanner(conn);
for (Entry<Key,Value> content : s) {
log.info(content.getKey().toStringNoTruncate() + " => " + content.getValue());
[26/50] [abbrv] git commit: ACCUMULO-2212 Add ZooReaderWriterFactory
Posted by el...@apache.org.
ACCUMULO-2212 Add ZooReaderWriterFactory
The o.a.a.server.zookeeper.ZooReaderWriter class is mostly copied into a new
ZooReaderWriterFactory class. Users of the factory can work directly with FATE
ZooReaderWriter instances, instead of those of the server type, which is no longer
needed. The logic for building instances from site configuration data is in the
factory.
As part of this change, the invocation handler used to retry ZK calls on connection
loss is refactored into a new RetryingInvocationHandler.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/484491d2
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/484491d2
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/484491d2
Branch: refs/heads/ACCUMULO-378
Commit: 484491d21abf84240912209ae16e33639cbfa595
Parents: cdcc1d4
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed Apr 9 16:02:41 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon May 5 10:40:06 2014 -0400
----------------------------------------------------------------------
.../zookeeper/RetryingInvocationHandler.java | 63 ++++++++++++++
.../fate/zookeeper/ZooReaderWriter.java | 24 +-----
.../RetryingInvocationHandlerTest.java | 87 ++++++++++++++++++++
.../zookeeper/ZooReaderWriterFactory.java | 86 +++++++++++++++++++
4 files changed, 238 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484491d2/fate/src/main/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandler.java
----------------------------------------------------------------------
diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandler.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandler.java
new file mode 100644
index 0000000..4597036
--- /dev/null
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandler.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.fate.zookeeper;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import org.apache.accumulo.fate.util.UtilWaitThread;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * An invocation handler for ZooKeeper reader/writers that retries calls that fail due to connection loss.
+ */
+public class RetryingInvocationHandler implements InvocationHandler {
+ private final IZooReaderWriter zrw;
+
+ /**
+ * Creates a new invocation handler.
+ *
+ * @param zrw
+ * ZooKeeper reader/writer being handled
+ */
+ public RetryingInvocationHandler(IZooReaderWriter zrw) {
+ this.zrw = zrw;
+ }
+
+ private static final long INITIAL_RETRY_TIME = 250L;
+ private static final long RETRY_INCREMENT = 250L;
+ private static final long MAXIMUM_RETRY_TIME = 5000L;
+
+ @Override
+ public Object invoke(Object obj, Method method, Object[] args) throws Throwable {
+ long retryTime = INITIAL_RETRY_TIME;
+ while (true) {
+ try {
+ return method.invoke(zrw, args);
+ } catch (InvocationTargetException e) {
+ if (e.getCause() instanceof KeeperException.ConnectionLossException) {
+ Logger.getLogger(ZooReaderWriter.class).warn("Error connecting to zookeeper, will retry in " + String.format("%.2f secs", retryTime / 1000.0), e.getCause());
+ UtilWaitThread.sleep(retryTime);
+ retryTime = Math.min(MAXIMUM_RETRY_TIME, retryTime + RETRY_INCREMENT);
+ } else {
+ throw e.getCause();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484491d2/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
----------------------------------------------------------------------
diff --git a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
index 24c1d85..16f7833 100644
--- a/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
+++ b/fate/src/main/java/org/apache/accumulo/fate/zookeeper/ZooReaderWriter.java
@@ -154,28 +154,8 @@ public class ZooReaderWriter extends ZooReader implements IZooReaderWriter {
public static synchronized IZooReaderWriter getRetryingInstance(String zookeepers, int timeInMillis, String scheme, byte[] auth) {
if (retryingInstance == null) {
- final IZooReaderWriter inst = getInstance(zookeepers, timeInMillis, scheme, auth);
-
- InvocationHandler ih = new InvocationHandler() {
- @Override
- public Object invoke(Object obj, Method method, Object[] args) throws Throwable {
- long retryTime = 250;
- while (true) {
- try {
- return method.invoke(inst, args);
- } catch (InvocationTargetException e) {
- if (e.getCause() instanceof KeeperException.ConnectionLossException) {
- Logger.getLogger(ZooReaderWriter.class).warn("Error connecting to zookeeper, will retry in " + String.format("%.2f secs", retryTime / 1000.0), e.getCause());
- UtilWaitThread.sleep(retryTime);
- retryTime = Math.min(5000, retryTime + 250);
- } else {
- throw e.getCause();
- }
- }
- }
- }
- };
-
+ IZooReaderWriter inst = getInstance(zookeepers, timeInMillis, scheme, auth);
+ InvocationHandler ih = new RetryingInvocationHandler(inst);
retryingInstance = (IZooReaderWriter) Proxy.newProxyInstance(ZooReaderWriter.class.getClassLoader(), new Class[] {IZooReaderWriter.class}, ih);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484491d2/fate/src/test/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandlerTest.java
----------------------------------------------------------------------
diff --git a/fate/src/test/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandlerTest.java b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandlerTest.java
new file mode 100644
index 0000000..0613a1f
--- /dev/null
+++ b/fate/src/test/java/org/apache/accumulo/fate/zookeeper/RetryingInvocationHandlerTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.fate.zookeeper;
+
+import java.lang.reflect.Method;
+import org.apache.zookeeper.KeeperException.ConnectionLossException;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.easymock.EasyMock.aryEq;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.eq;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+
+public class RetryingInvocationHandlerTest {
+ private static final String PATH = "/path/to/somewhere";
+ private static final byte[] DATA = {(byte) 1, (byte) 2};
+ private static final Object[] ARGS = {PATH, DATA};
+ private static final String RV = "OK";
+
+ private static Method putMethod;
+
+ @BeforeClass
+ public static void setUpClass() throws Exception {
+ putMethod = IZooReaderWriter.class.getMethod("putEphemeralData", String.class, byte[].class);
+ }
+
+ private IZooReaderWriter zrw;
+ private RetryingInvocationHandler ih;
+
+ @Before
+ public void setUp() throws Exception {
+ zrw = createMock(IZooReaderWriter.class);
+ ih = new RetryingInvocationHandler(zrw);
+ }
+
+ @Test
+ public void testInvokeSuccessful() throws Throwable {
+ expect(zrw.putEphemeralData(eq(PATH), aryEq(DATA))).andReturn(RV);
+ replay(zrw);
+ Object rv = ih.invoke(null, putMethod, ARGS);
+ verify(zrw);
+ assertEquals(RV, rv);
+ }
+
+ @Test
+ public void testInvokeRetrySuccessful() throws Throwable {
+ ConnectionLossException e = createMock(ConnectionLossException.class);
+ expect(zrw.putEphemeralData(eq(PATH), aryEq(DATA))).andThrow(e);
+ expect(zrw.putEphemeralData(eq(PATH), aryEq(DATA))).andThrow(e);
+ expect(zrw.putEphemeralData(eq(PATH), aryEq(DATA))).andReturn(RV);
+ replay(zrw);
+ Object rv = ih.invoke(null, putMethod, ARGS);
+ verify(zrw);
+ assertEquals(RV, rv);
+ }
+
+ @Test(expected = InterruptedException.class)
+ public void testInvokeRetryFailure() throws Throwable {
+ ConnectionLossException e = createMock(ConnectionLossException.class);
+ expect(zrw.putEphemeralData(eq(PATH), aryEq(DATA))).andThrow(e);
+ expect(zrw.putEphemeralData(eq(PATH), aryEq(DATA))).andThrow(new InterruptedException());
+ replay(zrw);
+ try {
+ ih.invoke(null, putMethod, ARGS);
+ } finally {
+ verify(zrw);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/484491d2/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
new file mode 100644
index 0000000..3c8d143
--- /dev/null
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.zookeeper;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Proxy;
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.fate.util.UtilWaitThread;
+import org.apache.accumulo.fate.zookeeper.IZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.ZooReaderWriter;
+import org.apache.accumulo.fate.zookeeper.RetryingInvocationHandler;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+
+/**
+ * A factory for {@link ZooReaderWriter} objects.
+ */
+public class ZooReaderWriterFactory {
+ private static final String SCHEME = "digest";
+ private static final String USER = "accumulo";
+ private static IZooReaderWriter instance = null;
+ private static IZooReaderWriter retryingInstance = null;
+
+ /**
+ * Gets a new reader/writer.
+ *
+ * @param string
+ * ZooKeeper connection string
+ * @param timeInMillis
+ * session timeout in milliseconds
+ * @param secret
+ * instance secret
+ * @return reader/writer
+ */
+ public IZooReaderWriter getZooReaderWriter(String string, int timeInMillis, String secret) {
+ return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
+ }
+
+ /**
+ * Gets a reader/writer, retrieving ZooKeeper information from the site configuration. The same instance may be returned for multiple calls.
+ *
+ * @return reader/writer
+ */
+ public IZooReaderWriter getInstance() {
+ synchronized (ZooReaderWriterFactory.class) {
+ if (instance == null) {
+ AccumuloConfiguration conf = ServerConfiguration.getSiteConfiguration();
+ instance = getZooReaderWriter(conf.get(Property.INSTANCE_ZK_HOST), (int) conf.getTimeInMillis(Property.INSTANCE_ZK_TIMEOUT),
+ conf.get(Property.INSTANCE_SECRET));
+ }
+ return instance;
+ }
+ }
+
+ /**
+ * Gets a reader/writer, retrieving ZooKeeper information from the site configuration, and that retries on connection loss. The same instance may be returned
+ * for multiple calls.
+ *
+ * @return retrying reader/writer
+ */
+ public IZooReaderWriter getRetryingInstance() {
+ synchronized (ZooReaderWriterFactory.class) {
+ if (retryingInstance == null) {
+ IZooReaderWriter inst = getInstance();
+ InvocationHandler ih = new RetryingInvocationHandler(inst);
+ retryingInstance = (IZooReaderWriter) Proxy.newProxyInstance(IZooReaderWriter.class.getClassLoader(), new Class[] {IZooReaderWriter.class}, ih);
+ }
+ return retryingInstance;
+ }
+ }
+}
[09/50] [abbrv] git commit: ACCUMULO-2581 Add tests for work
assignment using the distributed work queue
Posted by el...@apache.org.
ACCUMULO-2581 Add tests for work assignment using the distributed work queue
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cc9a72ac
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cc9a72ac
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cc9a72ac
Branch: refs/heads/ACCUMULO-378
Commit: cc9a72ac736c50fa798a821fcd603fc033994eff
Parents: 31f4e83
Author: Josh Elser <el...@apache.org>
Authored: Thu May 1 21:16:08 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 1 21:16:08 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/replication/StatusUtil.java | 2 +-
.../server/zookeeper/DistributedWorkQueue.java | 12 +++
.../replication/ReplicationWorkAssigner.java | 17 ++--
.../ReplicationWorkAssignerTest.java | 97 ++++++++++++++++++--
4 files changed, 112 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc9a72ac/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
index 74327fd..842d945 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
@@ -200,7 +200,7 @@ public class StatusUtil {
*/
public static boolean isWorkRequired(Status status) {
if (status.getInfiniteEnd()) {
- return Long.MAX_VALUE == status.getBegin();
+ return Long.MAX_VALUE != status.getBegin();
} else {
return status.getBegin() < status.getEnd();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc9a72ac/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
index 4738c2b..b1907fb 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/DistributedWorkQueue.java
@@ -16,6 +16,7 @@
*/
package org.apache.accumulo.server.zookeeper;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -205,6 +206,17 @@ public class DistributedWorkQueue {
}
}, r.nextInt(60 * 1000), 60 * 1000);
}
+
+ /**
+ * Adds work to the queue, automatically converting the String to bytes using {@link StandardCharsets#UTF_8}
+ * @param workId
+ * @param data
+ * @throws KeeperException
+ * @throws InterruptedException
+ */
+ public void addWork(String workId, String data) throws KeeperException, InterruptedException {
+ addWork(workId, data.getBytes(StandardCharsets.UTF_8));
+ }
public void addWork(String workId, byte[] data) throws KeeperException, InterruptedException {
if (workId.equalsIgnoreCase(LOCKS_NODE))
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc9a72ac/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
index f3a8825..370fcfd 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -215,11 +215,12 @@ public class ReplicationWorkAssigner implements Runnable {
if (StatusUtil.isWorkRequired(status)) {
Path p = new Path(file);
String filename = p.getName();
+ entry.getKey().getColumnQualifier(buffer);
+ String key = filename + "|" + buffer;
// And, we haven't already queued this file up for work already
- if (!queuedWork.contains(filename)) {
- entry.getKey().getColumnQualifier(buffer);
- queueWork(file, filename, buffer);
+ if (!queuedWork.contains(key)) {
+ queueWork(key, file);
}
}
}
@@ -238,10 +239,9 @@ public class ReplicationWorkAssigner implements Runnable {
* @param filename
* Filename
*/
- protected void queueWork(String path, String filename, Text serializedTarget) {
+ protected void queueWork(String key, String path) {
try {
- String key = filename + "|" + serializedTarget;
- workQueue.addWork(key, path.getBytes(StandardCharsets.UTF_8));
+ workQueue.addWork(key, path);
queuedWork.add(key);
} catch (KeeperException | InterruptedException e) {
log.warn("Could not queue work for {}", path, e);
@@ -252,11 +252,12 @@ public class ReplicationWorkAssigner implements Runnable {
* Iterate over the queued work to remove entries that have been completed.
*/
protected void cleanupFinishedWork() {
- Iterator<String> work = queuedWork.iterator();
+ final Iterator<String> work = queuedWork.iterator();
+ final String instanceId = master.getInstance().getInstanceID();
while (work.hasNext()) {
String filename = work.next();
// Null equates to the work was finished
- if (null == zooCache.get(ZooUtil.getRoot(master.getInstance()) + Constants.ZREPLICATION + "/" + filename)) {
+ if (null == zooCache.get(ZooUtil.getRoot(instanceId) + Constants.ZREPLICATION + "/" + filename)) {
work.remove();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cc9a72ac/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
index b596126..89a1ddd 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
@@ -20,16 +20,20 @@ import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Mutation;
@@ -41,6 +45,7 @@ import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.ZooCache;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.Assert;
@@ -79,13 +84,14 @@ public class ReplicationWorkAssignerTest {
assigner.setWorkQueue(workQueue);
Path p = new Path("/accumulo/wal/tserver+port/" + UUID.randomUUID());
- assigner.queueWork(p.toString(), p.getName(), serializedTarget);
- workQueue.addWork(p.getName(), p.toString().getBytes(StandardCharsets.UTF_8));
+ workQueue.addWork(p.getName() + "|" + serializedTarget.toString(), p.toString());
expectLastCall().once();
replay(workQueue);
+ assigner.queueWork(p.getName() + "|" + serializedTarget, p.toString());
+
Assert.assertEquals(1, queuedWork.size());
Assert.assertEquals(p.getName() + "|" + serializedTarget, queuedWork.iterator().next());
}
@@ -102,6 +108,8 @@ public class ReplicationWorkAssignerTest {
assigner.setWorkQueue(workQueue);
assigner.initializeQueuedWork();
+ verify(workQueue);
+
Set<String> queuedWork = assigner.getQueuedWork();
Assert.assertEquals("Expected existing work and queued work to be the same size", existingWork.size(), queuedWork.size());
Assert.assertTrue("Expected all existing work to be queued", queuedWork.containsAll(existingWork));
@@ -141,28 +149,34 @@ public class ReplicationWorkAssignerTest {
@SuppressWarnings("unchecked")
HashSet<String> queuedWork = createMock(HashSet.class);
assigner.setQueuedWork(queuedWork);
+ assigner.setWorkQueue(workQueue);
+ assigner.setMaxQueueSize(Integer.MAX_VALUE);
expect(queuedWork.size()).andReturn(0).anyTimes();
// Make sure we expect the invocations in the correct order (accumulo is sorted)
if (file1.compareTo(file2) <= 0) {
String key = filename1 + "|" + serializedTarget1;
- workQueue.addWork(key, file1.getBytes(StandardCharsets.UTF_8));
+ expect(queuedWork.contains(key)).andReturn(false);
+ workQueue.addWork(key, file1);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
key = filename2 + "|" + serializedTarget2;
- workQueue.addWork(key, file2.getBytes(StandardCharsets.UTF_8));
+ expect(queuedWork.contains(key)).andReturn(false);
+ workQueue.addWork(key, file2);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
} else {
String key = filename2 + "|" + serializedTarget2;
- workQueue.addWork(key, file2.getBytes(StandardCharsets.UTF_8));
+ expect(queuedWork.contains(key)).andReturn(false);
+ workQueue.addWork(key, file2);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
key = filename1 + "|" + serializedTarget1;
- workQueue.addWork(key, file1.getBytes(StandardCharsets.UTF_8));
+ expect(queuedWork.contains(key)).andReturn(false);
+ workQueue.addWork(key, file1);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
}
@@ -170,6 +184,8 @@ public class ReplicationWorkAssignerTest {
replay(queuedWork, workQueue);
assigner.createWork();
+
+ verify(queuedWork, workQueue);
}
@Test
@@ -192,6 +208,7 @@ public class ReplicationWorkAssignerTest {
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString();
String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
+
Mutation m = new Mutation(file1);
WorkSection.add(m, serializedTarget1, StatusUtil.newFileValue());
bw.addMutation(m);
@@ -206,12 +223,78 @@ public class ReplicationWorkAssignerTest {
@SuppressWarnings("unchecked")
HashSet<String> queuedWork = createMock(HashSet.class);
assigner.setQueuedWork(queuedWork);
+ assigner.setMaxQueueSize(Integer.MAX_VALUE);
- expect(queuedWork.size()).andReturn(0).anyTimes();
+ expect(queuedWork.size()).andReturn(0).times(2);
replay(queuedWork, workQueue);
assigner.createWork();
+
+ verify(queuedWork, workQueue);
+ }
+
+ @Test
+ public void workNotInZooKeeperIsCleanedUp() {
+ Set<String> queuedWork = new LinkedHashSet<>(Arrays.asList("wal1", "wal2"));
+ assigner.setQueuedWork(queuedWork);
+
+ Instance inst = createMock(Instance.class);
+ ZooCache cache = createMock(ZooCache.class);
+ assigner.setZooCache(cache);
+
+ expect(master.getInstance()).andReturn(inst);
+ expect(inst.getInstanceID()).andReturn("id");
+ expect(cache.get(Constants.ZROOT + "/id" + Constants.ZREPLICATION + "/wal1")).andReturn(null);
+ expect(cache.get(Constants.ZROOT + "/id" + Constants.ZREPLICATION + "/wal2")).andReturn(null);
+
+ replay(cache, inst, master);
+
+ assigner.cleanupFinishedWork();
+
+ verify(cache, inst, master);
+ Assert.assertTrue("Queued work was not emptied", queuedWork.isEmpty());
}
+ @Test
+ public void workNotReAdded() throws Exception {
+ Set<String> queuedWork = new HashSet<>();
+
+ assigner.setQueuedWork(queuedWork);
+
+ ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
+ Text serializedTarget = ReplicationTarget.toText(target);
+
+ queuedWork.add("wal1|" + serializedTarget.toString());
+
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Credentials creds = new Credentials("root", new PasswordToken(""));
+ Connector conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+
+ // Set the connector
+ assigner.setConnector(conn);
+
+ // Create and grant ourselves write to the replication table
+ ReplicationTable.create(conn);
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE);
+
+ // Create two mutations, both of which need replication work done
+ BatchWriter bw = ReplicationTable.getBatchWriter(conn);
+ String file1 = "/accumulo/wal/tserver+port/wal1";
+ Mutation m = new Mutation(file1);
+ WorkSection.add(m, serializedTarget, StatusUtil.openWithUnknownLengthValue());
+ bw.addMutation(m);
+
+ bw.close();
+
+ DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
+ assigner.setWorkQueue(workQueue);
+ assigner.setMaxQueueSize(Integer.MAX_VALUE);
+
+ replay(workQueue);
+
+ assigner.createWork();
+
+ verify(workQueue);
+ }
}
[50/50] [abbrv] git commit: Merge remote-tracking branch
'origin/master' into ACCUMULO-378
Posted by el...@apache.org.
Merge remote-tracking branch 'origin/master' into ACCUMULO-378
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/f2c0c16d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/f2c0c16d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/f2c0c16d
Branch: refs/heads/ACCUMULO-378
Commit: f2c0c16db3f37d630f0616aabe3a18f7deee64a5
Parents: e117a78 900d6ab
Author: Josh Elser <el...@apache.org>
Authored: Fri May 9 11:26:17 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri May 9 11:26:17 2014 -0400
----------------------------------------------------------------------
README | 2 +-
assemble/pom.xml | 2 +-
assemble/src/main/assemblies/component.xml | 2 +-
.../accumulo/core/conf/ConfigurationDocGen.java | 72 +-
.../core/conf/DefaultConfiguration.java | 4 +-
.../accumulo/core/conf/config-header.html | 2 -
.../apache/accumulo/core/conf/config-header.tex | 99 ---
.../apache/accumulo/core/conf/config-header.txt | 93 +++
docs/pom.xml | 67 +-
.../main/asciidoc/accumulo_user_manual.asciidoc | 62 ++
.../main/asciidoc/chapters/administration.txt | 396 +++++++++
docs/src/main/asciidoc/chapters/analytics.txt | 229 ++++++
docs/src/main/asciidoc/chapters/clients.txt | 320 ++++++++
docs/src/main/asciidoc/chapters/design.txt | 179 +++++
.../asciidoc/chapters/development_clients.txt | 107 +++
.../asciidoc/chapters/high_speed_ingest.txt | 125 +++
.../src/main/asciidoc/chapters/introduction.txt | 25 +
docs/src/main/asciidoc/chapters/multivolume.txt | 82 ++
docs/src/main/asciidoc/chapters/security.txt | 176 ++++
docs/src/main/asciidoc/chapters/shell.txt | 129 +++
.../asciidoc/chapters/table_configuration.txt | 635 +++++++++++++++
.../src/main/asciidoc/chapters/table_design.txt | 336 ++++++++
.../main/asciidoc/chapters/troubleshooting.txt | 758 ++++++++++++++++++
docs/src/main/asciidoc/images/accumulo-logo.png | Bin 0 -> 11427 bytes
.../main/asciidoc/images/data_distribution.png | Bin 0 -> 86937 bytes
.../main/asciidoc/images/failure_handling.png | Bin 0 -> 48905 bytes
.../accumulo_user_manual.tex | 55 --
.../chapters/administration.tex | 431 ----------
.../accumulo_user_manual/chapters/analytics.tex | 248 ------
.../accumulo_user_manual/chapters/clients.tex | 331 --------
.../accumulo_user_manual/chapters/design.tex | 186 -----
.../chapters/development_clients.tex | 109 ---
.../chapters/high_speed_ingest.tex | 133 ----
.../chapters/introduction.tex | 27 -
.../chapters/multivolume.tex | 85 --
.../accumulo_user_manual/chapters/security.tex | 179 -----
.../accumulo_user_manual/chapters/shell.tex | 138 ----
.../chapters/table_configuration.tex | 663 ----------------
.../chapters/table_design.tex | 343 --------
.../chapters/troubleshooting.tex | 794 -------------------
.../images/data_distribution.png | Bin 86937 -> 0 bytes
.../images/failure_handling.png | Bin 48905 -> 0 bytes
.../zookeeper/RetryingInvocationHandler.java | 63 ++
.../fate/zookeeper/ZooReaderWriter.java | 24 +-
.../RetryingInvocationHandlerTest.java | 87 ++
pom.xml | 12 +-
.../zookeeper/ZooReaderWriterFactory.java | 88 ++
47 files changed, 3994 insertions(+), 3904 deletions(-)
----------------------------------------------------------------------
[49/50] [abbrv] git commit: ACCUMULO-2583 Implement a basic
application of replicated data to the remote table.
Posted by el...@apache.org.
ACCUMULO-2583 Implement a basic application of replicated data to the remote table.
Very little consideration given to error handling, but it works as a first step.
Does not record progress on the remote side yet.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e117a78c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e117a78c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e117a78c
Branch: refs/heads/ACCUMULO-378
Commit: e117a78cb9b6d4324bf14dfdaecb04a86dc92cda
Parents: e84879c
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 22:58:07 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 22:58:07 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +-
.../replication/RemoteReplicationErrorCode.java | 27 +++++
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../replication/ReplicationServicerHandler.java | 103 ++++++++++++++++++-
.../test/replication/ReplicationIT.java | 61 +++++++----
5 files changed, 170 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e117a78c/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index a87b1b4..38e8cc3 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -461,7 +461,7 @@ public enum Property {
@Experimental
REPLICATION_WORK_ASSIGNMENT_SLEEP("replication.work.assignment.sleep", "30s", PropertyType.TIMEDURATION, "Amount of time to sleep between replication work assignment"),
@Experimental
- REPLICATION_WORKER_THREADS("replication.worker.threads", "4", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
+ REPLICATION_WORKER_THREADS("replication.worker.threads", "1", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
@Experimental
REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "10002", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
@Experimental
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e117a78c/core/src/main/java/org/apache/accumulo/core/replication/RemoteReplicationErrorCode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/RemoteReplicationErrorCode.java b/core/src/main/java/org/apache/accumulo/core/replication/RemoteReplicationErrorCode.java
new file mode 100644
index 0000000..b089307
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/replication/RemoteReplicationErrorCode.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.replication;
+
+/**
+ *
+ */
+public enum RemoteReplicationErrorCode {
+ COULD_NOT_DESERIALIZE,
+ COULD_NOT_APPLY,
+ TABLE_DOES_NOT_EXIST,
+ CANNOT_AUTHENTICATE
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e117a78c/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 0b5c6bf..f04bf6b 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -3122,7 +3122,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
private HostAndPort startReplicationService() throws UnknownHostException {
- ReplicationServicer.Iface repl = TraceWrap.service(new ReplicationServicerHandler(this));
+ ReplicationServicer.Iface repl = TraceWrap.service(new ReplicationServicerHandler(HdfsZooInstance.getInstance(), getSystemConfiguration()));
ReplicationServicer.Processor<ReplicationServicer.Iface> processor = new ReplicationServicer.Processor<ReplicationServicer.Iface>(repl);
AccumuloConfiguration conf = getSystemConfiguration();
Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null ? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e117a78c/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
index 5036cab..336bf87 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
@@ -16,10 +16,30 @@
*/
package org.apache.accumulo.tserver.replication;
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.replication.RemoteReplicationErrorCode;
import org.apache.accumulo.core.replication.thrift.KeyValues;
import org.apache.accumulo.core.replication.thrift.RemoteReplicationException;
import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Iface;
import org.apache.accumulo.core.replication.thrift.WalEdits;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.security.SystemCredentials;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,12 +50,93 @@ import org.slf4j.LoggerFactory;
public class ReplicationServicerHandler implements Iface {
private static final Logger log = LoggerFactory.getLogger(ReplicationServicerHandler.class);
+ private Instance inst;
+ private AccumuloConfiguration conf;
+
+ public ReplicationServicerHandler(Instance inst, AccumuloConfiguration conf) {
+ this.inst = inst;
+ this.conf = conf;
+ }
+
+
@Override
public long replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, TException {
- log.error("Got replication request to tableID {} with {} edits", remoteTableId, data.getEditsSize());
+ log.debug("Got replication request to tableID {} with {} edits", remoteTableId, data.getEditsSize());
+
+ String tableId = Integer.toString(remoteTableId);
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ BatchWriter bw = null;
+ try {
+ for (ByteBuffer edit : data.getEdits()) {
+ DataInputStream dis = new DataInputStream(new ByteArrayInputStream(edit.array()));
+ try {
+ key.readFields(dis);
+ value.readFields(dis);
+ } catch (IOException e) {
+ log.error("Could not deserialize edit from stream", e);
+ throw new RemoteReplicationException(RemoteReplicationErrorCode.COULD_NOT_DESERIALIZE.ordinal(), "Could not deserialize edit from stream");
+ }
+
+ // Create the batchScanner if we don't already have one.
+ if (null == bw) {
+ bw = getBatchWriter(tableId);
+ }
+
+ try {
+ bw.addMutations(value.mutations);
+ } catch (MutationsRejectedException e) {
+ log.error("Could not apply mutations to {}", remoteTableId);
+ throw new RemoteReplicationException(RemoteReplicationErrorCode.COULD_NOT_APPLY.ordinal(), "Could not apply mutations to " + remoteTableId);
+ }
+ }
+ } finally {
+ if (null != bw) {
+ try {
+ bw.close();
+ } catch (MutationsRejectedException e) {
+ log.error("Could not apply mutations to {}", remoteTableId);
+ throw new RemoteReplicationException(RemoteReplicationErrorCode.COULD_NOT_APPLY.ordinal(), "Could not apply mutations to " + remoteTableId);
+ }
+ }
+ }
+
+
return data.getEditsSize();
}
+ protected BatchWriter getBatchWriter(String tableId) throws RemoteReplicationException {
+ Credentials creds = SystemCredentials.get();
+ Connector conn;
+ String tableName = null;
+ try {
+ conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ log.error("Could not get connector with system credentials. Something is very wrong", e);
+ throw new RemoteReplicationException(RemoteReplicationErrorCode.CANNOT_AUTHENTICATE.ordinal(), "Cannot get Connector with system credentials");
+ }
+
+ for (Entry<String,String> nameToId : conn.tableOperations().tableIdMap().entrySet()) {
+ if (tableId.equals(nameToId.getValue())) {
+ tableName = nameToId.getKey();
+ break;
+ }
+ }
+
+ if (null == tableName) {
+ log.error("Table with id of " + tableId + " does not exist");
+ throw new RemoteReplicationException(RemoteReplicationErrorCode.TABLE_DOES_NOT_EXIST.ordinal(), "Table with id of " + tableId + " does not exist");
+ }
+
+ try {
+ return conn.createBatchWriter(tableName, new BatchWriterConfig());
+ } catch (TableNotFoundException e) {
+ log.error("Table with id of " + tableId + " does not exist");
+ throw new RemoteReplicationException(RemoteReplicationErrorCode.TABLE_DOES_NOT_EXIST.ordinal(), "Table with id of " + tableId + " does not exist");
+ }
+ }
+
@Override
public long replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, TException {
throw new UnsupportedOperationException();
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e117a78c/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
index 0ad8066..3c1ebac 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@ -26,21 +26,25 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.monitor.Monitor;
import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.protobuf.TextFormat;
+import com.google.common.collect.Iterables;
/**
*
@@ -51,7 +55,7 @@ public class ReplicationIT extends ConfigurableMacIT {
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setNumTservers(1);
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "32M");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "2M");
cfg.setProperty(Property.GC_CYCLE_START, "1s");
cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "5s");
@@ -59,18 +63,22 @@ public class ReplicationIT extends ConfigurableMacIT {
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
- @Test//(timeout = 60 * 1000)
- public void test() throws Exception {
+ @Test(timeout = 60 * 5000)
+ public void dataWasReplicatedToThePeer() throws Exception {
MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
peerCfg.setProperty(Property.MASTER_REPLICATION_COORDINATOR_PORT, "10003");
peerCfg.setProperty(Property.REPLICATION_RECEIPT_SERVICE_PORT, "10004");
+ peerCfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
MiniAccumuloClusterImpl peerCluster = peerCfg.build();
peerCluster.start();
+ Process monitor = peerCluster.exec(Monitor.class);
+
Connector connMaster = getConnector();
Connector connPeer = peerCluster.getConnector("root", ROOT_PASSWORD);
@@ -90,23 +98,23 @@ public class ReplicationIT extends ConfigurableMacIT {
String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
Assert.assertNotNull(peerTableId);
- // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGETS.getKey() + peerClusterName, peerTableId);
// Write some data to table1
BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
- for (int i = 0; i < 100; i++) {
- for (int rows = 0; rows < 1000; rows++) {
- Mutation m = new Mutation(i + "_" + Integer.toString(rows));
- for (int cols = 0; cols < 400; cols++) {
- String value = Integer.toString(cols);
- m.put(value, "", value);
- }
- bw.addMutation(m);
+ for (int rows = 0; rows < 5000; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 100; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
}
+ bw.addMutation(m);
}
+ log.info("Wrote all data to master cluster");
+
bw.close();
while (!connMaster.tableOperations().exists(ReplicationTable.NAME)) {
@@ -114,19 +122,28 @@ public class ReplicationIT extends ConfigurableMacIT {
}
connMaster.tableOperations().compact(masterTable, null, null, true, false);
- for (int i = 0; i < 10; i++) {
+
+ // Wait until we fully replicated something
+ boolean fullyReplicated = false;
+ for (int i = 0; i < 10 && !fullyReplicated; i++) {
+ UtilWaitThread.sleep(2000);
+
Scanner s = ReplicationTable.getScanner(connMaster);
- for (Entry<Key,Value> e : s) {
- Path p = new Path(e.getKey().getRow().toString());
- log.info(p.getName() + " " + e.getKey().getColumnFamily() + " " + e.getKey().getColumnQualifier() + " " + TextFormat.shortDebugString(Status.parseFrom(e.getValue().get())));
+ WorkSection.limit(s);
+ for (Entry<Key,Value> entry : s) {
+ Status status = Status.parseFrom(entry.getValue().get());
+ if (StatusUtil.isFullyReplicated(status)) {
+ fullyReplicated |= true;
+ }
}
+ }
- log.info("");
- log.info("");
+ Assert.assertNotEquals(0, fullyReplicated);
- Thread.sleep(3000);
- }
+ // Once we fully replicated some file, we are guaranteed to have data on the remote
+ Assert.assertTrue(0 < Iterables.size(connPeer.createScanner(peerTable, Authorizations.EMPTY)));
+ monitor.destroy();
peerCluster.stop();
}
[23/50] [abbrv] git commit: Merge branch '1.6.1-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.1-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/428ed9a0
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/428ed9a0
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/428ed9a0
Branch: refs/heads/ACCUMULO-378
Commit: 428ed9a0f720cd3f7dd26f25fbcb760afd844f6d
Parents: 49b1f08 5a72c37
Author: Bill Havanki <bh...@cloudera.com>
Authored: Mon May 5 09:26:38 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon May 5 09:26:38 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/util/AsyncSocketAppender.java | 112 +++++++++++++++++++
.../core/util/AsyncSocketAppenderTest.java | 79 +++++++++++++
.../test/functional/ConfigurableMacIT.java | 3 +
.../test/functional/MonitorLoggingIT.java | 101 +++++++++++++++++
test/src/test/resources/conf/generic_logger.xml | 83 ++++++++++++++
test/src/test/resources/conf/monitor_logger.xml | 64 +++++++++++
6 files changed, 442 insertions(+)
----------------------------------------------------------------------
[12/50] [abbrv] git commit: Merge remote-tracking branch
'origin/master' into ACCUMULO-378
Posted by el...@apache.org.
Merge remote-tracking branch 'origin/master' into ACCUMULO-378
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/23ea6d10
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/23ea6d10
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/23ea6d10
Branch: refs/heads/ACCUMULO-378
Commit: 23ea6d1095f7b3348c7f64b8357a852bef5be9eb
Parents: 274c185 1dae029
Author: Josh Elser <el...@apache.org>
Authored: Fri May 2 16:34:24 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri May 2 16:34:24 2014 -0400
----------------------------------------------------------------------
CHANGES | 914 +---------
README | 18 +
conf/examples/crypto/README | 23 +
conf/examples/crypto/readme.txt | 5 -
.../core/client/admin/ActiveCompaction.java | 79 +-
.../accumulo/core/client/admin/ActiveScan.java | 112 +-
.../client/admin/InstanceOperationsImpl.java | 215 ---
.../client/admin/NamespaceOperationsHelper.java | 214 ---
.../client/admin/NamespaceOperationsImpl.java | 254 ---
.../client/admin/SecurityOperationsImpl.java | 353 ----
.../core/client/admin/TableOperations.java | 15 +-
.../client/admin/TableOperationsHelper.java | 204 ---
.../core/client/admin/TableOperationsImpl.java | 1610 ------------------
.../core/client/impl/ActiveCompactionImpl.java | 109 ++
.../core/client/impl/ActiveScanImpl.java | 146 ++
.../core/client/impl/ConnectorImpl.java | 4 -
.../client/impl/InstanceOperationsImpl.java | 213 +++
.../client/impl/NamespaceOperationsHelper.java | 214 +++
.../client/impl/NamespaceOperationsImpl.java | 250 +++
.../client/impl/SecurityOperationsImpl.java | 351 ++++
.../core/client/impl/TableOperationsHelper.java | 205 +++
.../core/client/impl/TableOperationsImpl.java | 1606 +++++++++++++++++
.../client/mock/MockInstanceOperations.java | 27 +-
.../client/mock/MockNamespaceOperations.java | 4 +-
.../client/mock/MockSecurityOperations.java | 2 +-
.../core/client/mock/MockTableOperations.java | 8 +-
.../core/client/mock/MockTabletLocator.java | 70 -
.../client/mock/impl/MockTabletLocator.java | 70 +
.../constraints/DefaultKeySizeConstraint.java | 6 +
.../client/admin/TableOperationsHelperTest.java | 295 ----
.../client/impl/TableOperationsHelperTest.java | 305 ++++
.../client/mock/MockTableOperationsTest.java | 57 +
.../common/state_diagrams/HDFS_WAL_states.odg | Bin 15864 -> 0 bytes
.../common/state_diagrams/HDFS_WAL_states.pdf | Bin 26256 -> 0 bytes
.../common/state_diagrams/tablet_states.odg | Bin 30540 -> 0 bytes
.../common/state_diagrams/tablet_states.pdf | Bin 38759 -> 0 bytes
.../mapreduce/lib/impl/InputConfigurator.java | 2 +-
.../apache/accumulo/cluster/AccumuloConfig.java | 5 +
pom.xml | 13 +-
.../server/security/SecurityOperation.java | 2 +-
.../accumulo/master/FateServiceHandler.java | 2 +-
.../accumulo/master/tableOps/ImportTable.java | 2 +-
.../accumulo/shell/commands/HistoryCommand.java | 31 +-
.../apache/accumulo/shell/mock/MockShell.java | 8 +-
.../shell/command/HistoryCommandTest.java | 90 +
test/compat/japi-compliance/README | 24 +-
.../japi-compliance/japi-accumulo-1.5.xml | 1 -
.../japi-compliance/japi-accumulo-1.6.xml | 3 +-
.../accumulo/test/TestMultiTableIngest.java | 4 +
.../test/functional/BackupMasterIT.java | 2 +-
.../test/functional/HalfDeadTServerIT.java | 15 +-
.../accumulo/test/functional/ReadWriteIT.java | 9 +-
.../accumulo/test/randomwalk/FrameworkTest.java | 48 +-
test/system/continuous/ScaleTest.odp | Bin 21160 -> 0 bytes
54 files changed, 3854 insertions(+), 4365 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/23ea6d10/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
----------------------------------------------------------------------
[03/50] [abbrv] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/7a826251
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/7a826251
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/7a826251
Branch: refs/heads/ACCUMULO-378
Commit: 7a826251d6e988433e1a5cc33b08506640637663
Parents: 43a44ba b3c8936
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Apr 30 12:16:05 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Apr 30 12:16:05 2014 -0400
----------------------------------------------------------------------
.../accumulo/test/functional/HalfDeadTServerIT.java | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[27/50] [abbrv] git commit: Merge branch 'master' into ACCUMULO-378
Posted by el...@apache.org.
Merge branch 'master' into ACCUMULO-378
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/59ecd21b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/59ecd21b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/59ecd21b
Branch: refs/heads/ACCUMULO-378
Commit: 59ecd21baeaa499853682bb19f86ab4757706a8d
Parents: 102ec71 ab45354
Author: Josh Elser <el...@apache.org>
Authored: Mon May 5 12:50:58 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Mon May 5 12:50:58 2014 -0400
----------------------------------------------------------------------
conf/templates/generic_logger.properties | 58 ++++++++++
conf/templates/monitor_logger.properties | 44 ++++++++
.../accumulo/core/util/AsyncSocketAppender.java | 112 +++++++++++++++++++
.../core/util/AsyncSocketAppenderTest.java | 79 +++++++++++++
.../org/apache/accumulo/server/Accumulo.java | 39 ++++++-
.../server/watcher/MonitorLog4jWatcher.java | 19 +++-
.../apache/accumulo/server/AccumuloTest.java | 51 +++++++++
.../server/watcher/MonitorLog4jWatcherTest.java | 50 +++++++++
.../test/functional/ConfigurableMacIT.java | 3 +
.../test/functional/MonitorLoggingIT.java | 101 +++++++++++++++++
test/src/test/resources/conf/generic_logger.xml | 83 ++++++++++++++
test/src/test/resources/conf/monitor_logger.xml | 64 +++++++++++
12 files changed, 694 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[28/50] [abbrv] ACCUMULO-2583 First stab at setting up the actual
wire transfer to the replication peer
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java b/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
index 0497892..337a592 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
@@ -23,7 +23,7 @@ import java.util.Set;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.replication.thrift.NoServersAvailableException;
-import org.apache.accumulo.core.replication.thrift.RemoteReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.thrift.TException;
@@ -33,7 +33,7 @@ import com.google.common.base.Preconditions;
/**
* Choose a tserver to service a replication task
*/
-public class MasterReplicationCoordinator implements RemoteReplicationCoordinator.Iface {
+public class MasterReplicationCoordinator implements ReplicationCoordinator.Iface {
public static enum NoServersAvailable {
NO_ONLINE_SERVERS
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
index 370fcfd..699381f 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -16,7 +16,6 @@
*/
package org.apache.accumulo.master.replication;
-import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
@@ -38,6 +37,7 @@ import org.apache.accumulo.core.replication.proto.Replication.Status;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.server.replication.ReplicationWorkAssignerHelper;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
import org.apache.accumulo.server.zookeeper.ZooCache;
import org.apache.hadoop.fs.Path;
@@ -215,8 +215,8 @@ public class ReplicationWorkAssigner implements Runnable {
if (StatusUtil.isWorkRequired(status)) {
Path p = new Path(file);
String filename = p.getName();
- entry.getKey().getColumnQualifier(buffer);
- String key = filename + "|" + buffer;
+ WorkSection.getTarget(entry.getKey(), buffer);
+ String key = ReplicationWorkAssignerHelper.getQueueKey(filename, buffer.toString());
// And, we haven't already queued this file up for work already
if (!queuedWork.contains(key)) {
@@ -234,10 +234,10 @@ public class ReplicationWorkAssigner implements Runnable {
/**
* Distribute the work for the given path with filename
*
+ * @param key
+ * Unique key to identify this work in the queue
* @param path
* Full path to a file
- * @param filename
- * Filename
*/
protected void queueWork(String key, String path) {
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
index 40e9838..cfc756c 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
@@ -142,7 +142,7 @@ public class WorkMaker {
buffer.reset();
// Set up the writable
- target.setRemoteName(entry.getKey());
+ target.setPeerName(entry.getKey());
target.setRemoteIdentifier(entry.getValue());
target.write(buffer);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
index 89a1ddd..16e6757 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
@@ -22,7 +22,6 @@ import static org.easymock.EasyMock.expectLastCall;
import static org.easymock.EasyMock.replay;
import static org.easymock.EasyMock.verify;
-import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashSet;
@@ -76,7 +75,7 @@ public class ReplicationWorkAssignerTest {
@Test
public void workQueuedUsingFileName() throws Exception {
ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
- Text serializedTarget = ReplicationTarget.toText(target);
+ Text serializedTarget = target.toText();
DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
Set<String> queuedWork = new HashSet<>();
@@ -118,7 +117,7 @@ public class ReplicationWorkAssignerTest {
@Test
public void createWorkForFilesNeedingIt() throws Exception {
ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
- Text serializedTarget1 = ReplicationTarget.toText(target1), serializedTarget2 = ReplicationTarget.toText(target2);
+ Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText();
MockInstance inst = new MockInstance(test.getMethodName());
Credentials creds = new Credentials("root", new PasswordToken(""));
@@ -191,7 +190,7 @@ public class ReplicationWorkAssignerTest {
@Test
public void doNotCreateWorkForFilesNotNeedingIt() throws Exception {
ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
- Text serializedTarget1 = ReplicationTarget.toText(target1), serializedTarget2 = ReplicationTarget.toText(target2);
+ Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText();
MockInstance inst = new MockInstance(test.getMethodName());
Credentials creds = new Credentials("root", new PasswordToken(""));
@@ -263,7 +262,7 @@ public class ReplicationWorkAssignerTest {
assigner.setQueuedWork(queuedWork);
ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
- Text serializedTarget = ReplicationTarget.toText(target);
+ Text serializedTarget = target.toText();
queuedWork.add("wal1|" + serializedTarget.toString());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
index 82d7ae9..c0aa31d 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
@@ -16,22 +16,146 @@
*/
package org.apache.accumulo.tserver.replication;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.client.replication.ReplicaSystemFactory;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.protobuf.ProtobufUtil;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.server.replication.ReplicationWorkAssignerHelper;
+import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
+import org.apache.hadoop.fs.Path;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Iterables;
+import com.google.protobuf.InvalidProtocolBufferException;
/**
- *
+ * Transmit the given data to a peer
*/
public class ReplicationProcessor implements Processor {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationProcessor.class);
+
+ private Instance inst;
+ private AccumuloConfiguration conf;
+ private VolumeManager fs;
+ private Credentials creds;
+
+ public ReplicationProcessor(Instance inst, AccumuloConfiguration conf, VolumeManager fs) {
+ this.conf = conf;
+ this.fs = fs;
+ creds = SystemCredentials.get();
+ }
@Override
public ReplicationProcessor newProcessor() {
- return new ReplicationProcessor();
+ return new ReplicationProcessor(inst, conf, fs);
}
@Override
public void process(String workID, byte[] data) {
- // TODO Auto-generated method stub
+ ReplicationTarget target = ReplicationWorkAssignerHelper.fromQueueKey(workID).getValue();
+ String file = new String(data);
+
+ // Find the configured replication peer so we know how to replicate to it
+ Map<String,String> configuredPeers = conf.getAllPropertiesWithPrefix(Property.REPLICATION_PEERS);
+ String peerType = configuredPeers.get(target.getPeerName());
+ if (null == peerType) {
+ String msg = "Cannot process replication for unknown peer: " + target.getPeerName();
+ log.warn(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ // Get the peer that we're replicating to
+ ReplicaSystem replica = ReplicaSystemFactory.get(peerType);
+ Status status;
+ try {
+ status = getStatus(file, target);
+ } catch (TableNotFoundException | AccumuloException | AccumuloSecurityException e) {
+ log.error("Could not look for necessary replication record", e);
+ throw new IllegalStateException("Could not look for replication record", e);
+ } catch (InvalidProtocolBufferException e) {
+ log.error("Could not deserialize Status from Work section for {} and ", file, target);
+ throw new RuntimeException("Could not parse Status for work record", e);
+ }
+ // We don't need to do anything (shouldn't have gotten this work record in the first place)
+ if (!StatusUtil.isWorkRequired(status)) {
+ log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
+ return;
+ }
+
+ // Sanity check that nothing bad happened and our replication source still exists
+ Path filePath = new Path(file);
+ try {
+ if (!fs.exists(filePath)) {
+ log.warn("Received work request for {} and {}, but the file doesn't exist", filePath, target);
+ return;
+ }
+ } catch (IOException e) {
+ log.error("Could not determine if file exists {}", filePath, e);
+ throw new RuntimeException(e);
+ }
+
+ // Replicate that sucker
+ Status replicatedStatus = replica.replicate(filePath, status, target);
+
+ // If we got a different status
+ if (!replicatedStatus.equals(status)) {
+ // We actually did some work!
+ recordNewStatus(filePath, replicatedStatus, target);
+ }
+
+ // otherwise, we didn't actually replicate because there was error sending the data
+ // we can just not record any updates, and it will be picked up again by the work assigner
}
+ public Status getStatus(String file, ReplicationTarget target) throws TableNotFoundException, AccumuloException, AccumuloSecurityException, InvalidProtocolBufferException {
+ Scanner s = ReplicationTable.getScanner(inst.getConnector(creds.getPrincipal(), creds.getToken()));
+ s.setRange(Range.exact(file));
+ s.fetchColumn(WorkSection.NAME, target.toText());
+
+ return Status.parseFrom(Iterables.getOnlyElement(s).getValue().get());
+ }
+
+ /**
+ * Record the updated Status for this file and target
+ * @param filePath Path to file being replicated
+ * @param status Updated Status after replication
+ * @param target Peer that was replicated to
+ */
+ public void recordNewStatus(Path filePath, Status status, ReplicationTarget target) {
+ try {
+ Connector conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+ BatchWriter bw = ReplicationTable.getBatchWriter(conn);
+ Mutation m = new Mutation(filePath.toString());
+ WorkSection.add(m, target.toText(), ProtobufUtil.toValue(status));
+ bw.addMutation(m);
+ bw.close();
+ } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
+ log.error("Error recording updated Status for {}", filePath, e);
+ throw new RuntimeException(e);
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
index 627e51a..e00f343 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
@@ -49,7 +49,7 @@ public class ReplicationWorker implements Runnable {
@Override
public void run() {
try {
- new DistributedWorkQueue(ZooUtil.getRoot(inst) + Constants.ZREPLICATION, conf).startProcessing(new ReplicationProcessor(), executor);
+ new DistributedWorkQueue(ZooUtil.getRoot(inst) + Constants.ZREPLICATION, conf).startProcessing(new ReplicationProcessor(inst, conf, fs), executor);
} catch (KeeperException | InterruptedException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
index 33da8a0..92f6fb3 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
@@ -385,7 +385,7 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
s = ReplicationTable.getScanner(conn);
WorkSection.limit(s);
Entry<Key,Value> e = Iterables.getOnlyElement(s);
- Text expectedColqual = ReplicationTarget.toText(new ReplicationTarget("cluster1", "4"));
+ Text expectedColqual = new ReplicationTarget("cluster1", "4").toText();
Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier());
notFound = false;
} catch (NoSuchElementException e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
index 613a01b..defff65 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithMakerTest.java
@@ -303,7 +303,7 @@ public class ReplicationWithMakerTest extends ConfigurableMacIT {
WorkSection.limit(s);
try {
Entry<Key,Value> e = Iterables.getOnlyElement(s);
- Text expectedColqual = ReplicationTarget.toText(new ReplicationTarget("cluster1", "4"));
+ Text expectedColqual = new ReplicationTarget("cluster1", "4").toText();
Assert.assertEquals(expectedColqual, e.getKey().getColumnQualifier());
notFound = false;
} catch (NoSuchElementException e) {} catch (IllegalArgumentException e) {
[33/50] [abbrv] git commit: ACCUMULO-2583 Get compilation working
with slf4j dependencies. Lift the retry count into a Property.
Posted by el...@apache.org.
ACCUMULO-2583 Get compilation working with slf4j dependencies. Lift the retry count into a Property.
Left a todo for myself as well.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c7b6062c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c7b6062c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c7b6062c
Branch: refs/heads/ACCUMULO-378
Commit: c7b6062c315ecde8321f83e50afa757003c24a97
Parents: baa7a4f
Author: Josh Elser <el...@apache.org>
Authored: Wed May 7 00:55:15 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed May 7 00:55:15 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../core/replication/AccumuloReplicaSystem.java | 124 -----------------
server/base/pom.xml | 18 ++-
.../replication/AccumuloReplicaSystem.java | 137 +++++++++++++++++++
server/tserver/pom.xml | 18 ++-
5 files changed, 155 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b6062c/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index f31cd46..6d3b00c 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -462,6 +462,8 @@ public enum Property {
REPLICATION_WORKER_THREADS("replication.worker.threads", "4", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
@Experimental
REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "10001", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
+ @Experimental
+ REPLICATION_WORK_ATTEMPTS("replication.work.attempts", "10", PropertyType.COUNT, "Number of attempts to try to replicate some data before giving up and letting it naturally be retried later"),
;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b6062c/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java b/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java
deleted file mode 100644
index ca66b93..0000000
--- a/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.core.replication;
-
-import java.nio.ByteBuffer;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.impl.ClientExecReturn;
-import org.apache.accumulo.core.client.impl.ReplicationClient;
-import org.apache.accumulo.core.client.replication.ReplicaSystem;
-import org.apache.accumulo.core.replication.proto.Replication.Status;
-import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
-import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
-import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client;
-import org.apache.accumulo.core.util.ByteBufferUtil;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.hadoop.fs.Path;
-import org.apache.thrift.transport.TTransportException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- *
- */
-public class AccumuloReplicaSystem implements ReplicaSystem {
- private static final Logger log = LoggerFactory.getLogger(AccumuloReplicaSystem.class);
-
- private String instanceName, zookeepers;
-
- @Override
- public void configure(String configuration) {
- Preconditions.checkNotNull(configuration);
-
- int index = configuration.indexOf(',');
- if (-1 == index) {
- throw new IllegalArgumentException("Expected comma in configuration string");
- }
-
- instanceName = configuration.substring(0, index);
- zookeepers = configuration.substring(index + 1);
- }
-
- @Override
- public Status replicate(Path p, Status status, ReplicationTarget target) {
- Instance peerInstance = getPeerInstance(target);
- // Remote identifier is an integer (table id) in this case.
- final int remoteTableId = Integer.parseInt(target.getRemoteIdentifier());
-
- for (int i = 0; i < 10; i++) {
- String peerTserver;
- try {
- // Ask the master on the remote what TServer we should talk with to replicate the data
- peerTserver = ReplicationClient.executeCoordinatorWithReturn(peerInstance, new ClientExecReturn<String,ReplicationCoordinator.Client>() {
-
- @Override
- public String execute(ReplicationCoordinator.Client client) throws Exception {
- return client.getServicerAddress(remoteTableId);
- }
-
- });
- } catch (AccumuloException | AccumuloSecurityException e) {
- // No progress is made
- log.error("Could not connect to master at {}, cannot proceed with replication. Will retry", target, e);
- continue;
- }
-
- if (null == peerTserver) {
- // Something went wrong, and we didn't get a valid tserver from the remote for some reason
- log.warn("Did not receive tserver from master at {}, cannot proceed with replication. Will retry.", target);
- continue;
- }
-
- // We have a tserver on the remote -- send the data its way.
- ByteBuffer result;
- try {
- result = ReplicationClient.executeServicerWithReturn(peerInstance, peerTserver, new ClientExecReturn<ByteBuffer,ReplicationServicer.Client>() {
- @Override
- public ByteBuffer execute(Client client) throws Exception {
- return client.replicateLog(remoteTableId, null);
- }
- });
-
- // We need to be able to parse the returned Status,
- // if we can't, we don't know what the server actually parsed.
- try {
- return Status.parseFrom(ByteBufferUtil.toBytes(result));
- } catch (InvalidProtocolBufferException e) {
- log.error("Could not parse return Status from {}", peerTserver, e);
- throw new RuntimeException("Could not parse returned Status from " + peerTserver, e);
- }
- } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
- log.warn("Could not connect to remote server {}, will retry", peerTserver, e);
- UtilWaitThread.sleep(250);
- }
- }
-
- // We made no status, punt on it for now, and let it re-queue itself for work
- return status;
- }
-
- public Instance getPeerInstance(ReplicationTarget target) {
- return new ZooKeeperInstance(instanceName, zookeepers);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b6062c/server/base/pom.xml
----------------------------------------------------------------------
diff --git a/server/base/pom.xml b/server/base/pom.xml
index f698621..672fe91 100644
--- a/server/base/pom.xml
+++ b/server/base/pom.xml
@@ -92,23 +92,21 @@
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <scope>test</scope>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b6062c/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java b/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java
new file mode 100644
index 0000000..138eb71
--- /dev/null
+++ b/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.replication;
+
+import java.nio.ByteBuffer;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.ReplicationClient;
+import org.apache.accumulo.core.client.impl.ServerConfigurationUtil;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.hadoop.fs.Path;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ *
+ */
+public class AccumuloReplicaSystem implements ReplicaSystem {
+ private static final Logger log = LoggerFactory.getLogger(AccumuloReplicaSystem.class);
+
+ private String instanceName, zookeepers;
+
+ @Override
+ public void configure(String configuration) {
+ Preconditions.checkNotNull(configuration);
+
+ int index = configuration.indexOf(',');
+ if (-1 == index) {
+ throw new IllegalArgumentException("Expected comma in configuration string");
+ }
+
+ instanceName = configuration.substring(0, index);
+ zookeepers = configuration.substring(index + 1);
+ }
+
+ @Override
+ public Status replicate(Path p, Status status, ReplicationTarget target) {
+ Instance localInstance = HdfsZooInstance.getInstance();
+ AccumuloConfiguration localConf = ServerConfigurationUtil.getConfiguration(localInstance);
+
+ Instance peerInstance = getPeerInstance(target);
+ // Remote identifier is an integer (table id) in this case.
+ final int remoteTableId = Integer.parseInt(target.getRemoteIdentifier());
+
+ // Attempt the replication of this status a number of times before giving up and
+ // trying to replicate it again later some other time.
+ for (int i = 0; i < localConf.getCount(Property.REPLICATION_WORK_ATTEMPTS); i++) {
+ String peerTserver;
+ try {
+ // Ask the master on the remote what TServer we should talk with to replicate the data
+ peerTserver = ReplicationClient.executeCoordinatorWithReturn(peerInstance, new ClientExecReturn<String,ReplicationCoordinator.Client>() {
+
+ @Override
+ public String execute(ReplicationCoordinator.Client client) throws Exception {
+ return client.getServicerAddress(remoteTableId);
+ }
+
+ });
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ // No progress is made
+ log.error("Could not connect to master at {}, cannot proceed with replication. Will retry", target, e);
+ continue;
+ }
+
+ if (null == peerTserver) {
+ // Something went wrong, and we didn't get a valid tserver from the remote for some reason
+ log.warn("Did not receive tserver from master at {}, cannot proceed with replication. Will retry.", target);
+ continue;
+ }
+
+ // We have a tserver on the remote -- send the data its way.
+ ByteBuffer result;
+ //TODO should chunk up the given file into some configurable sizes instead of just sending the entire file all at once
+ // configuration should probably just be size based.
+ try {
+ result = ReplicationClient.executeServicerWithReturn(peerInstance, peerTserver, new ClientExecReturn<ByteBuffer,ReplicationServicer.Client>() {
+ @Override
+ public ByteBuffer execute(Client client) throws Exception {
+ //TODO This needs to actually send the appropriate data, and choose replicateLog or replicateKeyValues
+ return client.replicateLog(remoteTableId, null);
+ }
+ });
+
+ // We need to be able to parse the returned Status,
+ // if we can't, we don't know what the server actually parsed.
+ try {
+ return Status.parseFrom(ByteBufferUtil.toBytes(result));
+ } catch (InvalidProtocolBufferException e) {
+ log.error("Could not parse return Status from {}", peerTserver, e);
+ throw new RuntimeException("Could not parse returned Status from " + peerTserver, e);
+ }
+ } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
+ log.warn("Could not connect to remote server {}, will retry", peerTserver, e);
+ UtilWaitThread.sleep(250);
+ }
+ }
+
+ // We made no status, punt on it for now, and let it re-queue itself for work
+ return status;
+ }
+
+ public Instance getPeerInstance(ReplicationTarget target) {
+ return new ZooKeeperInstance(instanceName, zookeepers);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c7b6062c/server/tserver/pom.xml
----------------------------------------------------------------------
diff --git a/server/tserver/pom.xml b/server/tserver/pom.xml
index 100bc2d..e09f348 100644
--- a/server/tserver/pom.xml
+++ b/server/tserver/pom.xml
@@ -88,23 +88,21 @@
<artifactId>zookeeper</artifactId>
</dependency>
<dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <scope>test</scope>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
[15/50] [abbrv] git commit: ACCUMULO-2573 ACCUMULO-2709 Impl the
tserver side of the work queue. Remove the thrift interface.
Posted by el...@apache.org.
ACCUMULO-2573 ACCUMULO-2709 Impl the tserver side of the work queue. Remove the thrift interface.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/05de21fb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/05de21fb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/05de21fb
Branch: refs/heads/ACCUMULO-378
Commit: 05de21fb77dd8f1b381ce961e6362460810fbe70
Parents: 23ea6d1
Author: Josh Elser <el...@apache.org>
Authored: Fri May 2 18:40:01 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri May 2 18:40:01 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../thrift/TabletServerReplicationService.java | 1538 ------------------
core/src/main/thrift/tabletserver.thrift | 8 -
.../apache/accumulo/tserver/TabletServer.java | 7 +
.../replication/ReplicationProcessor.java | 37 +
.../tserver/replication/ReplicationWorker.java | 57 +
.../replication/TabletServerReplication.java | 38 -
7 files changed, 103 insertions(+), 1584 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 637600d..db810be 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -458,6 +458,8 @@ public enum Property {
REPLICATION_MAX_WORK_QUEUE("replication.max.work.queue", "20000000", PropertyType.COUNT, "Upper bound of the number of files queued for replication"),
@Experimental
REPLICATION_WORK_ASSIGNMENT_SLEEP("replication.work.assignment.sleep", "30s", PropertyType.TIMEDURATION, "Amount of time to sleep between replication work assignment"),
+ @Experimental
+ REPLICATION_WORKER_THREADS("replication.worker.threads", "4", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletServerReplicationService.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletServerReplicationService.java b/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletServerReplicationService.java
deleted file mode 100644
index dcea936..0000000
--- a/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletServerReplicationService.java
+++ /dev/null
@@ -1,1538 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.accumulo.core.tabletserver.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings("all") public class TabletServerReplicationService {
-
- public interface Iface {
-
- public boolean replicateData(org.apache.accumulo.trace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.TCredentials credentials, String file, long offset, long count) throws org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException, ReplicationFailedException, org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void replicateData(org.apache.accumulo.trace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.TCredentials credentials, String file, long offset, long count, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateData_call> resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public boolean replicateData(org.apache.accumulo.trace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.TCredentials credentials, String file, long offset, long count) throws org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException, ReplicationFailedException, org.apache.thrift.TException
- {
- send_replicateData(tinfo, credentials, file, offset, count);
- return recv_replicateData();
- }
-
- public void send_replicateData(org.apache.accumulo.trace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.TCredentials credentials, String file, long offset, long count) throws org.apache.thrift.TException
- {
- replicateData_args args = new replicateData_args();
- args.setTinfo(tinfo);
- args.setCredentials(credentials);
- args.setFile(file);
- args.setOffset(offset);
- args.setCount(count);
- sendBase("replicateData", args);
- }
-
- public boolean recv_replicateData() throws org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException, ReplicationFailedException, org.apache.thrift.TException
- {
- replicateData_result result = new replicateData_result();
- receiveBase(result, "replicateData");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.sec != null) {
- throw result.sec;
- }
- if (result.rfe != null) {
- throw result.rfe;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "replicateData failed: unknown result");
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void replicateData(org.apache.accumulo.trace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.TCredentials credentials, String file, long offset, long count, org.apache.thrift.async.AsyncMethodCallback<replicateData_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- replicateData_call method_call = new replicateData_call(tinfo, credentials, file, offset, count, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class replicateData_call extends org.apache.thrift.async.TAsyncMethodCall {
- private org.apache.accumulo.trace.thrift.TInfo tinfo;
- private org.apache.accumulo.core.security.thrift.TCredentials credentials;
- private String file;
- private long offset;
- private long count;
- public replicateData_call(org.apache.accumulo.trace.thrift.TInfo tinfo, org.apache.accumulo.core.security.thrift.TCredentials credentials, String file, long offset, long count, org.apache.thrift.async.AsyncMethodCallback<replicateData_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.tinfo = tinfo;
- this.credentials = credentials;
- this.file = file;
- this.offset = offset;
- this.count = count;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateData", org.apache.thrift.protocol.TMessageType.CALL, 0));
- replicateData_args args = new replicateData_args();
- args.setTinfo(tinfo);
- args.setCredentials(credentials);
- args.setFile(file);
- args.setOffset(offset);
- args.setCount(count);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public boolean getResult() throws org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException, ReplicationFailedException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_replicateData();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("replicateData", new replicateData());
- return processMap;
- }
-
- public static class replicateData<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateData_args> {
- public replicateData() {
- super("replicateData");
- }
-
- public replicateData_args getEmptyArgsInstance() {
- return new replicateData_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public replicateData_result getResult(I iface, replicateData_args args) throws org.apache.thrift.TException {
- replicateData_result result = new replicateData_result();
- try {
- result.success = iface.replicateData(args.tinfo, args.credentials, args.file, args.offset, args.count);
- result.setSuccessIsSet(true);
- } catch (org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException sec) {
- result.sec = sec;
- } catch (ReplicationFailedException rfe) {
- result.rfe = rfe;
- }
- return result;
- }
- }
-
- }
-
- public static class replicateData_args implements org.apache.thrift.TBase<replicateData_args, replicateData_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateData_args");
-
- private static final org.apache.thrift.protocol.TField TINFO_FIELD_DESC = new org.apache.thrift.protocol.TField("tinfo", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField CREDENTIALS_FIELD_DESC = new org.apache.thrift.protocol.TField("credentials", org.apache.thrift.protocol.TType.STRUCT, (short)2);
- private static final org.apache.thrift.protocol.TField FILE_FIELD_DESC = new org.apache.thrift.protocol.TField("file", org.apache.thrift.protocol.TType.STRING, (short)3);
- private static final org.apache.thrift.protocol.TField OFFSET_FIELD_DESC = new org.apache.thrift.protocol.TField("offset", org.apache.thrift.protocol.TType.I64, (short)4);
- private static final org.apache.thrift.protocol.TField COUNT_FIELD_DESC = new org.apache.thrift.protocol.TField("count", org.apache.thrift.protocol.TType.I64, (short)5);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateData_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateData_argsTupleSchemeFactory());
- }
-
- public org.apache.accumulo.trace.thrift.TInfo tinfo; // required
- public org.apache.accumulo.core.security.thrift.TCredentials credentials; // required
- public String file; // required
- public long offset; // required
- public long count; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- TINFO((short)1, "tinfo"),
- CREDENTIALS((short)2, "credentials"),
- FILE((short)3, "file"),
- OFFSET((short)4, "offset"),
- COUNT((short)5, "count");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // TINFO
- return TINFO;
- case 2: // CREDENTIALS
- return CREDENTIALS;
- case 3: // FILE
- return FILE;
- case 4: // OFFSET
- return OFFSET;
- case 5: // COUNT
- return COUNT;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __OFFSET_ISSET_ID = 0;
- private static final int __COUNT_ISSET_ID = 1;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.TINFO, new org.apache.thrift.meta_data.FieldMetaData("tinfo", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.trace.thrift.TInfo.class)));
- tmpMap.put(_Fields.CREDENTIALS, new org.apache.thrift.meta_data.FieldMetaData("credentials", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, org.apache.accumulo.core.security.thrift.TCredentials.class)));
- tmpMap.put(_Fields.FILE, new org.apache.thrift.meta_data.FieldMetaData("file", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.OFFSET, new org.apache.thrift.meta_data.FieldMetaData("offset", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- tmpMap.put(_Fields.COUNT, new org.apache.thrift.meta_data.FieldMetaData("count", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateData_args.class, metaDataMap);
- }
-
- public replicateData_args() {
- }
-
- public replicateData_args(
- org.apache.accumulo.trace.thrift.TInfo tinfo,
- org.apache.accumulo.core.security.thrift.TCredentials credentials,
- String file,
- long offset,
- long count)
- {
- this();
- this.tinfo = tinfo;
- this.credentials = credentials;
- this.file = file;
- this.offset = offset;
- setOffsetIsSet(true);
- this.count = count;
- setCountIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateData_args(replicateData_args other) {
- __isset_bitfield = other.__isset_bitfield;
- if (other.isSetTinfo()) {
- this.tinfo = new org.apache.accumulo.trace.thrift.TInfo(other.tinfo);
- }
- if (other.isSetCredentials()) {
- this.credentials = new org.apache.accumulo.core.security.thrift.TCredentials(other.credentials);
- }
- if (other.isSetFile()) {
- this.file = other.file;
- }
- this.offset = other.offset;
- this.count = other.count;
- }
-
- public replicateData_args deepCopy() {
- return new replicateData_args(this);
- }
-
- @Override
- public void clear() {
- this.tinfo = null;
- this.credentials = null;
- this.file = null;
- setOffsetIsSet(false);
- this.offset = 0;
- setCountIsSet(false);
- this.count = 0;
- }
-
- public org.apache.accumulo.trace.thrift.TInfo getTinfo() {
- return this.tinfo;
- }
-
- public replicateData_args setTinfo(org.apache.accumulo.trace.thrift.TInfo tinfo) {
- this.tinfo = tinfo;
- return this;
- }
-
- public void unsetTinfo() {
- this.tinfo = null;
- }
-
- /** Returns true if field tinfo is set (has been assigned a value) and false otherwise */
- public boolean isSetTinfo() {
- return this.tinfo != null;
- }
-
- public void setTinfoIsSet(boolean value) {
- if (!value) {
- this.tinfo = null;
- }
- }
-
- public org.apache.accumulo.core.security.thrift.TCredentials getCredentials() {
- return this.credentials;
- }
-
- public replicateData_args setCredentials(org.apache.accumulo.core.security.thrift.TCredentials credentials) {
- this.credentials = credentials;
- return this;
- }
-
- public void unsetCredentials() {
- this.credentials = null;
- }
-
- /** Returns true if field credentials is set (has been assigned a value) and false otherwise */
- public boolean isSetCredentials() {
- return this.credentials != null;
- }
-
- public void setCredentialsIsSet(boolean value) {
- if (!value) {
- this.credentials = null;
- }
- }
-
- public String getFile() {
- return this.file;
- }
-
- public replicateData_args setFile(String file) {
- this.file = file;
- return this;
- }
-
- public void unsetFile() {
- this.file = null;
- }
-
- /** Returns true if field file is set (has been assigned a value) and false otherwise */
- public boolean isSetFile() {
- return this.file != null;
- }
-
- public void setFileIsSet(boolean value) {
- if (!value) {
- this.file = null;
- }
- }
-
- public long getOffset() {
- return this.offset;
- }
-
- public replicateData_args setOffset(long offset) {
- this.offset = offset;
- setOffsetIsSet(true);
- return this;
- }
-
- public void unsetOffset() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __OFFSET_ISSET_ID);
- }
-
- /** Returns true if field offset is set (has been assigned a value) and false otherwise */
- public boolean isSetOffset() {
- return EncodingUtils.testBit(__isset_bitfield, __OFFSET_ISSET_ID);
- }
-
- public void setOffsetIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __OFFSET_ISSET_ID, value);
- }
-
- public long getCount() {
- return this.count;
- }
-
- public replicateData_args setCount(long count) {
- this.count = count;
- setCountIsSet(true);
- return this;
- }
-
- public void unsetCount() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- /** Returns true if field count is set (has been assigned a value) and false otherwise */
- public boolean isSetCount() {
- return EncodingUtils.testBit(__isset_bitfield, __COUNT_ISSET_ID);
- }
-
- public void setCountIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __COUNT_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case TINFO:
- if (value == null) {
- unsetTinfo();
- } else {
- setTinfo((org.apache.accumulo.trace.thrift.TInfo)value);
- }
- break;
-
- case CREDENTIALS:
- if (value == null) {
- unsetCredentials();
- } else {
- setCredentials((org.apache.accumulo.core.security.thrift.TCredentials)value);
- }
- break;
-
- case FILE:
- if (value == null) {
- unsetFile();
- } else {
- setFile((String)value);
- }
- break;
-
- case OFFSET:
- if (value == null) {
- unsetOffset();
- } else {
- setOffset((Long)value);
- }
- break;
-
- case COUNT:
- if (value == null) {
- unsetCount();
- } else {
- setCount((Long)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case TINFO:
- return getTinfo();
-
- case CREDENTIALS:
- return getCredentials();
-
- case FILE:
- return getFile();
-
- case OFFSET:
- return Long.valueOf(getOffset());
-
- case COUNT:
- return Long.valueOf(getCount());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case TINFO:
- return isSetTinfo();
- case CREDENTIALS:
- return isSetCredentials();
- case FILE:
- return isSetFile();
- case OFFSET:
- return isSetOffset();
- case COUNT:
- return isSetCount();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateData_args)
- return this.equals((replicateData_args)that);
- return false;
- }
-
- public boolean equals(replicateData_args that) {
- if (that == null)
- return false;
-
- boolean this_present_tinfo = true && this.isSetTinfo();
- boolean that_present_tinfo = true && that.isSetTinfo();
- if (this_present_tinfo || that_present_tinfo) {
- if (!(this_present_tinfo && that_present_tinfo))
- return false;
- if (!this.tinfo.equals(that.tinfo))
- return false;
- }
-
- boolean this_present_credentials = true && this.isSetCredentials();
- boolean that_present_credentials = true && that.isSetCredentials();
- if (this_present_credentials || that_present_credentials) {
- if (!(this_present_credentials && that_present_credentials))
- return false;
- if (!this.credentials.equals(that.credentials))
- return false;
- }
-
- boolean this_present_file = true && this.isSetFile();
- boolean that_present_file = true && that.isSetFile();
- if (this_present_file || that_present_file) {
- if (!(this_present_file && that_present_file))
- return false;
- if (!this.file.equals(that.file))
- return false;
- }
-
- boolean this_present_offset = true;
- boolean that_present_offset = true;
- if (this_present_offset || that_present_offset) {
- if (!(this_present_offset && that_present_offset))
- return false;
- if (this.offset != that.offset)
- return false;
- }
-
- boolean this_present_count = true;
- boolean that_present_count = true;
- if (this_present_count || that_present_count) {
- if (!(this_present_count && that_present_count))
- return false;
- if (this.count != that.count)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateData_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateData_args typedOther = (replicateData_args)other;
-
- lastComparison = Boolean.valueOf(isSetTinfo()).compareTo(typedOther.isSetTinfo());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetTinfo()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.tinfo, typedOther.tinfo);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCredentials()).compareTo(typedOther.isSetCredentials());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCredentials()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.credentials, typedOther.credentials);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetFile()).compareTo(typedOther.isSetFile());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetFile()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.file, typedOther.file);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetOffset()).compareTo(typedOther.isSetOffset());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetOffset()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.offset, typedOther.offset);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetCount()).compareTo(typedOther.isSetCount());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetCount()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.count, typedOther.count);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateData_args(");
- boolean first = true;
-
- sb.append("tinfo:");
- if (this.tinfo == null) {
- sb.append("null");
- } else {
- sb.append(this.tinfo);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("credentials:");
- if (this.credentials == null) {
- sb.append("null");
- } else {
- sb.append(this.credentials);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("file:");
- if (this.file == null) {
- sb.append("null");
- } else {
- sb.append(this.file);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("offset:");
- sb.append(this.offset);
- first = false;
- if (!first) sb.append(", ");
- sb.append("count:");
- sb.append(this.count);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (tinfo != null) {
- tinfo.validate();
- }
- if (credentials != null) {
- credentials.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateData_argsStandardSchemeFactory implements SchemeFactory {
- public replicateData_argsStandardScheme getScheme() {
- return new replicateData_argsStandardScheme();
- }
- }
-
- private static class replicateData_argsStandardScheme extends StandardScheme<replicateData_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateData_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // TINFO
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.tinfo = new org.apache.accumulo.trace.thrift.TInfo();
- struct.tinfo.read(iprot);
- struct.setTinfoIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // CREDENTIALS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.credentials = new org.apache.accumulo.core.security.thrift.TCredentials();
- struct.credentials.read(iprot);
- struct.setCredentialsIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 3: // FILE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.file = iprot.readString();
- struct.setFileIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 4: // OFFSET
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.offset = iprot.readI64();
- struct.setOffsetIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 5: // COUNT
- if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
- struct.count = iprot.readI64();
- struct.setCountIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateData_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.tinfo != null) {
- oprot.writeFieldBegin(TINFO_FIELD_DESC);
- struct.tinfo.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.credentials != null) {
- oprot.writeFieldBegin(CREDENTIALS_FIELD_DESC);
- struct.credentials.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.file != null) {
- oprot.writeFieldBegin(FILE_FIELD_DESC);
- oprot.writeString(struct.file);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldBegin(OFFSET_FIELD_DESC);
- oprot.writeI64(struct.offset);
- oprot.writeFieldEnd();
- oprot.writeFieldBegin(COUNT_FIELD_DESC);
- oprot.writeI64(struct.count);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateData_argsTupleSchemeFactory implements SchemeFactory {
- public replicateData_argsTupleScheme getScheme() {
- return new replicateData_argsTupleScheme();
- }
- }
-
- private static class replicateData_argsTupleScheme extends TupleScheme<replicateData_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateData_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetTinfo()) {
- optionals.set(0);
- }
- if (struct.isSetCredentials()) {
- optionals.set(1);
- }
- if (struct.isSetFile()) {
- optionals.set(2);
- }
- if (struct.isSetOffset()) {
- optionals.set(3);
- }
- if (struct.isSetCount()) {
- optionals.set(4);
- }
- oprot.writeBitSet(optionals, 5);
- if (struct.isSetTinfo()) {
- struct.tinfo.write(oprot);
- }
- if (struct.isSetCredentials()) {
- struct.credentials.write(oprot);
- }
- if (struct.isSetFile()) {
- oprot.writeString(struct.file);
- }
- if (struct.isSetOffset()) {
- oprot.writeI64(struct.offset);
- }
- if (struct.isSetCount()) {
- oprot.writeI64(struct.count);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateData_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(5);
- if (incoming.get(0)) {
- struct.tinfo = new org.apache.accumulo.trace.thrift.TInfo();
- struct.tinfo.read(iprot);
- struct.setTinfoIsSet(true);
- }
- if (incoming.get(1)) {
- struct.credentials = new org.apache.accumulo.core.security.thrift.TCredentials();
- struct.credentials.read(iprot);
- struct.setCredentialsIsSet(true);
- }
- if (incoming.get(2)) {
- struct.file = iprot.readString();
- struct.setFileIsSet(true);
- }
- if (incoming.get(3)) {
- struct.offset = iprot.readI64();
- struct.setOffsetIsSet(true);
- }
- if (incoming.get(4)) {
- struct.count = iprot.readI64();
- struct.setCountIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateData_result implements org.apache.thrift.TBase<replicateData_result, replicateData_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateData_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.BOOL, (short)0);
- private static final org.apache.thrift.protocol.TField SEC_FIELD_DESC = new org.apache.thrift.protocol.TField("sec", org.apache.thrift.protocol.TType.STRUCT, (short)1);
- private static final org.apache.thrift.protocol.TField RFE_FIELD_DESC = new org.apache.thrift.protocol.TField("rfe", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateData_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateData_resultTupleSchemeFactory());
- }
-
- public boolean success; // required
- public org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException sec; // required
- public ReplicationFailedException rfe; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- SEC((short)1, "sec"),
- RFE((short)2, "rfe");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // SEC
- return SEC;
- case 2: // RFE
- return RFE;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __SUCCESS_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL)));
- tmpMap.put(_Fields.SEC, new org.apache.thrift.meta_data.FieldMetaData("sec", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- tmpMap.put(_Fields.RFE, new org.apache.thrift.meta_data.FieldMetaData("rfe", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateData_result.class, metaDataMap);
- }
-
- public replicateData_result() {
- }
-
- public replicateData_result(
- boolean success,
- org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException sec,
- ReplicationFailedException rfe)
- {
- this();
- this.success = success;
- setSuccessIsSet(true);
- this.sec = sec;
- this.rfe = rfe;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateData_result(replicateData_result other) {
- __isset_bitfield = other.__isset_bitfield;
- this.success = other.success;
- if (other.isSetSec()) {
- this.sec = new org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException(other.sec);
- }
- if (other.isSetRfe()) {
- this.rfe = new ReplicationFailedException(other.rfe);
- }
- }
-
- public replicateData_result deepCopy() {
- return new replicateData_result(this);
- }
-
- @Override
- public void clear() {
- setSuccessIsSet(false);
- this.success = false;
- this.sec = null;
- this.rfe = null;
- }
-
- public boolean isSuccess() {
- return this.success;
- }
-
- public replicateData_result setSuccess(boolean success) {
- this.success = success;
- setSuccessIsSet(true);
- return this;
- }
-
- public void unsetSuccess() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID);
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID);
- }
-
- public void setSuccessIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value);
- }
-
- public org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException getSec() {
- return this.sec;
- }
-
- public replicateData_result setSec(org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException sec) {
- this.sec = sec;
- return this;
- }
-
- public void unsetSec() {
- this.sec = null;
- }
-
- /** Returns true if field sec is set (has been assigned a value) and false otherwise */
- public boolean isSetSec() {
- return this.sec != null;
- }
-
- public void setSecIsSet(boolean value) {
- if (!value) {
- this.sec = null;
- }
- }
-
- public ReplicationFailedException getRfe() {
- return this.rfe;
- }
-
- public replicateData_result setRfe(ReplicationFailedException rfe) {
- this.rfe = rfe;
- return this;
- }
-
- public void unsetRfe() {
- this.rfe = null;
- }
-
- /** Returns true if field rfe is set (has been assigned a value) and false otherwise */
- public boolean isSetRfe() {
- return this.rfe != null;
- }
-
- public void setRfeIsSet(boolean value) {
- if (!value) {
- this.rfe = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((Boolean)value);
- }
- break;
-
- case SEC:
- if (value == null) {
- unsetSec();
- } else {
- setSec((org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException)value);
- }
- break;
-
- case RFE:
- if (value == null) {
- unsetRfe();
- } else {
- setRfe((ReplicationFailedException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return Boolean.valueOf(isSuccess());
-
- case SEC:
- return getSec();
-
- case RFE:
- return getRfe();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case SEC:
- return isSetSec();
- case RFE:
- return isSetRfe();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateData_result)
- return this.equals((replicateData_result)that);
- return false;
- }
-
- public boolean equals(replicateData_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true;
- boolean that_present_success = true;
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (this.success != that.success)
- return false;
- }
-
- boolean this_present_sec = true && this.isSetSec();
- boolean that_present_sec = true && that.isSetSec();
- if (this_present_sec || that_present_sec) {
- if (!(this_present_sec && that_present_sec))
- return false;
- if (!this.sec.equals(that.sec))
- return false;
- }
-
- boolean this_present_rfe = true && this.isSetRfe();
- boolean that_present_rfe = true && that.isSetRfe();
- if (this_present_rfe || that_present_rfe) {
- if (!(this_present_rfe && that_present_rfe))
- return false;
- if (!this.rfe.equals(that.rfe))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateData_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateData_result typedOther = (replicateData_result)other;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetSec()).compareTo(typedOther.isSetSec());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSec()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.sec, typedOther.sec);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetRfe()).compareTo(typedOther.isSetRfe());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRfe()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.rfe, typedOther.rfe);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateData_result(");
- boolean first = true;
-
- sb.append("success:");
- sb.append(this.success);
- first = false;
- if (!first) sb.append(", ");
- sb.append("sec:");
- if (this.sec == null) {
- sb.append("null");
- } else {
- sb.append(this.sec);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("rfe:");
- if (this.rfe == null) {
- sb.append("null");
- } else {
- sb.append(this.rfe);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateData_resultStandardSchemeFactory implements SchemeFactory {
- public replicateData_resultStandardScheme getScheme() {
- return new replicateData_resultStandardScheme();
- }
- }
-
- private static class replicateData_resultStandardScheme extends StandardScheme<replicateData_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateData_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) {
- struct.success = iprot.readBool();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // SEC
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.sec = new org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException();
- struct.sec.read(iprot);
- struct.setSecIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // RFE
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.rfe = new ReplicationFailedException();
- struct.rfe.read(iprot);
- struct.setRfeIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateData_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.isSetSuccess()) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeBool(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.sec != null) {
- oprot.writeFieldBegin(SEC_FIELD_DESC);
- struct.sec.write(oprot);
- oprot.writeFieldEnd();
- }
- if (struct.rfe != null) {
- oprot.writeFieldBegin(RFE_FIELD_DESC);
- struct.rfe.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateData_resultTupleSchemeFactory implements SchemeFactory {
- public replicateData_resultTupleScheme getScheme() {
- return new replicateData_resultTupleScheme();
- }
- }
-
- private static class replicateData_resultTupleScheme extends TupleScheme<replicateData_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateData_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetSec()) {
- optionals.set(1);
- }
- if (struct.isSetRfe()) {
- optionals.set(2);
- }
- oprot.writeBitSet(optionals, 3);
- if (struct.isSetSuccess()) {
- oprot.writeBool(struct.success);
- }
- if (struct.isSetSec()) {
- struct.sec.write(oprot);
- }
- if (struct.isSetRfe()) {
- struct.rfe.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateData_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(3);
- if (incoming.get(0)) {
- struct.success = iprot.readBool();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.sec = new org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException();
- struct.sec.read(iprot);
- struct.setSecIsSet(true);
- }
- if (incoming.get(2)) {
- struct.rfe = new ReplicationFailedException();
- struct.rfe.read(iprot);
- struct.setRfeIsSet(true);
- }
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/core/src/main/thrift/tabletserver.thrift
----------------------------------------------------------------------
diff --git a/core/src/main/thrift/tabletserver.thrift b/core/src/main/thrift/tabletserver.thrift
index 3595262..25e0b10 100644
--- a/core/src/main/thrift/tabletserver.thrift
+++ b/core/src/main/thrift/tabletserver.thrift
@@ -38,10 +38,6 @@ exception ConstraintViolationException {
1:list<data.TConstraintViolationSummary> violationSummaries
}
-exception ReplicationFailedException {
- 1:string reason
-}
-
struct ActionStats {
1:i32 status
2:double elapsed
@@ -203,8 +199,4 @@ service TabletClientService extends client.ClientService {
oneway void removeLogs(1:trace.TInfo tinfo, 2:security.TCredentials credentials, 3:list<string> filenames)
}
-service TabletServerReplicationService {
- bool replicateData(1:trace.TInfo tinfo, 2:security.TCredentials credentials, 3:string file, 4:i64 offset, 5:i64 count) throws (1:client.ThriftSecurityException sec, 2:ReplicationFailedException rfe)
-}
-
typedef i32 TabletID
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 4c705cd..5e07a70 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -222,6 +222,7 @@ import org.apache.accumulo.tserver.metrics.TabletServerMBean;
import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
+import org.apache.accumulo.tserver.replication.ReplicationWorker;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileSystem;
@@ -257,6 +258,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
private ServerConfiguration serverConfig;
private LogSorter logSorter = null;
+ private ReplicationWorker replWorker = null;
public TabletServer(ServerConfiguration conf, VolumeManager fs) {
super();
@@ -265,6 +267,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
this.fs = fs;
AccumuloConfiguration aconf = getSystemConfiguration();
this.logSorter = new LogSorter(instance, fs, aconf);
+ this.replWorker = new ReplicationWorker(instance, fs, aconf);
SimpleTimer.getInstance(aconf).schedule(new Runnable() {
@Override
public void run() {
@@ -3201,6 +3204,10 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException(ex);
}
+ ThreadPoolExecutor replicationThreadPool = new SimpleThreadPool(getSystemConfiguration().getCount(Property.REPLICATION_WORKER_THREADS), "replication task");
+ replWorker.setExecutor(replicationThreadPool);
+ replWorker.run();
+
try {
OBJECT_NAME = new ObjectName("accumulo.server.metrics:service=TServerInfo,name=TabletServerMBean,instance=" + Thread.currentThread().getName());
// Do this because interface not in same package.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
new file mode 100644
index 0000000..82d7ae9
--- /dev/null
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
+
+/**
+ *
+ */
+public class ReplicationProcessor implements Processor {
+
+ @Override
+ public ReplicationProcessor newProcessor() {
+ return new ReplicationProcessor();
+ }
+
+ @Override
+ public void process(String workID, byte[] data) {
+ // TODO Auto-generated method stub
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
new file mode 100644
index 0000000..627e51a
--- /dev/null
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
+import java.util.concurrent.ThreadPoolExecutor;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.zookeeper.KeeperException;
+
+/**
+ *
+ */
+public class ReplicationWorker implements Runnable {
+
+ private Instance inst;
+ private VolumeManager fs;
+ private AccumuloConfiguration conf;
+ private ThreadPoolExecutor executor;
+
+ public ReplicationWorker(Instance inst, VolumeManager fs, AccumuloConfiguration conf) {
+ this.inst = inst;
+ this.fs = fs;
+ this.conf = conf;
+ }
+
+ public void setExecutor(ThreadPoolExecutor executor) {
+ this.executor = executor;
+ }
+
+ @Override
+ public void run() {
+ try {
+ new DistributedWorkQueue(ZooUtil.getRoot(inst) + Constants.ZREPLICATION, conf).startProcessing(new ReplicationProcessor(), executor);
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/05de21fb/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/TabletServerReplication.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/TabletServerReplication.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/TabletServerReplication.java
deleted file mode 100644
index b12f699..0000000
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/TabletServerReplication.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.tserver.replication;
-
-import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
-import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.tabletserver.thrift.ReplicationFailedException;
-import org.apache.accumulo.core.tabletserver.thrift.TabletServerReplicationService.Iface;
-import org.apache.accumulo.trace.thrift.TInfo;
-import org.apache.thrift.TException;
-
-/**
- *
- */
-public class TabletServerReplication implements Iface {
-
- @Override
- public boolean replicateData(TInfo tinfo, TCredentials credentials, String file, long offset, long count) throws ThriftSecurityException,
- ReplicationFailedException, TException {
- // TODO Auto-generated method stub
- return false;
- }
-
-}
[39/50] [abbrv] ACCUMULO-1327 converted latex manual to asciidoc
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/high_speed_ingest.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/high_speed_ingest.txt b/docs/src/main/asciidoc/chapters/high_speed_ingest.txt
new file mode 100644
index 0000000..909f4c4
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/high_speed_ingest.txt
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== High-Speed Ingest
+
+Accumulo is often used as part of a larger data processing and storage system. To
+maximize the performance of a parallel system involving Accumulo, the ingestion
+and query components should be designed to provide enough parallelism and
+concurrency to avoid creating bottlenecks for users and other systems writing to
+and reading from Accumulo. There are several ways to achieve high ingest
+performance.
+
+=== Pre-Splitting New Tables
+
+New tables consist of a single tablet by default. As mutations are applied, the table
+grows and splits into multiple tablets which are balanced by the Master across
+TabletServers. This implies that the aggregate ingest rate will be limited to fewer
+servers than are available within the cluster until the table has reached the point
+where there are tablets on every TabletServer.
+
+Pre-splitting a table ensures that there are as many tablets as desired available
+before ingest begins to take advantage of all the parallelism possible with the cluster
+hardware. Tables can be split at any time by using the shell:
+
+ user@myinstance mytable> addsplits -sf /local_splitfile -t mytable
+
+For the purposes of providing parallelism to ingest it is not necessary to create more
+tablets than there are physical machines within the cluster as the aggregate ingest
+rate is a function of the number of physical machines. Note that the aggregate ingest
+rate is still subject to the number of machines running ingest clients, and the
+distribution of rowIDs across the table. The aggregation ingest rate will be
+suboptimal if there are many inserts into a small number of rowIDs.
+
+=== Multiple Ingester Clients
+
+Accumulo is capable of scaling to very high rates of ingest, which is dependent upon
+not just the number of TabletServers in operation but also the number of ingest
+clients. This is because a single client, while capable of batching mutations and
+sending them to all TabletServers, is ultimately limited by the amount of data that
+can be processed on a single machine. The aggregate ingest rate will scale linearly
+with the number of clients up to the point at which either the aggregate I/O of
+TabletServers or total network bandwidth capacity is reached.
+
+In operational settings where high rates of ingest are paramount, clusters are often
+configured to dedicate some number of machines solely to running Ingester Clients.
+The exact ratio of clients to TabletServers necessary for optimum ingestion rates
+will vary according to the distribution of resources per machine and by data type.
+
+=== Bulk Ingest
+
+Accumulo supports the ability to import files produced by an external process such
+as MapReduce into an existing table. In some cases it may be faster to load data this
+way rather than via ingesting through clients using BatchWriters. This allows a large
+number of machines to format data the way Accumulo expects. The new files can
+then simply be introduced to Accumulo via a shell command.
+
+To configure MapReduce to format data in preparation for bulk loading, the job
+should be set to use a range partitioner instead of the default hash partitioner. The
+range partitioner uses the split points of the Accumulo table that will receive the
+data. The split points can be obtained from the shell and used by the MapReduce
+RangePartitioner. Note that this is only useful if the existing table is already split
+into multiple tablets.
+
+ user@myinstance mytable> getsplits
+ aa
+ ab
+ ac
+ ...
+ zx
+ zy
+ zz
+
+Run the MapReduce job, using the AccumuloFileOutputFormat to create the files to
+be introduced to Accumulo. Once this is complete, the files can be added to
+Accumulo via the shell:
+
+ user@myinstance mytable> importdirectory /files_dir /failures
+
+Note that the paths referenced are directories within the same HDFS instance over
+which Accumulo is running. Accumulo places any files that failed to be added to the
+second directory specified.
+
+A complete example of using Bulk Ingest can be found at
++accumulo/docs/examples/README.bulkIngest+.
+
+=== Logical Time for Bulk Ingest
+
+Logical time is important for bulk imported data, for which the client code may
+be choosing a timestamp. At bulk import time, the user can choose to enable
+logical time for the set of files being imported. When its enabled, Accumulo
+uses a specialized system iterator to lazily set times in a bulk imported file.
+This mechanism guarantees that times set by unsynchronized multi-node
+applications (such as those running on MapReduce) will maintain some semblance
+of causal ordering. This mitigates the problem of the time being wrong on the
+system that created the file for bulk import. These times are not set when the
+file is imported, but whenever it is read by scans or compactions. At import, a
+time is obtained and always used by the specialized system iterator to set that
+time.
+
+The timestamp assigned by Accumulo will be the same for every key in the file.
+This could cause problems if the file contains multiple keys that are identical
+except for the timestamp. In this case, the sort order of the keys will be
+undefined. This could occur if an insert and an update were in the same bulk
+import file.
+
+=== MapReduce Ingest
+It is possible to efficiently write many mutations to Accumulo in parallel via a
+MapReduce job. In this scenario the MapReduce is written to process data that lives
+in HDFS and write mutations to Accumulo using the AccumuloOutputFormat. See
+the MapReduce section under Analytics for details.
+
+An example of using MapReduce can be found under
++accumulo/docs/examples/README.mapred+.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/introduction.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/introduction.txt b/docs/src/main/asciidoc/chapters/introduction.txt
new file mode 100644
index 0000000..1b964b4
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/introduction.txt
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Introduction
+Apache Accumulo is a highly scalable structured store based on Google's BigTable.
+Accumulo is written in Java and operates over the Hadoop Distributed File System
+(HDFS), which is part of the popular Apache Hadoop project. Accumulo supports
+efficient storage and retrieval of structured data, including queries for ranges, and
+provides support for using Accumulo tables as input and output for MapReduce
+jobs.
+
+Accumulo features automatic load-balancing and partitioning, data compression
+and fine-grained security labels.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/multivolume.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/multivolume.txt b/docs/src/main/asciidoc/chapters/multivolume.txt
new file mode 100644
index 0000000..8921f2d
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/multivolume.txt
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Multi-Volume Installations
+
+This is an advanced configuration setting for very large clusters
+under a lot of write pressure.
+
+The HDFS NameNode holds all of the metadata about the files in
+HDFS. For fast performance, all of this information needs to be stored
+in memory. A single NameNode with 64G of memory can store the
+metadata for tens of millions of files.However, when scaling beyond a
+thousand nodes, an active Accumulo system can generate lots of updates
+to the file system, especially when data is being ingested. The large
+number of write transactions to the NameNode, and the speed of a
+single edit log, can become the limiting factor for large scale
+Accumulo installations.
+
+You can see the effect of slow write transactions when the Accumulo
+Garbage Collector takes a long time (more than 5 minutes) to delete
+the files Accumulo no longer needs. If your Garbage Collector
+routinely runs in less than a minute, the NameNode is performing well.
+
+However, if you do begin to experience slow-down and poor GC
+performance, Accumulo can be configured to use multiple NameNode
+servers. The configuration ``instance.volumes'' should be set to a
+comma-separated list, using full URI references to different NameNode
+servers:
+
+[source,xml]
+<property>
+ <name>instance.volumes</name>
+ <value>hdfs://ns1:9001,hdfs://ns2:9001</value>
+</property>
+
+The introduction of multiple volume support in 1.6 changed the way Accumulo
+stores pointers to files. It now stores fully qualified URI references to
+files. Before 1.6, Accumulo stored paths that were relative to a table
+directory. After an upgrade these relative paths will still exist and are
+resolved using instance.dfs.dir, instance.dfs.uri, and Hadoop configuration in
+the same way they were before 1.6.
+
+If the URI for a namenode changes (e.g. namenode was running on host1 and its
+moved to host2), then Accumulo will no longer function. Even if Hadoop and
+Accumulo configurations are changed, the fully qualified URIs stored in
+Accumulo will still contain the old URI. To handle this Accumulo has the
+following configuration property for replacing URI stored in its metadata. The
+example configuration below will replace ns1 with nsA and ns2 with nsB in
+Accumulo metadata. For this property to take affect, Accumulo will need to be
+restarted.
+
+[source,xml]
+<property>
+ <name>instance.volumes.replacements</name>
+ <value>hdfs://ns1:9001 hdfs://nsA:9001, hdfs://ns2:9001 hdfs://nsB:9001</value>
+</property>
+
+Using viewfs or HA namenode, introduced in Hadoop 2, offers another option for
+managing the fully qualified URIs stored in Accumulo. Viewfs and HA namenode
+both introduce a level of indirection in the Hadoop configuration. For
+example assume viewfs:///nn1 maps to hdfs://nn1 in the Hadoop configuration.
+If viewfs://nn1 is used by Accumulo, then its easy to map viewfs://nn1 to
+hdfs://nnA by changing the Hadoop configuration w/o doing anything to Accumulo.
+A production system should probably use a HA namenode. Viewfs may be useful on
+a test system with a single non HA namenode.
+
+You may also want to configure your cluster to use Federation,
+available in Hadoop 2.0, which allows DataNodes to respond to multiple
+NameNode servers, so you do not have to partition your DataNodes by
+NameNode.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/security.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/security.txt b/docs/src/main/asciidoc/chapters/security.txt
new file mode 100644
index 0000000..98a82a1
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/security.txt
@@ -0,0 +1,176 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Security
+
+Accumulo extends the BigTable data model to implement a security mechanism
+known as cell-level security. Every key-value pair has its own security label, stored
+under the column visibility element of the key, which is used to determine whether
+a given user meets the security requirements to read the value. This enables data of
+various security levels to be stored within the same row, and users of varying
+degrees of access to query the same table, while preserving data confidentiality.
+
+=== Security Label Expressions
+
+When mutations are applied, users can specify a security label for each value. This is
+done as the Mutation is created by passing a ColumnVisibility object to the put()
+method:
+
+[source,java]
+----
+Text rowID = new Text("row1");
+Text colFam = new Text("myColFam");
+Text colQual = new Text("myColQual");
+ColumnVisibility colVis = new ColumnVisibility("public");
+long timestamp = System.currentTimeMillis();
+
+Value value = new Value("myValue");
+
+Mutation mutation = new Mutation(rowID);
+mutation.put(colFam, colQual, colVis, timestamp, value);
+----
+
+=== Security Label Expression Syntax
+
+Security labels consist of a set of user-defined tokens that are required to read the
+value the label is associated with. The set of tokens required can be specified using
+syntax that supports logical AND and OR combinations of tokens, as well as nesting
+groups of tokens together.
+
+For example, suppose within our organization we want to label our data values with
+security labels defined in terms of user roles. We might have tokens such as:
+
+ admin
+ audit
+ system
+
+These can be specified alone or combined using logical operators:
+
+----
+// Users must have admin privileges
+admin
+
+// Users must have admin and audit privileges
+admin&audit
+
+// Users with either admin or audit privileges
+admin|audit
+
+// Users must have audit and one or both of admin or system
+(admin|system)&audit
+----
+
+When both +|+ and +&+ operators are used, parentheses must be used to specify
+precedence of the operators.
+
+=== Authorization
+
+When clients attempt to read data from Accumulo, any security labels present are
+examined against the set of authorizations passed by the client code when the
+Scanner or BatchScanner are created. If the authorizations are determined to be
+insufficient to satisfy the security label, the value is suppressed from the set of
+results sent back to the client.
+
+Authorizations are specified as a comma-separated list of tokens the user possesses:
+
+[source,java]
+----
+// user possesses both admin and system level access
+Authorization auths = new Authorization("admin","system");
+
+Scanner s = connector.createScanner("table", auths);
+----
+
+=== User Authorizations
+
+Each Accumulo user has a set of associated security labels. To manipulate
+these in the shell while using the default authorizor, use the setuaths and getauths commands.
+These may also be modified for the default authorizor using the java security operations API.
+
+When a user creates a scanner a set of Authorizations is passed. If the
+authorizations passed to the scanner are not a subset of the users
+authorizations, then an exception will be thrown.
+
+To prevent users from writing data they can not read, add the visibility
+constraint to a table. Use the -evc option in the createtable shell command to
+enable this constraint. For existing tables use the following shell command to
+enable the visibility constraint. Ensure the constraint number does not
+conflict with any existing constraints.
+
+ config -t table -s table.constraint.1=org.apache.accumulo.core.security.VisibilityConstraint
+
+Any user with the alter table permission can add or remove this constraint.
+This constraint is not applied to bulk imported data, if this a concern then
+disable the bulk import permission.
+
+=== Pluggable Security
+
+New in 1.5 of Accumulo is a pluggable security mechanism. It can be broken into three actions --
+authentication, authorization, and permission handling. By default all of these are handled in
+Zookeeper, which is how things were handled in Accumulo 1.4 and before. It is worth noting at this
+point, that it is a new feature in 1.5 and may be adjusted in future releases without the standard
+deprecation cycle.
+
+Authentication simply handles the ability for a user to verify their integrity. A combination of
+principal and authentication token are used to verify a user is who they say they are. An
+authentication token should be constructed, either directly through its constructor, but it is
+advised to use the +init(Property)+ method to populate an authentication token. It is expected that a
+user knows what the appropriate token to use for their system is. The default token is
++PasswordToken+.
+
+Once a user is authenticated by the Authenticator, the user has access to the other actions within
+Accumulo. All actions in Accumulo are ACLed, and this ACL check is handled by the Permission
+Handler. This is what manages all of the permissions, which are divided in system and per table
+level. From there, if a user is doing an action which requires authorizations, the Authorizor is
+queried to determine what authorizations the user has.
+
+This setup allows a variety of different mechanisms to be used for handling different aspects of
+Accumulo's security. A system like Kerberos can be used for authentication, then a system like LDAP
+could be used to determine if a user has a specific permission, and then it may default back to the
+default ZookeeperAuthorizor to determine what Authorizations a user is ultimately allowed to use.
+This is a pluggable system so custom components can be created depending on your need.
+
+=== Secure Authorizations Handling
+
+For applications serving many users, it is not expected that an Accumulo user
+will be created for each application user. In this case an Accumulo user with
+all authorizations needed by any of the applications users must be created. To
+service queries, the application should create a scanner with the application
+user's authorizations. These authorizations could be obtained from a trusted 3rd
+party.
+
+Often production systems will integrate with Public-Key Infrastructure (PKI) and
+designate client code within the query layer to negotiate with PKI servers in order
+to authenticate users and retrieve their authorization tokens (credentials). This
+requires users to specify only the information necessary to authenticate themselves
+to the system. Once user identity is established, their credentials can be accessed by
+the client code and passed to Accumulo outside of the reach of the user.
+
+=== Query Services Layer
+
+Since the primary method of interaction with Accumulo is through the Java API,
+production environments often call for the implementation of a Query layer. This
+can be done using web services in containers such as Apache Tomcat, but is not a
+requirement. The Query Services Layer provides a mechanism for providing a
+platform on which user facing applications can be built. This allows the application
+designers to isolate potentially complex query logic, and enables a convenient point
+at which to perform essential security functions.
+
+Several production environments choose to implement authentication at this layer,
+where users identifiers are used to retrieve their access credentials which are then
+cached within the query layer and presented to Accumulo through the
+Authorizations mechanism.
+
+Typically, the query services layer sits between Accumulo and user workstations.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/shell.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/shell.txt b/docs/src/main/asciidoc/chapters/shell.txt
new file mode 100644
index 0000000..7afcd7d
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/shell.txt
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Accumulo Shell
+Accumulo provides a simple shell that can be used to examine the contents and
+configuration settings of tables, insert/update/delete values, and change
+configuration settings.
+
+The shell can be started by the following command:
+
+ $ACCUMULO_HOME/bin/accumulo shell -u [username]
+
+The shell will prompt for the corresponding password to the username specified
+and then display the following prompt:
+
+ Shell - Apache Accumulo Interactive Shell
+ -
+ - version 1.6
+ - instance name: myinstance
+ - instance id: 00000000-0000-0000-0000-000000000000
+ -
+ - type 'help' for a list of available commands
+ -
+
+=== Basic Administration
+
+The Accumulo shell can be used to create and delete tables, as well as to configure
+table and instance specific options.
+
+----
+root@myinstance> tables
+accumulo.metadata
+accumulo.root
+
+root@myinstance> createtable mytable
+
+root@myinstance mytable>
+
+root@myinstance mytable> tables
+accumulo.metadata
+accumulo.root
+mytable
+
+root@myinstance mytable> createtable testtable
+
+root@myinstance testtable>
+
+root@myinstance testtable> deletetable testtable
+deletetable { testtable } (yes|no)? yes
+Table: [testtable] has been deleted.
+
+root@myinstance>
+----
+
+The Shell can also be used to insert updates and scan tables. This is useful for
+inspecting tables.
+
+----
+root@myinstance mytable> scan
+
+root@myinstance mytable> insert row1 colf colq value1
+insert successful
+
+root@myinstance mytable> scan
+row1 colf:colq [] value1
+----
+
+The value in brackets ``[]'' would be the visibility labels. Since none were used, this is empty for this row.
+You can use the +-st+ option to scan to see the timestamp for the cell, too.
+
+=== Table Maintenance
+
+The *compact* command instructs Accumulo to schedule a compaction of the table during which
+files are consolidated and deleted entries are removed.
+
+ root@myinstance mytable> compact -t mytable
+ 07 16:13:53,201 [shell.Shell] INFO : Compaction of table mytable started for given range
+
+The *flush* command instructs Accumulo to write all entries currently in memory for a given table
+to disk.
+
+ root@myinstance mytable> flush -t mytable
+ 07 16:14:19,351 [shell.Shell] INFO : Flush of table mytable
+ initiated...
+
+=== User Administration
+
+The Shell can be used to add, remove, and grant privileges to users.
+
+----
+root@myinstance mytable> createuser bob
+Enter new password for 'bob': *********
+Please confirm new password for 'bob': *********
+
+root@myinstance mytable> authenticate bob
+Enter current password for 'bob': *********
+Valid
+
+root@myinstance mytable> grant System.CREATE_TABLE -s -u bob
+
+root@myinstance mytable> user bob
+Enter current password for 'bob': *********
+
+bob@myinstance mytable> userpermissions
+System permissions: System.CREATE_TABLE
+Table permissions (accumulo.metadata): Table.READ
+Table permissions (mytable): NONE
+
+bob@myinstance mytable> createtable bobstable
+
+bob@myinstance bobstable>
+
+bob@myinstance bobstable> user root
+Enter current password for 'root': *********
+
+root@myinstance bobstable> revoke System.CREATE_TABLE -s -u bob
+----
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/table_configuration.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/table_configuration.txt b/docs/src/main/asciidoc/chapters/table_configuration.txt
new file mode 100644
index 0000000..fae8cd3
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/table_configuration.txt
@@ -0,0 +1,635 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Table Configuration
+
+Accumulo tables have a few options that can be configured to alter the default
+behavior of Accumulo as well as improve performance based on the data stored.
+These include locality groups, constraints, bloom filters, iterators, and block
+cache. For a complete list of available configuration options, see <<configuration>>.
+
+=== Locality Groups
+Accumulo supports storing sets of column families separately on disk to allow
+clients to efficiently scan over columns that are frequently used together and to avoid
+scanning over column families that are not requested. After a locality group is set,
+Scanner and BatchScanner operations will automatically take advantage of them
+whenever the fetchColumnFamilies() method is used.
+
+By default, tables place all column families into the same ``default'' locality group.
+Additional locality groups can be configured at any time via the shell or
+programmatically as follows:
+
+==== Managing Locality Groups via the Shell
+
+ usage: setgroups <group>=<col fam>{,<col fam>}{ <group>=<col fam>{,<col fam>}}
+ [-?] -t <table>
+
+ user@myinstance mytable> setgroups group_one=colf1,colf2 -t mytable
+
+ user@myinstance mytable> getgroups -t mytable
+
+==== Managing Locality Groups via the Client API
+
+[source,java]
+----
+Connector conn;
+
+HashMap<String,Set<Text>> localityGroups = new HashMap<String, Set<Text>>();
+
+HashSet<Text> metadataColumns = new HashSet<Text>();
+metadataColumns.add(new Text("domain"));
+metadataColumns.add(new Text("link"));
+
+HashSet<Text> contentColumns = new HashSet<Text>();
+contentColumns.add(new Text("body"));
+contentColumns.add(new Text("images"));
+
+localityGroups.put("metadata", metadataColumns);
+localityGroups.put("content", contentColumns);
+
+conn.tableOperations().setLocalityGroups("mytable", localityGroups);
+
+// existing locality groups can be obtained as follows
+Map<String, Set<Text>> groups =
+ conn.tableOperations().getLocalityGroups("mytable");
+----
+
+The assignment of Column Families to Locality Groups can be changed at any time. The
+physical movement of column families into their new locality groups takes place via
+the periodic Major Compaction process that takes place continuously in the
+background. Major Compaction can also be scheduled to take place immediately
+through the shell:
+
+ user@myinstance mytable> compact -t mytable
+
+=== Constraints
+
+Accumulo supports constraints applied on mutations at insert time. This can be
+used to disallow certain inserts according to a user defined policy. Any mutation
+that fails to meet the requirements of the constraint is rejected and sent back to the
+client.
+
+Constraints can be enabled by setting a table property as follows:
+
+----
+user@myinstance mytable> constraint -t mytable -a com.test.ExampleConstraint com.test.AnotherConstraint
+
+user@myinstance mytable> constraint -l
+com.test.ExampleConstraint=1
+com.test.AnotherConstraint=2
+----
+
+Currently there are no general-purpose constraints provided with the Accumulo
+distribution. New constraints can be created by writing a Java class that implements
+the following interface:
+
+ org.apache.accumulo.core.constraints.Constraint
+
+To deploy a new constraint, create a jar file containing the class implementing the
+new constraint and place it in the lib directory of the Accumulo installation. New
+constraint jars can be added to Accumulo and enabled without restarting but any
+change to an existing constraint class requires Accumulo to be restarted.
+
+An example of constraints can be found in
++accumulo/docs/examples/README.constraints+ with corresponding code under
++accumulo/examples/simple/src/main/java/accumulo/examples/simple/constraints+ .
+
+=== Bloom Filters
+As mutations are applied to an Accumulo table, several files are created per tablet. If
+bloom filters are enabled, Accumulo will create and load a small data structure into
+memory to determine whether a file contains a given key before opening the file.
+This can speed up lookups considerably.
+
+To enable bloom filters, enter the following command in the Shell:
+
+ user@myinstance> config -t mytable -s table.bloom.enabled=true
+
+An extensive example of using Bloom Filters can be found at
++accumulo/docs/examples/README.bloom+ .
+
+=== Iterators
+Iterators provide a modular mechanism for adding functionality to be executed by
+TabletServers when scanning or compacting data. This allows users to efficiently
+summarize, filter, and aggregate data. In fact, the built-in features of cell-level
+security and column fetching are implemented using Iterators.
+Some useful Iterators are provided with Accumulo and can be found in the
+*+org.apache.accumulo.core.iterators.user+* package.
+In each case, any custom Iterators must be included in Accumulo's classpath,
+typically by including a jar in +$ACCUMULO_HOME/lib+ or
++$ACCUMULO_HOME/lib/ext+, although the VFS classloader allows for
+classpath manipulation using a variety of schemes including URLs and HDFS URIs.
+
+==== Setting Iterators via the Shell
+
+Iterators can be configured on a table at scan, minor compaction and/or major
+compaction scopes. If the Iterator implements the OptionDescriber interface, the
+setiter command can be used which will interactively prompt the user to provide
+values for the given necessary options.
+
+ usage: setiter [-?] -ageoff | -agg | -class <name> | -regex |
+ -reqvis | -vers [-majc] [-minc] [-n <itername>] -p <pri>
+ [-scan] [-t <table>]
+
+ user@myinstance mytable> setiter -t mytable -scan -p 15 -n myiter -class com.company.MyIterator
+
+The config command can always be used to manually configure iterators which is useful
+in cases where the Iterator does not implement the OptionDescriber interface.
+
+ config -t mytable -s table.iterator.scan.myiter=15,com.company.MyIterator
+ config -t mytable -s table.iterator.minc.myiter=15,com.company.MyIterator
+ config -t mytable -s table.iterator.majc.myiter=15,com.company.MyIterator
+ config -t mytable -s table.iterator.scan.myiter.opt.myoptionname=myoptionvalue
+ config -t mytable -s table.iterator.minc.myiter.opt.myoptionname=myoptionvalue
+ config -t mytable -s table.iterator.majc.myiter.opt.myoptionname=myoptionvalue
+
+==== Setting Iterators Programmatically
+
+[source,java]
+scanner.addIterator(new IteratorSetting(
+ 15, // priority
+ "myiter", // name this iterator
+ "com.company.MyIterator" // class name
+));
+
+Some iterators take additional parameters from client code, as in the following
+example:
+
+[source,java]
+IteratorSetting iter = new IteratorSetting(...);
+iter.addOption("myoptionname", "myoptionvalue");
+scanner.addIterator(iter)
+
+Tables support separate Iterator settings to be applied at scan time, upon minor
+compaction and upon major compaction. For most uses, tables will have identical
+iterator settings for all three to avoid inconsistent results.
+
+==== Versioning Iterators and Timestamps
+
+Accumulo provides the capability to manage versioned data through the use of
+timestamps within the Key. If a timestamp is not specified in the key created by the
+client then the system will set the timestamp to the current time. Two keys with
+identical rowIDs and columns but different timestamps are considered two versions
+of the same key. If two inserts are made into Accumulo with the same rowID,
+column, and timestamp, then the behavior is non-deterministic.
+
+Timestamps are sorted in descending order, so the most recent data comes first.
+Accumulo can be configured to return the top k versions, or versions later than a
+given date. The default is to return the one most recent version.
+
+The version policy can be changed by changing the VersioningIterator options for a
+table as follows:
+
+----
+user@myinstance mytable> config -t mytable -s table.iterator.scan.vers.opt.maxVersions=3
+
+user@myinstance mytable> config -t mytable -s table.iterator.minc.vers.opt.maxVersions=3
+
+user@myinstance mytable> config -t mytable -s table.iterator.majc.vers.opt.maxVersions=3
+----
+
+When a table is created, by default its configured to use the
+VersioningIterator and keep one version. A table can be created without the
+VersioningIterator with the -ndi option in the shell. Also the Java API
+has the following method
+
+[source,java]
+connector.tableOperations.create(String tableName, boolean limitVersion);
+
+
+===== Logical Time
+
+Accumulo 1.2 introduces the concept of logical time. This ensures that timestamps
+set by Accumulo always move forward. This helps avoid problems caused by
+TabletServers that have different time settings. The per tablet counter gives unique
+one up time stamps on a per mutation basis. When using time in milliseconds, if
+two things arrive within the same millisecond then both receive the same
+timestamp. When using time in milliseconds, Accumulo set times will still
+always move forward and never backwards.
+
+A table can be configured to use logical timestamps at creation time as follows:
+
+ user@myinstance> createtable -tl logical
+
+===== Deletes
+Deletes are special keys in Accumulo that get sorted along will all the other data.
+When a delete key is inserted, Accumulo will not show anything that has a
+timestamp less than or equal to the delete key. During major compaction, any keys
+older than a delete key are omitted from the new file created, and the omitted keys
+are removed from disk as part of the regular garbage collection process.
+
+==== Filters
+When scanning over a set of key-value pairs it is possible to apply an arbitrary
+filtering policy through the use of a Filter. Filters are types of iterators that return
+only key-value pairs that satisfy the filter logic. Accumulo has a few built-in filters
+that can be configured on any table: AgeOff, ColumnAgeOff, Timestamp, NoVis, and RegEx. More can be added
+by writing a Java class that extends the
++org.apache.accumulo.core.iterators.Filter+ class.
+
+The AgeOff filter can be configured to remove data older than a certain date or a fixed
+amount of time from the present. The following example sets a table to delete
+everything inserted over 30 seconds ago:
+
+----
+user@myinstance> createtable filtertest
+
+user@myinstance filtertest> setiter -t filtertest -scan -minc -majc -p 10 -n myfilter -ageoff
+AgeOffFilter removes entries with timestamps more than <ttl> milliseconds old
+----------> set org.apache.accumulo.core.iterators.user.AgeOffFilter parameter negate, default false
+ keeps k/v that pass accept method, true rejects k/v that pass accept method:
+----------> set org.apache.accumulo.core.iterators.user.AgeOffFilter parameter ttl, time to
+ live (milliseconds): 3000
+----------> set org.apache.accumulo.core.iterators.user.AgeOffFilter parameter currentTime, if set,
+ use the given value as the absolute time in milliseconds as the current time of day:
+
+user@myinstance filtertest>
+
+user@myinstance filtertest> scan
+
+user@myinstance filtertest> insert foo a b c
+
+user@myinstance filtertest> scan
+foo a:b [] c
+
+user@myinstance filtertest> sleep 4
+
+user@myinstance filtertest> scan
+
+user@myinstance filtertest>
+----
+
+To see the iterator settings for a table, use:
+
+ user@example filtertest> config -t filtertest -f iterator
+ ---------+---------------------------------------------+------------------
+ SCOPE | NAME | VALUE
+ ---------+---------------------------------------------+------------------
+ table | table.iterator.majc.myfilter .............. | 10,org.apache.accumulo.core.iterators.user.AgeOffFilter
+ table | table.iterator.majc.myfilter.opt.ttl ...... | 3000
+ table | table.iterator.majc.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
+ table | table.iterator.majc.vers.opt.maxVersions .. | 1
+ table | table.iterator.minc.myfilter .............. | 10,org.apache.accumulo.core.iterators.user.AgeOffFilter
+ table | table.iterator.minc.myfilter.opt.ttl ...... | 3000
+ table | table.iterator.minc.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
+ table | table.iterator.minc.vers.opt.maxVersions .. | 1
+ table | table.iterator.scan.myfilter .............. | 10,org.apache.accumulo.core.iterators.user.AgeOffFilter
+ table | table.iterator.scan.myfilter.opt.ttl ...... | 3000
+ table | table.iterator.scan.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
+ table | table.iterator.scan.vers.opt.maxVersions .. | 1
+ ---------+---------------------------------------------+------------------
+
+==== Combiners
+
+Accumulo allows Combiners to be configured on tables and column
+families. When a Combiner is set it is applied across the values
+associated with any keys that share rowID, column family, and column qualifier.
+This is similar to the reduce step in MapReduce, which applied some function to all
+the values associated with a particular key.
+
+For example, if a summing combiner were configured on a table and the following
+mutations were inserted:
+
+ Row Family Qualifier Timestamp Value
+ rowID1 colfA colqA 20100101 1
+ rowID1 colfA colqA 20100102 1
+
+The table would reflect only one aggregate value:
+
+ rowID1 colfA colqA - 2
+
+Combiners can be enabled for a table using the setiter command in the shell. Below is an example.
+
+----
+root@a14 perDayCounts> setiter -t perDayCounts -p 10 -scan -minc -majc -n daycount
+ -class org.apache.accumulo.core.iterators.user.SummingCombiner
+TypedValueCombiner can interpret Values as a variety of number encodings
+ (VLong, Long, or String) before combining
+----------> set SummingCombiner parameter columns,
+ <col fam>[:<col qual>]{,<col fam>[:<col qual>]} : day
+----------> set SummingCombiner parameter type, <VARNUM|LONG|STRING>: STRING
+
+root@a14 perDayCounts> insert foo day 20080101 1
+root@a14 perDayCounts> insert foo day 20080101 1
+root@a14 perDayCounts> insert foo day 20080103 1
+root@a14 perDayCounts> insert bar day 20080101 1
+root@a14 perDayCounts> insert bar day 20080101 1
+
+root@a14 perDayCounts> scan
+bar day:20080101 [] 2
+foo day:20080101 [] 2
+foo day:20080103 [] 1
+----
+
+Accumulo includes some useful Combiners out of the box. To find these look in
+the *+org.apache.accumulo.core.iterators.user+* package.
+
+Additional Combiners can be added by creating a Java class that extends
++org.apache.accumulo.core.iterators.Combiner+ and adding a jar containing that
+class to Accumulo's lib/ext directory.
+
+An example of a Combiner can be found under
+
+ accumulo/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java
+
+
+=== Block Cache
+
+In order to increase throughput of commonly accessed entries, Accumulo employs a block cache.
+This block cache buffers data in memory so that it doesn't have to be read off of disk.
+The RFile format that Accumulo prefers is a mix of index blocks and data blocks, where the index blocks are used to find the appropriate data blocks.
+Typical queries to Accumulo result in a binary search over several index blocks followed by a linear scan of one or more data blocks.
+
+The block cache can be configured on a per-table basis, and all tablets hosted on a tablet server share a single resource pool.
+To configure the size of the tablet server's block cache, set the following properties:
+
+ tserver.cache.data.size: Specifies the size of the cache for file data blocks.
+ tserver.cache.index.size: Specifies the size of the cache for file indices.
+
+To enable the block cache for your table, set the following properties:
+
+ table.cache.block.enable: Determines whether file (data) block cache is enabled.
+ table.cache.index.enable: Determines whether index cache is enabled.
+
+The block cache can have a significant effect on alleviating hot spots, as well as reducing query latency.
+It is enabled by default for the metadata tables.
+
+=== Compaction
+
+As data is written to Accumulo it is buffered in memory. The data buffered in
+memory is eventually written to HDFS on a per tablet basis. Files can also be
+added to tablets directly by bulk import. In the background tablet servers run
+major compactions to merge multiple files into one. The tablet server has to
+decide which tablets to compact and which files within a tablet to compact.
+This decision is made using the compaction ratio, which is configurable on a
+per table basis. To configure this ratio modify the following property:
+
+ table.compaction.major.ratio
+
+Increasing this ratio will result in more files per tablet and less compaction
+work. More files per tablet means more higher query latency. So adjusting
+this ratio is a trade off between ingest and query performance. The ratio
+defaults to 3.
+
+The way the ratio works is that a set of files is compacted into one file if the
+sum of the sizes of the files in the set is larger than the ratio multiplied by
+the size of the largest file in the set. If this is not true for the set of all
+files in a tablet, the largest file is removed from consideration, and the
+remaining files are considered for compaction. This is repeated until a
+compaction is triggered or there are no files left to consider.
+
+The number of background threads tablet servers use to run major compactions is
+configurable. To configure this modify the following property:
+
+ tserver.compaction.major.concurrent.max
+
+Also, the number of threads tablet servers use for minor compactions is
+configurable. To configure this modify the following property:
+
+ tserver.compaction.minor.concurrent.max
+
+The numbers of minor and major compactions running and queued is visible on the
+Accumulo monitor page. This allows you to see if compactions are backing up
+and adjustments to the above settings are needed. When adjusting the number of
+threads available for compactions, consider the number of cores and other tasks
+running on the nodes such as maps and reduces.
+
+If major compactions are not keeping up, then the number of files per tablet
+will grow to a point such that query performance starts to suffer. One way to
+handle this situation is to increase the compaction ratio. For example, if the
+compaction ratio were set to 1, then every new file added to a tablet by minor
+compaction would immediately queue the tablet for major compaction. So if a
+tablet has a 200M file and minor compaction writes a 1M file, then the major
+compaction will attempt to merge the 200M and 1M file. If the tablet server
+has lots of tablets trying to do this sort of thing, then major compactions
+will back up and the number of files per tablet will start to grow, assuming
+data is being continuously written. Increasing the compaction ratio will
+alleviate backups by lowering the amount of major compaction work that needs to
+be done.
+
+Another option to deal with the files per tablet growing too large is to adjust
+the following property:
+
+ table.file.max
+
+When a tablet reaches this number of files and needs to flush its in-memory
+data to disk, it will choose to do a merging minor compaction. A merging minor
+compaction will merge the tablet's smallest file with the data in memory at
+minor compaction time. Therefore the number of files will not grow beyond this
+limit. This will make minor compactions take longer, which will cause ingest
+performance to decrease. This can cause ingest to slow down until major
+compactions have enough time to catch up. When adjusting this property, also
+consider adjusting the compaction ratio. Ideally, merging minor compactions
+never need to occur and major compactions will keep up. It is possible to
+configure the file max and compaction ratio such that only merging minor
+compactions occur and major compactions never occur. This should be avoided
+because doing only merging minor compactions causes O(_N_^2^) work to be done.
+The amount of work done by major compactions is O(_N_*log~_R_~(_N_)) where
+_R_ is the compaction ratio.
+
+Compactions can be initiated manually for a table. To initiate a minor
+compaction, use the flush command in the shell. To initiate a major compaction,
+use the compact command in the shell. The compact command will compact all
+tablets in a table to one file. Even tablets with one file are compacted. This
+is useful for the case where a major compaction filter is configured for a
+table. In 1.4 the ability to compact a range of a table was added. To use this
+feature specify start and stop rows for the compact command. This will only
+compact tablets that overlap the given row range.
+
+=== Pre-splitting tables
+
+Accumulo will balance and distribute tables across servers. Before a
+table gets large, it will be maintained as a single tablet on a single
+server. This limits the speed at which data can be added or queried
+to the speed of a single node. To improve performance when the a table
+is new, or small, you can add split points and generate new tablets.
+
+In the shell:
+
+ root@myinstance> createtable newTable
+ root@myinstance> addsplits -t newTable g n t
+
+This will create a new table with 4 tablets. The table will be split
+on the letters ``g'', ``n'', and ``t'' which will work nicely if the
+row data start with lower-case alphabetic characters. If your row
+data includes binary information or numeric information, or if the
+distribution of the row information is not flat, then you would pick
+different split points. Now ingest and query can proceed on 4 nodes
+which can improve performance.
+
+=== Merging tablets
+
+Over time, a table can get very large, so large that it has hundreds
+of thousands of split points. Once there are enough tablets to spread
+a table across the entire cluster, additional splits may not improve
+performance, and may create unnecessary bookkeeping. The distribution
+of data may change over time. For example, if row data contains date
+information, and data is continually added and removed to maintain a
+window of current information, tablets for older rows may be empty.
+
+Accumulo supports tablet merging, which can be used to reduce
+the number of split points. The following command will merge all rows
+from ``A'' to ``Z'' into a single tablet:
+
+ root@myinstance> merge -t myTable -s A -e Z
+
+If the result of a merge produces a tablet that is larger than the
+configured split size, the tablet may be split by the tablet server.
+Be sure to increase your tablet size prior to any merges if the goal
+is to have larger tablets:
+
+ root@myinstance> config -t myTable -s table.split.threshold=2G
+
+In order to merge small tablets, you can ask Accumulo to merge
+sections of a table smaller than a given size.
+
+ root@myinstance> merge -t myTable -s 100M
+
+By default, small tablets will not be merged into tablets that are
+already larger than the given size. This can leave isolated small
+tablets. To force small tablets to be merged into larger tablets use
+the +--force+ option:
+
+ root@myinstance> merge -t myTable -s 100M --force
+
+Merging away small tablets works on one section at a time. If your
+table contains many sections of small split points, or you are
+attempting to change the split size of the entire table, it will be
+faster to set the split point and merge the entire table:
+
+ root@myinstance> config -t myTable -s table.split.threshold=256M
+ root@myinstance> merge -t myTable
+
+=== Delete Range
+
+Consider an indexing scheme that uses date information in each row.
+For example ``20110823-15:20:25.013'' might be a row that specifies a
+date and time. In some cases, we might like to delete rows based on
+this date, say to remove all the data older than the current year.
+Accumulo supports a delete range operation which efficiently
+removes data between two rows. For example:
+
+ root@myinstance> deleterange -t myTable -s 2010 -e 2011
+
+This will delete all rows starting with ``2010'' and it will stop at
+any row starting ``2011''. You can delete any data prior to 2011
+with:
+
+ root@myinstance> deleterange -t myTable -e 2011 --force
+
+The shell will not allow you to delete an unbounded range (no start)
+unless you provide the +--force+ option.
+
+Range deletion is implemented using splits at the given start/end
+positions, and will affect the number of splits in the table.
+
+=== Cloning Tables
+
+A new table can be created that points to an existing table's data. This is a
+very quick metadata operation, no data is actually copied. The cloned table
+and the source table can change independently after the clone operation. One
+use case for this feature is testing. For example to test a new filtering
+iterator, clone the table, add the filter to the clone, and force a major
+compaction. To perform a test on less data, clone a table and then use delete
+range to efficiently remove a lot of data from the clone. Another use case is
+generating a snapshot to guard against human error. To create a snapshot,
+clone a table and then disable write permissions on the clone.
+
+The clone operation will point to the source table's files. This is why the
+flush option is present and is enabled by default in the shell. If the flush
+option is not enabled, then any data the source table currently has in memory
+will not exist in the clone.
+
+A cloned table copies the configuration of the source table. However the
+permissions of the source table are not copied to the clone. After a clone is
+created, only the user that created the clone can read and write to it.
+
+In the following example we see that data inserted after the clone operation is
+not visible in the clone.
+
+----
+root@a14> createtable people
+
+root@a14 people> insert 890435 name last Doe
+root@a14 people> insert 890435 name first John
+
+root@a14 people> clonetable people test
+
+root@a14 people> insert 890436 name first Jane
+root@a14 people> insert 890436 name last Doe
+
+root@a14 people> scan
+890435 name:first [] John
+890435 name:last [] Doe
+890436 name:first [] Jane
+890436 name:last [] Doe
+
+root@a14 people> table test
+
+root@a14 test> scan
+890435 name:first [] John
+890435 name:last [] Doe
+
+root@a14 test>
+----
+
+The du command in the shell shows how much space a table is using in HDFS.
+This command can also show how much overlapping space two cloned tables have in
+HDFS. In the example below du shows table ci is using 428M. Then ci is cloned
+to cic and du shows that both tables share 428M. After three entries are
+inserted into cic and its flushed, du shows the two tables still share 428M but
+cic has 226 bytes to itself. Finally, table cic is compacted and then du shows
+that each table uses 428M.
+
+----
+root@a14> du ci
+ 428,482,573 [ci]
+
+root@a14> clonetable ci cic
+
+root@a14> du ci cic
+ 428,482,573 [ci, cic]
+
+root@a14> table cic
+
+root@a14 cic> insert r1 cf1 cq1 v1
+root@a14 cic> insert r1 cf1 cq2 v2
+root@a14 cic> insert r1 cf1 cq3 v3
+
+root@a14 cic> flush -t cic -w
+27 15:00:13,908 [shell.Shell] INFO : Flush of table cic completed.
+
+root@a14 cic> du ci cic
+ 428,482,573 [ci, cic]
+ 226 [cic]
+
+root@a14 cic> compact -t cic -w
+27 15:00:35,871 [shell.Shell] INFO : Compacting table ...
+27 15:03:03,303 [shell.Shell] INFO : Compaction of table cic completed for given range
+
+root@a14 cic> du ci cic
+ 428,482,573 [ci]
+ 428,482,612 [cic]
+
+root@a14 cic>
+----
+
+=== Exporting Tables
+
+Accumulo supports exporting tables for the purpose of copying tables to another
+cluster. Exporting and importing tables preserves the tables configuration,
+splits, and logical time. Tables are exported and then copied via the hadoop
+distcp command. To export a table, it must be offline and stay offline while
+discp runs. The reason it needs to stay offline is to prevent files from being
+deleted. A table can be cloned and the clone taken offline inorder to avoid
+losing access to the table. See +docs/examples/README.export+ for an example.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/table_design.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/table_design.txt b/docs/src/main/asciidoc/chapters/table_design.txt
new file mode 100644
index 0000000..31fa49a
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/table_design.txt
@@ -0,0 +1,336 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Table Design
+
+=== Basic Table
+
+Since Accumulo tables are sorted by row ID, each table can be thought of as being
+indexed by the row ID. Lookups performed by row ID can be executed quickly, by doing
+a binary search, first across the tablets, and then within a tablet. Clients should
+choose a row ID carefully in order to support their desired application. A simple rule
+is to select a unique identifier as the row ID for each entity to be stored and assign
+all the other attributes to be tracked to be columns under this row ID. For example,
+if we have the following data in a comma-separated file:
+
+ userid,age,address,account-balance
+
+We might choose to store this data using the userid as the rowID, the column
+name in the column family, and a blank column qualifier:
+
+[source,java]
+----
+Mutation m = new Mutation(userid);
+final String column_qualifier = "";
+m.put("age", column_qualifier, age);
+m.put("address", column_qualifier, address);
+m.put("balance", column_qualifier, account_balance);
+
+writer.add(m);
+----
+
+We could then retrieve any of the columns for a specific userid by specifying the
+userid as the range of a scanner and fetching specific columns:
+
+[source,java]
+----
+Range r = new Range(userid, userid); // single row
+Scanner s = conn.createScanner("userdata", auths);
+s.setRange(r);
+s.fetchColumnFamily(new Text("age"));
+
+for(Entry<Key,Value> entry : s) {
+ System.out.println(entry.getValue().toString());
+}
+----
+
+=== RowID Design
+
+Often it is necessary to transform the rowID in order to have rows ordered in a way
+that is optimal for anticipated access patterns. A good example of this is reversing
+the order of components of internet domain names in order to group rows of the
+same parent domain together:
+
+ com.google.code
+ com.google.labs
+ com.google.mail
+ com.yahoo.mail
+ com.yahoo.research
+
+Some data may result in the creation of very large rows - rows with many columns.
+In this case the table designer may wish to split up these rows for better load
+balancing while keeping them sorted together for scanning purposes. This can be
+done by appending a random substring at the end of the row:
+
+ com.google.code_00
+ com.google.code_01
+ com.google.code_02
+ com.google.labs_00
+ com.google.mail_00
+ com.google.mail_01
+
+It could also be done by adding a string representation of some period of time such as date to the week
+or month:
+
+ com.google.code_201003
+ com.google.code_201004
+ com.google.code_201005
+ com.google.labs_201003
+ com.google.mail_201003
+ com.google.mail_201004
+
+Appending dates provides the additional capability of restricting a scan to a given
+date range.
+
+=== Lexicoders
+Since Keys in Accumulo are sorted lexicographically by default, it's often useful to encode
+common data types into a byte format in which their sort order corresponds to the sort order
+in their native form. An example of this is encoding dates and numerical data so that they can
+be better seeked or searched in ranges.
+
+The lexicoders are a standard and extensible way of encoding Java types. Here's an example
+of a lexicoder that encodes a java Date object so that it sorts lexicographically:
+
+[source,java]
+----
+// create new date lexicoder
+DateLexicoder dateEncoder = new DateLexicoder();
+
+// truncate time to hours
+long epoch = System.currentTimeMillis();
+Date hour = new Date(epoch - (epoch % 3600000));
+
+// encode the rowId so that it is sorted lexicographically
+Mutation mutation = new Mutation(dateEncoder.encode(hour));
+mutation.put(new Text("colf"), new Text("colq"), new Value(new byte[]{}));
+----
+
+If we want to return the most recent date first, we can reverse the sort order
+with the reverse lexicoder:
+
+[source,java]
+----
+// create new date lexicoder and reverse lexicoder
+DateLexicoder dateEncoder = new DateLexicoder();
+ReverseLexicoder reverseEncoder = new ReverseLexicoder(dateEncoder);
+
+// truncate date to hours
+long epoch = System.currentTimeMillis();
+Date hour = new Date(epoch - (epoch % 3600000));
+
+// encode the rowId so that it sorts in reverse lexicographic order
+Mutation mutation = new Mutation(reverseEncoder.encode(hour));
+mutation.put(new Text("colf"), new Text("colq"), new Value(new byte[]{}));
+----
+
+
+=== Indexing
+In order to support lookups via more than one attribute of an entity, additional
+indexes can be built. However, because Accumulo tables can support any number of
+columns without specifying them beforehand, a single additional index will often
+suffice for supporting lookups of records in the main table. Here, the index has, as
+the rowID, the Value or Term from the main table, the column families are the same,
+and the column qualifier of the index table contains the rowID from the main table.
+
+[width="75%",cols="^,^,^,^"]
+[grid="rows"]
+[options="header"]
+|=============================================
+|RowID |Column Family |Column Qualifier |Value
+|Term |Field Name |MainRowID |
+|=============================================
+
+Note: We store rowIDs in the column qualifier rather than the Value so that we can
+have more than one rowID associated with a particular term within the index. If we
+stored this in the Value we would only see one of the rows in which the value
+appears since Accumulo is configured by default to return the one most recent
+value associated with a key.
+
+Lookups can then be done by scanning the Index Table first for occurrences of the
+desired values in the columns specified, which returns a list of row ID from the main
+table. These can then be used to retrieve each matching record, in their entirety, or a
+subset of their columns, from the Main Table.
+
+To support efficient lookups of multiple rowIDs from the same table, the Accumulo
+client library provides a BatchScanner. Users specify a set of Ranges to the
+BatchScanner, which performs the lookups in multiple threads to multiple servers
+and returns an Iterator over all the rows retrieved. The rows returned are NOT in
+sorted order, as is the case with the basic Scanner interface.
+
+[source,java]
+----
+// first we scan the index for IDs of rows matching our query
+Text term = new Text("mySearchTerm");
+
+HashSet<Range> matchingRows = new HashSet<Range>();
+
+Scanner indexScanner = createScanner("index", auths);
+indexScanner.setRange(new Range(term, term));
+
+// we retrieve the matching rowIDs and create a set of ranges
+for(Entry<Key,Value> entry : indexScanner) {
+ matchingRows.add(new Range(entry.getKey().getColumnQualifier()));
+}
+
+// now we pass the set of rowIDs to the batch scanner to retrieve them
+BatchScanner bscan = conn.createBatchScanner("table", auths, 10);
+bscan.setRanges(matchingRows);
+bscan.fetchColumnFamily(new Text("attributes"));
+
+for(Entry<Key,Value> entry : bscan) {
+ System.out.println(entry.getValue());
+}
+----
+
+One advantage of the dynamic schema capabilities of Accumulo is that different
+fields may be indexed into the same physical table. However, it may be necessary to
+create different index tables if the terms must be formatted differently in order to
+maintain proper sort order. For example, real numbers must be formatted
+differently than their usual notation in order to be sorted correctly. In these cases,
+usually one index per unique data type will suffice.
+
+=== Entity-Attribute and Graph Tables
+
+Accumulo is ideal for storing entities and their attributes, especially of the
+attributes are sparse. It is often useful to join several datasets together on common
+entities within the same table. This can allow for the representation of graphs,
+including nodes, their attributes, and connections to other nodes.
+
+Rather than storing individual events, Entity-Attribute or Graph tables store
+aggregate information about the entities involved in the events and the
+relationships between entities. This is often preferrable when single events aren't
+very useful and when a continuously updated summarization is desired.
+
+The physical schema for an entity-attribute or graph table is as follows:
+
+[width="75%",cols="^,^,^,^"]
+[grid="rows"]
+[options="header"]
+|==================================================
+|RowID |Column Family |Column Qualifier |Value
+|EntityID |Attribute Name |Attribute Value |Weight
+|EntityID |Edge Type |Related EntityID |Weight
+|==================================================
+
+For example, to keep track of employees, managers and products the following
+entity-attribute table could be used. Note that the weights are not always necessary
+and are set to 0 when not used.
+
+[width="75%",cols="^,^,^,^"]
+[grid="rows"]
+[options="header"]
+|=============================================
+|RowID |Column Family |Column Qualifier |Value
+| E001 | name | bob | 0
+| E001 | department | sales | 0
+| E001 | hire_date | 20030102 | 0
+| E001 | units_sold | P001 | 780
+| E002 | name | george | 0
+| E002 | department | sales | 0
+| E002 | manager_of | E001 | 0
+| E002 | manager_of | E003 | 0
+| E003 | name | harry | 0
+| E003 | department | accounts_recv | 0
+| E003 | hire_date | 20000405 | 0
+| E003 | units_sold | P002 | 566
+| E003 | units_sold | P001 | 232
+| P001 | product_name | nike_airs | 0
+| P001 | product_type | shoe | 0
+| P001 | in_stock | germany | 900
+| P001 | in_stock | brazil | 200
+| P002 | product_name | basic_jacket | 0
+| P002 | product_type | clothing | 0
+| P002 | in_stock | usa | 3454
+| P002 | in_stock | germany | 700
+|=============================================
+
+To allow efficient updating of edge weights, an aggregating iterator can be
+configured to add the value of all mutations applied with the same key. These types
+of tables can easily be created from raw events by simply extracting the entities,
+attributes, and relationships from individual events and inserting the keys into
+Accumulo each with a count of 1. The aggregating iterator will take care of
+maintaining the edge weights.
+
+=== Document-Partitioned Indexing
+
+Using a simple index as described above works well when looking for records that
+match one of a set of given criteria. When looking for records that match more than
+one criterion simultaneously, such as when looking for documents that contain all of
+the words `the' and `white' and `house', there are several issues.
+
+First is that the set of all records matching any one of the search terms must be sent
+to the client, which incurs a lot of network traffic. The second problem is that the
+client is responsible for performing set intersection on the sets of records returned
+to eliminate all but the records matching all search terms. The memory of the client
+may easily be overwhelmed during this operation.
+
+For these reasons Accumulo includes support for a scheme known as sharded
+indexing, in which these set operations can be performed at the TabletServers and
+decisions about which records to include in the result set can be made without
+incurring network traffic.
+
+This is accomplished via partitioning records into bins that each reside on at most
+one TabletServer, and then creating an index of terms per record within each bin as
+follows:
+
+[width="75%",cols="^,^,^,^"]
+[grid="rows"]
+[options="header"]
+|==============================================
+|RowID |Column Family |Column Qualifier |Value
+|BinID |Term |DocID |Weight
+|==============================================
+
+Documents or records are mapped into bins by a user-defined ingest application. By
+storing the BinID as the RowID we ensure that all the information for a particular
+bin is contained in a single tablet and hosted on a single TabletServer since
+Accumulo never splits rows across tablets. Storing the Terms as column families
+serves to enable fast lookups of all the documents within this bin that contain the
+given term.
+
+Finally, we perform set intersection operations on the TabletServer via a special
+iterator called the Intersecting Iterator. Since documents are partitioned into many
+bins, a search of all documents must search every bin. We can use the BatchScanner
+to scan all bins in parallel. The Intersecting Iterator should be enabled on a
+BatchScanner within user query code as follows:
+
+[source,java]
+----
+Text[] terms = {new Text("the"), new Text("white"), new Text("house")};
+
+BatchScanner bscan = conn.createBatchScanner(table, auths, 20);
+
+IteratorSetting iter = new IteratorSetting(20, "ii", IntersectingIterator.class);
+IntersectingIterator.setColumnFamilies(iter, terms);
+
+bscan.addScanIterator(iter);
+bscan.setRanges(Collections.singleton(new Range()));
+
+for(Entry<Key,Value> entry : bscan) {
+ System.out.println(" " + entry.getKey().getColumnQualifier());
+}
+----
+
+This code effectively has the BatchScanner scan all tablets of a table, looking for
+documents that match all the given terms. Because all tablets are being scanned for
+every query, each query is more expensive than other Accumulo scans, which
+typically involve a small number of TabletServers. This reduces the number of
+concurrent queries supported and is subject to what is known as the `straggler'
+problem in which every query runs as slow as the slowest server participating.
+
+Of course, fast servers will return their results to the client which can display them
+to the user immediately while they wait for the rest of the results to arrive. If the
+results are unordered this is quite effective as the first results to arrive are as good
+as any others to the user.
[43/50] [abbrv] git commit: ACCUMULO-2709 Don't create the work entry
unless there's work to be done
Posted by el...@apache.org.
ACCUMULO-2709 Don't create the work entry unless there's work to be done
We can save on writing on some mutations by checking if the status record
needs replication. The WorkAssigner already had a check for this down stream,
but we can be a bit more efficient having it earlier on in the pipeline.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b5cd35a6
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b5cd35a6
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b5cd35a6
Branch: refs/heads/ACCUMULO-378
Commit: b5cd35a6d683c5ff315d5f6ab7f1879d798c1951
Parents: 3579b67
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 12:19:37 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 12:19:37 2014 -0400
----------------------------------------------------------------------
.../accumulo/master/replication/WorkMaker.java | 23 +++++++-
.../master/replication/WorkMakerTest.java | 61 +++++++++++++++++---
2 files changed, 74 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b5cd35a6/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
index cfc756c..956d550 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/WorkMaker.java
@@ -34,6 +34,8 @@ import org.apache.accumulo.core.replication.ReplicationSchema;
import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.replication.ReplicationTable;
@@ -41,13 +43,16 @@ import org.apache.accumulo.trace.instrument.Span;
import org.apache.accumulo.trace.instrument.Trace;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.protobuf.InvalidProtocolBufferException;
/**
* Reads replication records from the replication table and creates work records which include target replication system information.
*/
public class WorkMaker {
- private static final Logger log = Logger.getLogger(WorkMaker.class);
+ private static final Logger log = LoggerFactory.getLogger(WorkMaker.class);
private final Connector conn;
@@ -88,6 +93,20 @@ public class WorkMaker {
ReplicationSchema.StatusSection.getFile(entry.getKey(), file);
ReplicationSchema.StatusSection.getTableId(entry.getKey(), tableId);
log.info("Processing replication status record for " + file + " on table "+ tableId);
+
+ Status status;
+ try {
+ status = Status.parseFrom(entry.getValue().get());
+ } catch (InvalidProtocolBufferException e) {
+ log.error("Could not parse protobuf for {} from table {}", file, tableId);
+ continue;
+ }
+
+ // Don't create the record if we have nothing to do
+ // TODO put this into a filter on serverside
+ if (!StatusUtil.isWorkRequired(status)) {
+ continue;
+ }
// Get the table configuration for the table specified by the status record
tableConf = ServerConfiguration.getTableConfiguration(conn.getInstance(), tableId.toString());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b5cd35a6/server/master/src/test/java/org/apache/accumulo/master/replication/WorkMakerTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/WorkMakerTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/WorkMakerTest.java
index bebab51..34bb567 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/replication/WorkMakerTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/WorkMakerTest.java
@@ -28,16 +28,16 @@ import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
import org.apache.accumulo.core.replication.ReplicationTarget;
import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.replication.ReplicationTable;
-import org.apache.accumulo.server.util.ReplicationTableUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.Assert;
@@ -47,6 +47,7 @@ import org.junit.Test;
import org.junit.rules.TestName;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Iterables;
/**
*
@@ -82,19 +83,24 @@ public class WorkMakerTest {
String tableId = conn.tableOperations().tableIdMap().get(table);
String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678";
- KeyExtent extent = new KeyExtent(new Text(tableId), null, null);
- Mutation m = ReplicationTableUtil.createUpdateMutation(new Path(file), StatusUtil.fileClosedValue(), extent);
+ Mutation m = new Mutation(new Path(file).toString());
+ m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileClosedValue());
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
bw.addMutation(m);
bw.flush();
+ // Assert that we have one record in the status section
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Assert.assertEquals(1, Iterables.size(s));
+
WorkMaker workMaker = new WorkMaker(conn);
ReplicationTarget expected = new ReplicationTarget("remote_cluster_1", "4");
workMaker.setBatchWriter(bw);
workMaker.addWorkRecord(new Text(file), StatusUtil.fileClosedValue(), ImmutableMap.of("remote_cluster_1", "4"));
- Scanner s = ReplicationTable.getScanner(conn);
+ s = ReplicationTable.getScanner(conn);
WorkSection.limit(s);
Iterator<Entry<Key,Value>> iter = s.iterator();
@@ -119,12 +125,17 @@ public class WorkMakerTest {
String tableId = conn.tableOperations().tableIdMap().get(table);
String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678";
- KeyExtent extent = new KeyExtent(new Text(tableId), null, null);
- Mutation m = ReplicationTableUtil.createUpdateMutation(new Path(file), StatusUtil.fileClosedValue(), extent);
+ Mutation m = new Mutation(new Path(file).toString());
+ m.put(StatusSection.NAME, new Text(tableId), StatusUtil.fileClosedValue());
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
bw.addMutation(m);
bw.flush();
+ // Assert that we have one record in the status section
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Assert.assertEquals(1, Iterables.size(s));
+
WorkMaker workMaker = new WorkMaker(conn);
Map<String,String> targetClusters = ImmutableMap.of("remote_cluster_1", "4", "remote_cluster_2", "6", "remote_cluster_3", "8");
@@ -135,7 +146,7 @@ public class WorkMakerTest {
workMaker.setBatchWriter(bw);
workMaker.addWorkRecord(new Text(file), StatusUtil.fileClosedValue(), targetClusters);
- Scanner s = ReplicationTable.getScanner(conn);
+ s = ReplicationTable.getScanner(conn);
WorkSection.limit(s);
Set<ReplicationTarget> actualTargets = new HashSet<>();
@@ -154,4 +165,38 @@ public class WorkMakerTest {
Assert.assertTrue("Found extra replication work entries: " + actualTargets, actualTargets.isEmpty());
}
+
+ @Test
+ public void dontCreateWorkForEntriesWithNothingToReplicate() throws Exception {
+ String table = name.getMethodName();
+ conn.tableOperations().create(name.getMethodName());
+ String tableId = conn.tableOperations().tableIdMap().get(table);
+ String file = "hdfs://localhost:8020/accumulo/wal/123456-1234-1234-12345678";
+
+ Mutation m = new Mutation(new Path(file).toString());
+ m.put(StatusSection.NAME, new Text(tableId), StatusUtil.newFileValue());
+ BatchWriter bw = ReplicationTable.getBatchWriter(conn);
+ bw.addMutation(m);
+ bw.flush();
+
+ // Assert that we have one record in the status section
+ Scanner s = ReplicationTable.getScanner(conn);
+ StatusSection.limit(s);
+ Assert.assertEquals(1, Iterables.size(s));
+
+ WorkMaker workMaker = new WorkMaker(conn);
+
+ conn.tableOperations().setProperty(ReplicationTable.NAME, Property.TABLE_REPLICATION_TARGETS.getKey() + "remote_cluster_1", "4");
+
+ workMaker.setBatchWriter(bw);
+
+ // If we don't shortcircuit out, we should get an exception because ServerConfiguration.getTableConfiguration
+ // won't work with MockAccumulo
+ workMaker.run();
+
+ s = ReplicationTable.getScanner(conn);
+ WorkSection.limit(s);
+
+ Assert.assertEquals(0, Iterables.size(s));
+ }
}
[36/50] [abbrv] ACCUMULO-1327 converted latex manual to asciidoc
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/table_design.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/table_design.tex b/docs/src/main/latex/accumulo_user_manual/chapters/table_design.tex
deleted file mode 100644
index ff1cebd..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/table_design.tex
+++ /dev/null
@@ -1,343 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Table Design}
-
-\section{Basic Table}
-
-Since Accumulo tables are sorted by row ID, each table can be thought of as being
-indexed by the row ID. Lookups performed by row ID can be executed quickly, by doing
-a binary search, first across the tablets, and then within a tablet. Clients should
-choose a row ID carefully in order to support their desired application. A simple rule
-is to select a unique identifier as the row ID for each entity to be stored and assign
-all the other attributes to be tracked to be columns under this row ID. For example,
-if we have the following data in a comma-separated file:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- userid,age,address,account-balance
-\end{verbatim}\endgroup
-
-We might choose to store this data using the userid as the rowID, the column
-name in the column family, and a blank column qualifier:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Mutation m = new Mutation(userid);
-final String column_qualifier = "";
-m.put("age", column_qualifier, age);
-m.put("address", column_qualifier, address);
-m.put("balance", column_qualifier, account_balance);
-
-writer.add(m);
-\end{verbatim}\endgroup
-
-We could then retrieve any of the columns for a specific userid by specifying the
-userid as the range of a scanner and fetching specific columns:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Range r = new Range(userid, userid); // single row
-Scanner s = conn.createScanner("userdata", auths);
-s.setRange(r);
-s.fetchColumnFamily(new Text("age"));
-
-for(Entry<Key,Value> entry : s)
- System.out.println(entry.getValue().toString());
-\end{verbatim}\endgroup
-
-\section{RowID Design}
-
-Often it is necessary to transform the rowID in order to have rows ordered in a way
-that is optimal for anticipated access patterns. A good example of this is reversing
-the order of components of internet domain names in order to group rows of the
-same parent domain together:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-com.google.code
-com.google.labs
-com.google.mail
-com.yahoo.mail
-com.yahoo.research
-\end{verbatim}\endgroup
-
-Some data may result in the creation of very large rows - rows with many columns.
-In this case the table designer may wish to split up these rows for better load
-balancing while keeping them sorted together for scanning purposes. This can be
-done by appending a random substring at the end of the row:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-com.google.code_00
-com.google.code_01
-com.google.code_02
-com.google.labs_00
-com.google.mail_00
-com.google.mail_01
-\end{verbatim}\endgroup
-
-It could also be done by adding a string representation of some period of time such as date to the week
-or month:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-com.google.code_201003
-com.google.code_201004
-com.google.code_201005
-com.google.labs_201003
-com.google.mail_201003
-com.google.mail_201004
-\end{verbatim}\endgroup
-
-Appending dates provides the additional capability of restricting a scan to a given
-date range.
-
-\section{Lexicoders}
-Since Keys in Accumulo are sorted lexicographically by default, it's often useful to encode
-common data types into a byte format in which their sort order corresponds to the sort order
-in their native form. An example of this is encoding dates and numerical data so that they can
-be better seeked or searched in ranges.
-
-The lexicoders are a standard and extensible way of encoding Java types. Here's an example
-of a lexicoder that encodes a java Date object so that it sorts lexicographically:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// create new date lexicoder
-DateLexicoder dateEncoder = new DateLexicoder();
-
-// truncate time to hours
-long epoch = System.currentTimeMillis();
-Date hour = new Date(epoch - (epoch % 3600000));
-
-// encode the rowId so that it is sorted lexicographically
-Mutation mutation = new Mutation(dateEncoder.encode(hour));
-mutation.put(new Text("colf"), new Text("colq"), new Value(new byte[]{}));
-\end{verbatim}\endgroup
-
-If we want to return the most recent date first, we can reverse the sort order
-with the reverse lexicoder:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// create new date lexicoder and reverse lexicoder
-DateLexicoder dateEncoder = new DateLexicoder();
-ReverseLexicoder reverseEncoder = new ReverseLexicoder(dateEncoder);
-
-// truncate date to hours
-long epoch = System.currentTimeMillis();
-Date hour = new Date(epoch - (epoch % 3600000));
-
-// encode the rowId so that it sorts in reverse lexicographic order
-Mutation mutation = new Mutation(reverseEncoder.encode(hour));
-mutation.put(new Text("colf"), new Text("colq"), new Value(new byte[]{}));
-\end{verbatim}\endgroup
-
-
-\section{Indexing}
-In order to support lookups via more than one attribute of an entity, additional
-indexes can be built. However, because Accumulo tables can support any number of
-columns without specifying them beforehand, a single additional index will often
-suffice for supporting lookups of records in the main table. Here, the index has, as
-the rowID, the Value or Term from the main table, the column families are the same,
-and the column qualifier of the index table contains the rowID from the main table.
-
-\begin{center}
-$\begin{array}{|c|c|c|c|c|c|} \hline
-\multicolumn{5}{|c|}{\mbox{Key}} & \multirow{3}{*}{\mbox{Value}}\\ \cline{1-5}
-\multirow{2}{*}{\mbox{Row ID}}& \multicolumn{3}{|c|}{\mbox{Column}} & \multirow{2}{*}{\mbox{Timestamp}} & \\ \cline{2-4}
-& \mbox{Family} & \mbox{Qualifier} & \mbox{Visibility} & & \\ \hline \hline
-\mbox{Term} & \mbox{Field Name} & \mbox{MainRowID} & & &\\ \hline
-\end{array}$
-\end{center}
-
-Note: We store rowIDs in the column qualifier rather than the Value so that we can
-have more than one rowID associated with a particular term within the index. If we
-stored this in the Value we would only see one of the rows in which the value
-appears since Accumulo is configured by default to return the one most recent
-value associated with a key.
-
-Lookups can then be done by scanning the Index Table first for occurrences of the
-desired values in the columns specified, which returns a list of row ID from the main
-table. These can then be used to retrieve each matching record, in their entirety, or a
-subset of their columns, from the Main Table.
-
-To support efficient lookups of multiple rowIDs from the same table, the Accumulo
-client library provides a BatchScanner. Users specify a set of Ranges to the
-BatchScanner, which performs the lookups in multiple threads to multiple servers
-and returns an Iterator over all the rows retrieved. The rows returned are NOT in
-sorted order, as is the case with the basic Scanner interface.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// first we scan the index for IDs of rows matching our query
-
-Text term = new Text("mySearchTerm");
-
-HashSet<Range> matchingRows = new HashSet<Range>();
-
-Scanner indexScanner = createScanner("index", auths);
-indexScanner.setRange(new Range(term, term));
-
-// we retrieve the matching rowIDs and create a set of ranges
-for(Entry<Key,Value> entry : indexScanner)
- matchingRows.add(new Range(entry.getKey().getColumnQualifier()));
-
-// now we pass the set of rowIDs to the batch scanner to retrieve them
-BatchScanner bscan = conn.createBatchScanner("table", auths, 10);
-
-bscan.setRanges(matchingRows);
-bscan.fetchColumnFamily(new Text("attributes"));
-
-for(Entry<Key,Value> entry : bscan)
- System.out.println(entry.getValue());
-\end{verbatim}\endgroup
-
-One advantage of the dynamic schema capabilities of Accumulo is that different
-fields may be indexed into the same physical table. However, it may be necessary to
-create different index tables if the terms must be formatted differently in order to
-maintain proper sort order. For example, real numbers must be formatted
-differently than their usual notation in order to be sorted correctly. In these cases,
-usually one index per unique data type will suffice.
-
-\section{Entity-Attribute and Graph Tables}
-
-Accumulo is ideal for storing entities and their attributes, especially of the
-attributes are sparse. It is often useful to join several datasets together on common
-entities within the same table. This can allow for the representation of graphs,
-including nodes, their attributes, and connections to other nodes.
-
-Rather than storing individual events, Entity-Attribute or Graph tables store
-aggregate information about the entities involved in the events and the
-relationships between entities. This is often preferrable when single events aren't
-very useful and when a continuously updated summarization is desired.
-
-The physical schema for an entity-attribute or graph table is as follows:
-
-\begin{center}
-$\begin{array}{|c|c|c|c|c|c|} \hline
-\multicolumn{5}{|c|}{\mbox{Key}} & \multirow{3}{*}{\mbox{Value}}\\ \cline{1-5}
-\multirow{2}{*}{\mbox{Row ID}}& \multicolumn{3}{|c|}{\mbox{Column}} & \multirow{2}{*}{\mbox{Timestamp}} & \\ \cline{2-4}
-& \mbox{Family} & \mbox{Qualifier} & \mbox{Visibility} & & \\ \hline \hline
-\mbox{EntityID} & \mbox{Attribute Name} & \mbox{Attribute Value} & & & \mbox{Weight} \\ \hline
-\mbox{EntityID} & \mbox{Edge Type} & \mbox{Related EntityID} & & & \mbox{Weight} \\ \hline
-\end{array}$
-\end{center}
-
-For example, to keep track of employees, managers and products the following
-entity-attribute table could be used. Note that the weights are not always necessary
-and are set to 0 when not used.
-
-$\begin{array}{llll}
-\bf{RowID} & \bf{Column Family} & \bf{Column Qualifier} & \bf{Value} \\
-\\
-E001 & name & bob & 0 \\
-E001 & department & sales & 0 \\
-E001 & hire\_date & 20030102 & 0 \\
-E001 & units\_sold & P001 & 780 \\
-\\
-E002 & name & george & 0 \\
-E002 & department & sales & 0 \\
-E002 & manager\_of & E001 & 0 \\
-E002 & manager\_of & E003 & 0 \\
-\\
-E003 & name & harry & 0 \\
-E003 & department & accounts\_recv & 0 \\
-E003 & hire\_date & 20000405 & 0 \\
-E003 & units\_sold & P002 & 566 \\
-E003 & units\_sold & P001 & 232 \\
-\\
-P001 & product\_name & nike\_airs & 0 \\
-P001 & product\_type & shoe & 0 \\
-P001 & in\_stock & germany & 900 \\
-P001 & in\_stock & brazil & 200 \\
-\\
-P002 & product\_name & basic\_jacket & 0 \\
-P002 & product\_type & clothing & 0 \\
-P002 & in\_stock & usa & 3454 \\
-P002 & in\_stock & germany & 700 \\
-\end{array}$
-\vspace{5mm}
-
-To allow efficient updating of edge weights, an aggregating iterator can be
-configured to add the value of all mutations applied with the same key. These types
-of tables can easily be created from raw events by simply extracting the entities,
-attributes, and relationships from individual events and inserting the keys into
-Accumulo each with a count of 1. The aggregating iterator will take care of
-maintaining the edge weights.
-
-\section{Document-Partitioned Indexing}
-
-Using a simple index as described above works well when looking for records that
-match one of a set of given criteria. When looking for records that match more than
-one criterion simultaneously, such as when looking for documents that contain all of
-the words `the' and `white' and `house', there are several issues.
-
-First is that the set of all records matching any one of the search terms must be sent
-to the client, which incurs a lot of network traffic. The second problem is that the
-client is responsible for performing set intersection on the sets of records returned
-to eliminate all but the records matching all search terms. The memory of the client
-may easily be overwhelmed during this operation.
-
-For these reasons Accumulo includes support for a scheme known as sharded
-indexing, in which these set operations can be performed at the TabletServers and
-decisions about which records to include in the result set can be made without
-incurring network traffic.
-
-This is accomplished via partitioning records into bins that each reside on at most
-one TabletServer, and then creating an index of terms per record within each bin as
-follows:
-
-\begin{center}
-$\begin{array}{|c|c|c|c|c|c|} \hline
-\multicolumn{5}{|c|}{\mbox{Key}} & \multirow{3}{*}{\mbox{Value}}\\ \cline{1-5}
-\multirow{2}{*}{\mbox{Row ID}}& \multicolumn{3}{|c|}{\mbox{Column}} & \multirow{2}{*}{\mbox{Timestamp}} & \\ \cline{2-4}
-& \mbox{Family} & \mbox{Qualifier} & \mbox{Visibility} & & \\ \hline \hline
-\mbox{BinID} & \mbox{Term} & \mbox{DocID} & & & \mbox{Weight} \\ \hline
-\end{array}$
-\end{center}
-
-Documents or records are mapped into bins by a user-defined ingest application. By
-storing the BinID as the RowID we ensure that all the information for a particular
-bin is contained in a single tablet and hosted on a single TabletServer since
-Accumulo never splits rows across tablets. Storing the Terms as column families
-serves to enable fast lookups of all the documents within this bin that contain the
-given term.
-
-Finally, we perform set intersection operations on the TabletServer via a special
-iterator called the Intersecting Iterator. Since documents are partitioned into many
-bins, a search of all documents must search every bin. We can use the BatchScanner
-to scan all bins in parallel. The Intersecting Iterator should be enabled on a
-BatchScanner within user query code as follows:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Text[] terms = {new Text("the"), new Text("white"), new Text("house")};
-
-BatchScanner bs = conn.createBatchScanner(table, auths, 20);
-IteratorSetting iter = new IteratorSetting(20, "ii", IntersectingIterator.class);
-IntersectingIterator.setColumnFamilies(iter, terms);
-bs.addScanIterator(iter);
-bs.setRanges(Collections.singleton(new Range()));
-
-for(Entry<Key,Value> entry : bs) {
- System.out.println(" " + entry.getKey().getColumnQualifier());
-}
-\end{verbatim}\endgroup
-
-This code effectively has the BatchScanner scan all tablets of a table, looking for
-documents that match all the given terms. Because all tablets are being scanned for
-every query, each query is more expensive than other Accumulo scans, which
-typically involve a small number of TabletServers. This reduces the number of
-concurrent queries supported and is subject to what is known as the `straggler'
-problem in which every query runs as slow as the slowest server participating.
-
-Of course, fast servers will return their results to the client which can display them
-to the user immediately while they wait for the rest of the results to arrive. If the
-results are unordered this is quite effective as the first results to arrive are as good
-as any others to the user.
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex b/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
deleted file mode 100644
index 203fe0c..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/troubleshooting.tex
+++ /dev/null
@@ -1,794 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Troubleshooting}
-
-\section{Logs}
-
-Q. The tablet server does not seem to be running!? What happened?
-
-Accumulo is a distributed system. It is supposed to run on remote
-equipment, across hundreds of computers. Each program that runs on
-these remote computers writes down events as they occur, into a local
-file. By default, this is defined in
-\texttt{\$ACCUMULO\_HOME}/conf/accumule-env.sh as ACCUMULO\_LOG\_DIR.
-
-A. Look in the \texttt{\$ACCUMULO\_LOG\_DIR}/tserver*.log file. Specifically, check the end of the file.
-
-Q. The tablet server did not start and the debug log does not exists! What happened?
-
-When the individual programs are started, the stdout and stderr output
-of these programs are stored in ``.out'' and ``.err'' files in
-\texttt{\$ACCUMULO\_LOG\_DIR}. Often, when there are missing configuration
-options, files or permissions, messages will be left in these files.
-
-A. Probably a start-up problem. Look in \texttt{\$ACCUMULO\_LOG\_DIR}/tserver*.err
-
-\section{Monitor}
-
-Q. Accumulo is not working, what's wrong?
-
-There's a small web server that collects information about all the
-components that make up a running Accumulo instance. It will highlight
-unusual or unexpected conditions.
-
-A. Point your browser to the monitor (typically the master host, on port 50095). Is anything red or yellow?
-
-Q. My browser is reporting connection refused, and I cannot get to the monitor
-
-The monitor program's output is also written to .err and .out files in
-the \texttt{\$ACCUMULO\_LOG\_DIR}. Look for problems in this file if the
-\texttt{\$ACCUMULO\_LOG\_DIR/monitor*.log} file does not exist.
-
-A. The monitor program is probably not running. Check the log files for errors.
-
-Q. My browser hangs trying to talk to the monitor.
-
-Your browser needs to be able to reach the monitor program. Often
-large clusters are firewalled, or use a VPN for internal
-communications. You can use SSH to proxy your browser to the cluster,
-or consult with your system administrator to gain access to the server
-from your browser.
-
-It is sometimes helpful to use a text-only browser to sanity-check the
-monitor while on the machine running the monitor:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ links http://localhost:50095
-\end{verbatim}\endgroup
-
-A. Verify that you are not firewalled from the monitor if it is running on a remote host.
-
-Q. The monitor responds, but there are no numbers for tservers and tables. The summary page says the master is down.
-
-The monitor program gathers all the details about the master and the
-tablet servers through the master. It will be mostly blank if the
-master is down.
-
-A. Check for a running master.
-
-\section{HDFS}
-
-Accumulo reads and writes to the Hadoop Distributed File System.
-Accumulo needs this file system available at all times for normal operations.
-
-Q. Accumulo is having problems ``getting a block blk\_1234567890123.'' How do I fix it?
-
-This troubleshooting guide does not cover HDFS, but in general, you
-want to make sure that all the datanodes are running and an fsck check
-finds the file system clean:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ hadoop fsck /accumulo
-\end{verbatim}\endgroup
-
-You can use:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ hadoop fsck /accumulo/path/to/corrupt/file -locations -blocks -files
-\end{verbatim}\endgroup
-
-to locate the block references of individual corrupt files and use those
-references to search the name node and individual data node logs to determine which
-servers those blocks have been assigned and then try to fix any underlying file
-system issues on those nodes.
-
-On a larger cluster, you may need to increase the number of Xceivers
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- <property>
- <name>dfs.datanode.max.xcievers</name>
- <value>4096</value>
- </property>
-\end{verbatim}\endgroup
-
-A. Verify HDFS is healthy, check the datanode logs.
-
-\section{Zookeeper}
-
-Q. \texttt{accumulo init} is hanging. It says something about talking to zookeeper.
-
-Zookeeper is also a distributed service. You will need to ensure that
-it is up. You can run the zookeeper command line tool to connect to
-any one of the zookeeper servers:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ zkCli.sh -server zoohost
-...
-[zk: zoohost:2181(CONNECTED) 0]
-\end{verbatim}\endgroup
-
-It is important to see the word \texttt{CONNECTED}! If you only see
-\texttt{CONNECTING} you will need to diagnose zookeeper errors.
-
-A. Check to make sure that zookeeper is up, and that
-\texttt{\$ACCUMULO\_HOME/conf/accumulo-site.xml} has been pointed to
-your zookeeper server(s).
-
-Q. Zookeeper is running, but it does not say \texttt{CONNECTED}
-
-Zookeeper processes talk to each other to elect a leader. All updates
-go through the leader and propagate to a majority of all the other
-nodes. If a majority of the nodes cannot be reached, zookeeper will
-not allow updates. Zookeeper also limits the number connections to a
-server from any other single host. By default, this limit can be as small as 10
-and can be reached in some everything-on-one-machine test configurations.
-
-You can check the election status and connection status of clients by
-asking the zookeeper nodes for their status. You connect to zookeeper
-and ask it with the four-letter ``stat'' command:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ nc zoohost 2181
-stat
-Zookeeper version: 3.4.5-1392090, built on 09/30/2012 17:52 GMT
-Clients:
- /127.0.0.1:58289[0](queued=0,recved=1,sent=0)
- /127.0.0.1:60231[1](queued=0,recved=53910,sent=53915)
-
-Latency min/avg/max: 0/5/3008
-Received: 1561459
-Sent: 1561592
-Connections: 2
-Outstanding: 0
-Zxid: 0x621a3b
-Mode: standalone
-Node count: 22524
-$
-\end{verbatim}\endgroup
-
-
-A. Check zookeeper status, verify that it has a quorum, and has not exceeded maxClientCnxns.
-
-Q. My tablet server crashed! The logs say that it lost it's zookeeper lock.
-
-Tablet servers reserve a lock in zookeeper to maintain their ownership
-over the tablets that have been assigned to them. Part of their
-responsibility for keeping the lock is to send zookeeper a keep-alive
-message periodically. If the tablet server fails to send a message in
-a timely fashion, zookeeper will remove the lock and notify the tablet
-server. If the tablet server does not receive a message from
-zookeeper, it will assume its lock has been lost, too. If a tablet
-server loses its lock, it kills itself: everything assumes it is dead
-already.
-
-A. Investigate why the tablet server did not send a timely message to
-zookeeper.
-
-\subsection{Keeping the tablet server lock}
-
-Q. My tablet server lost its lock. Why?
-
-The primary reason a tablet server loses its lock is that it has been pushed into swap.
-
-A large java program (like the tablet server) may have a large portion
-of its memory image unused. The operation system will favor pushing
-this allocated, but unused memory into swap so that the memory can be
-re-used as a disk buffer. When the java virtual machine decides to
-access this memory, the OS will begin flushing disk buffers to return that
-memory to the VM. This can cause the entire process to block long
-enough for the zookeeper lock to be lost.
-
-A. Configure your system to reduce the kernel parameter ``swappiness'' from the default (60) to zero.
-
-Q. My tablet server lost its lock, and I have already set swappiness to
-zero. Why?
-
-Be careful not to over-subscribe memory. This can be easy to do if
-your accumulo processes run on the same nodes as hadoop's map-reduce
-framework. Remember to add up:
-
-\begin{itemize}
-\item{size of the JVM for the tablet server}
-\item{size of the in-memory map, if using the native map implementation}
-\item{size of the JVM for the data node}
-\item{size of the JVM for the task tracker}
-\item{size of the JVM times the maximum number of mappers and reducers}
-\item{size of the kernel and any support processes}
-\end{itemize}
-
-If a 16G node can run 2 mappers and 2 reducers, and each can be 2G,
-then there is only 8G for the data node, tserver, task tracker and OS.
-
-A. Reduce the memory footprint of each component until it fits comfortably.
-
-Q. My tablet server lost its lock, swappiness is zero, and my node has lots of unused memory!
-
-The JVM memory garbage collector may fall behind and cause a
-``stop-the-world'' garbage collection. On a large memory virtual
-machine, this collection can take a long time. This happens more
-frequently when the JVM is getting low on free memory. Check the logs
-of the tablet server. You will see lines like this:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-2013-06-20 13:43:20,607 [tabletserver.TabletServer] DEBUG: gc ParNew=0.00(+0.00) secs
- ConcurrentMarkSweep=0.00(+0.00) secs freemem=1,868,325,952(+1,868,325,952) totalmem=2,040,135,680
-\end{verbatim}\endgroup
-
-When ``freemem'' becomes small relative to the amount of memory
-needed, the JVM will spend more time finding free memory than
-performing work. This can cause long delays in sending keep-alive
-messages to zookeeper.
-
-A. Ensure the tablet server JVM is not running low on memory.
-
-\section{Tools}
-
-The accumulo script can be used to run classes from the command line.
-This section shows how a few of the utilities work, but there are many
-more.
-
-There's a class that will examine an accumulo storage file and print
-out basic metadata.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.core.file.rfile.PrintInfo /accumulo/tables/1/default_tablet/A000000n.rf
-2013-07-16 08:17:14,778 [util.NativeCodeLoader] INFO : Loaded the native-hadoop library
-Locality group : <DEFAULT>
- Start block : 0
- Num blocks : 1
- Index level 0 : 62 bytes 1 blocks
- First key : 288be9ab4052fe9e span:34078a86a723e5d3:3da450f02108ced5 [] 1373373521623 false
- Last key : start:13fc375709e id:615f5ee2dd822d7a [] 1373373821660 false
- Num entries : 466
- Column families : [waitForCommits, start, md major compactor 1, md major compactor 2, md major compactor 3,
- bringOnline, prep, md major compactor 4, md major compactor 5, md root major compactor 3,
- minorCompaction, wal, compactFiles, md root major compactor 4, md root major compactor 1,
- md root major compactor 2, compact, id, client:update, span, update, commit, write,
- majorCompaction]
-
-Meta block : BCFile.index
- Raw size : 4 bytes
- Compressed size : 12 bytes
- Compression type : gz
-
-Meta block : RFile.index
- Raw size : 780 bytes
- Compressed size : 344 bytes
- Compression type : gz
-\end{verbatim}\endgroup
-
-When trying to diagnose problems related to key size, the PrintInfo tool can provide a histogram of the individual key sizes:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.core.file.rfile.PrintInfo --histogram /accumulo/tables/1/default_tablet/A000000n.rf
-...
-Up to size count %-age
- 10 : 222 28.23%
- 100 : 244 71.77%
- 1000 : 0 0.00%
- 10000 : 0 0.00%
- 100000 : 0 0.00%
- 1000000 : 0 0.00%
- 10000000 : 0 0.00%
- 100000000 : 0 0.00%
- 1000000000 : 0 0.00%
-10000000000 : 0 0.00%
-\end{verbatim}\endgroup
-
-Likewise, PrintInfo will dump the key-value pairs and show you the contents of the RFile:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.core.file.rfile.PrintInfo --dump /accumulo/tables/1/default_tablet/A000000n.rf
-row columnFamily:columnQualifier [visibility] timestamp deleteFlag -> Value
-...
-\end{verbatim}\endgroup
-
-Q. Accumulo is not showing me any data!
-
-A. Do you have your auths set so that it matches your visibilities?
-
-Q. What are my visibilities?
-
-A. Use ``PrintInfo'' on a representative file to get some idea of the visibilities in the underlying data.
-
-Note that the use of PrintInfo is an administrative tool and can only
-by used by someone who can access the underlying Accumulo data. It
-does not provide the normal access controls in Accumulo.
-
-If you would like to backup, or otherwise examine the contents of Zookeeper, there are commands to dump and load to/from XML.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.DumpZookeeper --root /accumulo >dump.xml
-$ ./bin/accumulo org.apache.accumulo.server.util.RestoreZookeeper --overwrite < dump.xml
-\end{verbatim}\endgroup
-
-Q. How can I get the information in the monitor page for my cluster monitoring system?
-
-A. Use GetMasterStats:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.test.GetMasterStats | grep Load
- OS Load Average: 0.27
-\end{verbatim}\endgroup
-
-Q. The monitor page is showing an offline tablet. How can I find out which tablet it is?
-
-A. Use FindOfflineTablets:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.FindOfflineTablets
-2<<@(null,null,localhost:9997) is UNASSIGNED #walogs:2
-\end{verbatim}\endgroup
-
-Here's what the output means:
-
-\begin{enumerate}
-\item{\texttt{2<<} This is the tablet from (-inf, +inf) for the
- table with id 2. ``tables -l'' in the shell will show table ids for
- tables.}
-\item{@(null, null, localhost:9997)} Location information. The
- format is \texttt{@(assigned, hosted, last)}. In this case, the
- tablet has not been assigned, is not hosted anywhere, and was once
- hosted on localhost.
-\item{\#walogs:2} The number of write-ahead logs that this tablet requires for recovery.
-\end{enumerate}
-
-An unassigned tablet with write-ahead logs is probably waiting for
-logs to be sorted for efficient recovery.
-
-Q. How can I be sure that the metadata tables are up and consistent?
-
-A. \texttt{CheckForMetadataProblems} will verify the start/end of
-every tablet matches, and the start and stop for the table is empty:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.CheckForMetadataProblems -u root --password
-Enter the connection password:
-All is well for table !0
-All is well for table 1
-\end{verbatim}\endgroup
-
-Q. My hadoop cluster has lost a file due to a NameNode failure. How can I remove the file?
-
-A. There's a utility that will check every file reference and ensure
-that the file exists in HDFS. Optionally, it will remove the
-reference:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.RemoveEntriesForMissingFiles -u root --password
-Enter the connection password:
-2013-07-16 13:10:57,293 [util.RemoveEntriesForMissingFiles] INFO : File /accumulo/tables/2/default_tablet/F0000005.rf
- is missing
-2013-07-16 13:10:57,296 [util.RemoveEntriesForMissingFiles] INFO : 1 files of 3 missing
-\end{verbatim}\endgroup
-
-Q. I have many entries in zookeeper for old instances I no longer need. How can I remove them?
-
-A. Use CleanZookeeper:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.CleanZookeeper
-\end{verbatim}\endgroup
-
-This command will not delete the instance pointed to by the local \texttt{conf/accumulo-site.xml} file.
-
-Q. I need to decommission a node. How do I stop the tablet server on it?
-
-A. Use the admin command:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo admin stop hostname:9997
-2013-07-16 13:15:38,403 [util.Admin] INFO : Stopping server 12.34.56.78:9997
-\end{verbatim}\endgroup
-
-Q. I cannot login to a tablet server host, and the tablet server will not shut down. How can I kill the server?
-
-A. Sometimes you can kill a ``stuck'' tablet server by deleting it's lock in zookeeper:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.TabletServerLocks --list
- 127.0.0.1:9997 TSERV_CLIENT=127.0.0.1:9997
-$ ./bin/accumulo org.apache.accumulo.server.util.TabletServerLocks -delete 127.0.0.1:9997
-$ ./bin/accumulo org.apache.accumulo.server.util.TabletServerLocks -list
- 127.0.0.1:9997 null
-\end{verbatim}\endgroup
-
-You can find the master and instance id for any accumulo instances using the same zookeeper instance:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo org.apache.accumulo.server.util.ListInstances
-INFO : Using ZooKeepers localhost:2181
-
- Instance Name | Instance ID | Master
----------------------+--------------------------------------+-------------------------------
- "test" | 6140b72e-edd8-4126-b2f5-e74a8bbe323b | 127.0.0.1:9999
-\end{verbatim}\endgroup
-
-\section{System Metadata Tables}
-\label{sec:metadata}
-
-Accumulo tracks information about tables in metadata tables. The metadata for
-most tables is contained within the metadata table in the accumulo namespace,
-while metadata for that table is contained in the root table in the accumulo
-namespace. The root table is composed of a single tablet, which does not
-split, so it is also called the root tablet. Information about the root
-table, such as its location and write-ahead logs, are stored in ZooKeeper.
-
-Let's create a table and put some data into it:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-shell> createtable test
-shell> tables -l
-accumulo.metadata => !0
-accumulo.root => +r
-test => 2
-trace => 1
-shell> insert a b c d
-shell> flush -w
-\end{verbatim}\endgroup
-
-Now let's take a look at the metadata for this table:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-shell> table accumulo.metadata
-shell> scan -b 3; -e 3<
-3< file:/default_tablet/F000009y.rf [] 186,1
-3< last:13fe86cd27101e5 [] 127.0.0.1:9997
-3< loc:13fe86cd27101e5 [] 127.0.0.1:9997
-3< log:127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995 [] 127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995|6
-3< srv:dir [] /default_tablet
-3< srv:flush [] 1
-3< srv:lock [] tservers/127.0.0.1:9997/zlock-0000000001$13fe86cd27101e5
-3< srv:time [] M1373998392323
-3< ~tab:~pr [] \x00
-\end{verbatim}\endgroup
-
-Let's decode this little session:
-
-\begin{enumerate}
-\item{\texttt{scan -b 3; -e 3<}\\
- Every tablet gets its own row. Every row starts with the table id followed by
- ``;'' or ``<'', and followed by the end row split point for that tablet.}
-\item{\texttt{file:/default\_tablet/F000009y.rf [] 186,1}\\
- File entry for this tablet. This tablet contains a single file reference. The
- file is ``/accumulo/tables/3/default\_tablet/F000009y.rf''. It contains 1
- key/value pair, and is 186 bytes long.}
-\item{\texttt{last:13fe86cd27101e5 [] 127.0.0.1:9997}\\
- Last location for this tablet. It was last held on 127.0.0.1:9997, and the
- unique tablet server lock data was ``13fe86cd27101e5''. The default balancer
- will tend to put tablets back on their last location.}
-\item{\texttt{loc:13fe86cd27101e5 [] 127.0.0.1:9997}\\
- The current location of this tablet.}
-\item{\texttt{log:127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995 [] 127.0. ...}\\
- This tablet has a reference to a single write-ahead log. This file can be found in\\
- /accumulo/wal/127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995. The value
- of this entry could refer to multiple files. This tablet's data is encoded as
- ``6'' within the log.}
-\item{\texttt{srv:dir [] /default\_tablet}\\
- Files written for this tablet will be placed into
- /accumulo/tables/3/default\_tablet.}
-\item{\texttt{srv:flush [] 1}\\
- Flush id. This table has successfully completed the flush with the id of
- ``1''.}
-\item{\texttt{srv:lock [] tservers/127.0.0.1:9997/zlock-0000000001\$13fe86cd27101e5}\\
- This is the lock information for the tablet holding the present lock. This
- information is checked against zookeeper whenever this is updated, which
- prevents a metadata update from a tablet server that no longer holds its
- lock.}
-\item{\texttt{srv:time [] M1373998392323} }
-\item{\texttt{\textasciitilde{}tab:\textasciitilde{}pr [] \textbackslash{}x00}\\
- The end-row marker for the previous tablet (prev-row). The first byte
- indicates the presence of a prev-row. This tablet has the range (-inf, +inf),
- so it has no prev-row (or end row).}
-\end{enumerate}
-
-Besides these columns, you may see:
-
-\begin{enumerate}
-\item{\texttt{rowId future:zooKeeperID location} Tablet has been assigned to a tablet, but not yet loaded.}
-\item{\texttt{\textasciitilde{}del:filename} When a tablet server is done use a file, it will create a delete marker in the appropriate metadata table, unassociated with any tablet. The garbage collector will remove the marker, and the file, when no other reference to the file exists.}
-\item{\texttt{\textasciitilde{}blip:txid} Bulk-Load In Progress marker}
-\item{\texttt{rowId loaded:filename} A file has been bulk-loaded into this tablet, however the bulk load has not yet completed on other tablets, so this is marker prevents the file from being loaded multiple times.}
-\item{\texttt{rowId !cloned} A marker that indicates that this tablet has been successfully cloned.}
-\item{\texttt{rowId splitRatio:ratio} A marker that indicates a split is in progress, and the files are being split at the given ratio.}
-\item{\texttt{rowId chopped} A marker that indicates that the files in the tablet do not contain keys outside the range of the tablet.}
-\item{\texttt{rowId scan} A marker that prevents a file from being removed while there are still active scans using it.}
-
-\end{enumerate}
-
-\section{Simple System Recovery}
-
-Q. One of my Accumulo processes died. How do I bring it back?
-
-The easiest way to bring all services online for an Accumulo instance is to run the ``start-all.sh`` script.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ bin/start-all.sh
-\end{verbatim}\endgroup
-
-This process will check the process listing, using ``jps`` on each host before attempting to restart a service on the given host.
-Typically, this check is sufficient except in the face of a hung/zombie process. For large clusters, it may be
-undesirable to ssh to every node in the cluster to ensure that all hosts are running the appropriate processes and ``start-here.sh`` may be of use.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ ssh host_with_dead_process
- $ bin/start-here.sh
-\end{verbatim}\endgroup
-
-``start-here.sh`` should be invoked on the host which is missing a given process. Like start-all.sh, it will start all
-necessary processes that are not currently running, but only on the current host and not cluster-wide. Tools such as ``pssh`` or
-``pdsh`` can be used to automate this process.
-
-``start-server.sh`` can also be used to start a process on a given host; however, it is not generally recommended for
-users to issue this directly as the ``start-all.sh`` and ``start-here.sh`` scripts provide the same functionality with
-more automation and are less prone to user error.
-
-A. Use ``start-all.sh`` or ``start-here.sh``.
-
-Q. My process died again. Should I restart it via ``cron`` or tools like ``supervisord``?
-
-A. A repeatedly dying Accumulo process is a sign of a larger problem. Typically these problems are due to a
-misconfiguration of Accumulo or over-saturation of resources. Blind automation of any service restart inside of Accumulo
-is generally an undesirable situation as it is indicative of a problem that is being masked and ignored. Accumulo
-processes should be stable on the order of months and not require frequent restart.
-
-
-\section{Advanced System Recovery}
-
-\subsection{HDFS Failure}
-Q. I had disasterous HDFS failure. After bringing everything back up, several tablets refuse to go online.
-
-Data written to tablets is written into memory before being written into indexed files. In case the server
-is lost before the data is saved into a an indexed file, all data stored in memory is first written into a
-write-ahead log (WAL). When a tablet is re-assigned to a new tablet server, the write-ahead logs are read to
-recover any mutations that were in memory when the tablet was last hosted.
-
-If a write-ahead log cannot be read, then the tablet is not re-assigned. All it takes is for one of
-the blocks in the write-ahead log to be missing. This is unlikely unless multiple data nodes in HDFS have been
-lost.
-
-A. Get the WAL files online and healthy. Restore any data nodes that may be down.
-
-Q. How do find out which tablets are offline?
-
-A. Use ``accumulo admin checkTablets''
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ bin/accumulo admin checkTablets
-\end{verbatim}\endgroup
-
-Q. I lost three data nodes, and I'm missing blocks in a WAL. I don't care about data loss, how
-can I get those tablets online?
-
-See the discussion in section~\ref{sec:metadata}, which shows a typical metadata table listing.
-The entries with a column family of ``log'' are references to the WAL for that tablet.
-If you know what WAL is bad, you can find all the references with a grep in the shell:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-shell> grep 0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995
-3< log:127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995 [] 127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995|6
-\end{verbatim}\endgroup
-
-A. You can remove the WAL references in the metadata table.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-shell> grant -u root Table.WRITE -t accumulo.metadata
-shell> delete 3< log 127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995
-\end{verbatim}\endgroup
-
-Note: the colon (``:'') is omitted when specifying the ``row cf cq'' for the delete command.
-
-The master will automatically discover the tablet no longer has a bad WAL reference and will
-assign the tablet. You will need to remove the reference from all the tablets to get them
-online.
-
-
-Q. The metadata (or root) table has references to a corrupt WAL.
-
-This is a much more serious state, since losing updates to the metadata table will result
-in references to old files which may not exist, or lost references to new files, resulting
-in tablets that cannot be read, or large amounts of data loss.
-
-The best hope is to restore the WAL by fixing HDFS data nodes and bringing the data back online.
-If this is not possible, the best approach is to re-create the instance and bulk import all files from
-the old instance into a new tables.
-
-A complete set of instructions for doing this is outside the scope of this guide,
-but the basic approach is:
-
-\begin{itemize}
- \item Use ``tables -l'' in the shell to discover the table name to table id mapping
- \item Stop all accumulo processes on all nodes
- \item Move the accumulo directory in HDFS out of the way:
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $ hadoop fs -mv /accumulo /corrupt
-\end{verbatim}\endgroup
- \item Re-initalize accumulo
- \item Recreate tables, users and permissions
- \item Import the directories under \texttt{/corrupt/tables/<id>} into the new instance
-\end{itemize}
-
-Q. One or more HDFS Files under /accumulo/tables are corrupt
-
-Accumulo maintains multiple references into the tablet files in the METADATA
-table and within the tablet server hosting the file, this makes it difficult to
-reliably just remove those references.
-
-The directory structure in HDFS for tables will follow the general structure:
-
-\small
-\begin{verbatim}
- /accumulo
- /accumulo/tables/
- /accumulo/tables/!0
- /accumulo/tables/!0/default_tablet/A000001.rf
- /accumulo/tables/!0/t-00001/A000002.rf
- /accumulo/tables/1
- /accumulo/tables/1/default_tablet/A000003.rf
- /accumulo/tables/1/t-00001/A000004.rf
- /accumulo/tables/1/t-00001/A000005.rf
- /accumulo/tables/2/default_tablet/A000006.rf
- /accumulo/tables/2/t-00001/A000007.rf
-\end{verbatim}
-\normalsize
-
-If files under /accumulo/tables are corrupt, the best course of action is to
-recover those files in hdsf see the section on HDFS. Once these recovery efforts
-have been exhausted, the next step depends on where the missing file(s) are
-located. Different actions are required when the bad files are in Accumulo data
-table files or if they are metadata table files.
-
-{\bf Data File Corruption}
-
-When an Accumulo data file is corrupt, the most reliable way to restore Accumulo
-operations is to replace the missing file with an “empty” file so that
-references to the file in the METADATA table and within the tablet server
-hosting the file can be resolved by Accumulo. An empty file can be created using
-the CreateEmpty utiity:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $accumulo org.apache.accumulo.core.file.rfile.CreateEmpty /path/to/empty/file/empty.rf
-\end{verbatim}\endgroup
-
-The process is to delete the corrupt file and then move the empty file into its
-place (The generated empty file can be copied and used multiple times if necessary and does not need
-to be regenerated each time)
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- $hadoop fs –rm /accumulo/tables/corrupt/file/thename.rf; \
- hadoop fs -mv /path/to/empty/file/empty.rf /accumulo/tables/corrupt/file/thename.rf
-\end{verbatim}\endgroup
-
-{\bf Metadata File Corruption}
-
-If the corrupt files are metadata files, see \ref{sec:metadata} (under the path
-\begin{verbatim}/accumulo/tables/!0\end{verbatim}) then you will need to rebuild
-the metadata table by initializing a new instance of Accumulo and then importing
-all of the existing data into the new instance. This is the same procedure as
-recovering from a zookeeper failure (see \ref{ZooKeeper Failure}, except that
-you will have the benefit of having the existing user and table authorizations
-that are maintained in zookeeper.
-
-You can use the DumpZookeeper utility to save this information for reference
-before creating the new instance. You will not be able to use RestoreZookeeper
-because the table names and references are likely to be different between the
-original and the new instances, but it can serve as a reference.
-
-A. If the files cannot be recovered, replace corrupt data files with a empty
-rfiles to allow references in the metadata table and in the tablet servers to be
-resolved. Rebuild the metadata table if the corrupt files are metadata files.
-
-\subsection{ZooKeeper Failure}
-Q. I lost my ZooKeeper quorum (hardware failure), but HDFS is still intact. How can I recover my Accumulo instance?
-
-ZooKeeper, in addition to its lock-service capabilities, also serves to bootstrap an Accumulo
-instance from some location in HDFS. It contains the pointers to the root tablet in HDFS which
-is then used to load the Accumulo metadata tablets, which then loads all user tables. ZooKeeper
-also stores all namespace and table configuration, the user database, the mapping of table IDs to
-table names, and more across Accumulo restarts.
-
-Presently, the only way to recover such an instance is to initialize a new instance and import all
-of the old data into the new instance. The easiest way to tackle this problem is to first recreate
-the mapping of table ID to table name and then recreate each of those tables in the new instance.
-Set any necessary configuration on the new tables and add some split points to the tables to close
-the gap between how many splits the old table had and no splits.
-
-The directory structure in HDFS for tables will follow the general structure:
-
-\small
-\begin{verbatim}
- /accumulo
- /accumulo/tables/
- /accumulo/tables/1
- /accumulo/tables/1/default_tablet/A000001.rf
- /accumulo/tables/1/t-00001/A000002.rf
- /accumulo/tables/1/t-00001/A000003.rf
- /accumulo/tables/2/default_tablet/A000004.rf
- /accumulo/tables/2/t-00001/A000005.rf
-\end{verbatim}
-\normalsize
-
-For each table, make a new directory that you can move (or copy if you have the HDFS space to do so)
-all of the rfiles for a given table into. For example, to process the table with an ID of ``1``, make a new directory,
-say ``/new-table-1`` and then copy all files from ``/accumulo/tables/1/*/*.rf`` into that directory. Additionally,
-make a directory, ``/new-table-1-failures``, for any failures during the import process. Then, issue the import
-command using the Accumulo shell into the new table, telling Accumulo to not re-set the timestamp:
-
-\small
-\begin{verbatim}
-user@instance new_table> importdirectory /new-table-1 /new-table-1-failures false
-\end{verbatim}
-\normalsize
-
-Any RFiles which were failed to be loaded will be placed in ``/new-table-1-failures``. Rfiles that were successfully
-imported will no longer exist in ``/new-table-1``. For failures, move them back to the import directory and retry
-the ``importdirectory`` command.
-
-It is \textbf{extremely} important to note that this approach may introduce stale data back into
-the tables. For a few reasons, RFiles may exist in the table directory which are candidates for deletion but have
-not yet been deleted. Additionally, deleted data which was not compacted away, but still exists in write-ahead logs if
-the original instance was somehow recoverable, will be re-introduced in the new instance. Table splits and merges
-(which also include the deleteRows API call on TableOperations, are also vulnerable to this problem. This process should
-\textbf{not} be used if these are unacceptable risks. It is possible to try to re-create a view of the ``accumulo.metadata``
-table to prune out files that are candidates for deletion, but this is a difficult task that also may not be entirely accurate.
-
-Likewise, it is also possible that data loss may occur from write-ahead log (WAL) files which existed on the old table but
-were not minor-compacted into an RFile. Again, it may be possible to reconstruct the state of these WAL files to
-replay data not yet in an RFile; however, this is a difficult task and is not implemented in any automated fashion.
-
-A. The ``importdirectory`` shell command can be used to import RFiles from the old instance into a newly created instance,
-but extreme care should go into the decision to do this as it may result in reintroduction of stale data or the
-omission of new data.
-
-\section{File Naming Conventions}
-
-Q. Why are files named like they are? Why do some start with ``C'' and others with ``F''?
-
-A. The file names give you a basic idea for the source of the file.
-
-The base of the filename is a base-36 unique number. All filenames in accumulo are coordinated
-with a counter in zookeeper, so they are always unique, which is useful for debugging.
-
-The leading letter gives you an idea of how the file was created:
-
-\begin{itemize}
- \item F - Flush: entries in memory were written to a file (Minor Compaction)
- \item M - Merging compaction: entries in memory were combined with the smallest file to create one new file
- \item C - Several files, but not all files, were combined to produce this file (Major Compaction)
- \item A - All files were compacted, delete entries were dropped
- \item I - Bulk import, complete, sorted index files. Always in a directory starting with "b-"
-\end{itemize}
-
-This simple file naming convention allows you to see the basic structure of the files from just
-their filenames, and reason about what should be happening to them next, just
-by scanning their entries in the metadata tables.
-
-For example, if you see multiple files with ``M'' prefixes, the tablet is, or was, up against it's
-maximum file limit, so it began merging memory updates with files to keep the file count reasonable. This
-slows down ingest performance, so knowing there are many files like this tells you that the system
-is struggling to keep up with ingest vs the compaction strategy which reduces the number of files.
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/images/data_distribution.png
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/images/data_distribution.png b/docs/src/main/latex/accumulo_user_manual/images/data_distribution.png
deleted file mode 100644
index 7f18d3f..0000000
Binary files a/docs/src/main/latex/accumulo_user_manual/images/data_distribution.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/images/failure_handling.png
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/images/failure_handling.png b/docs/src/main/latex/accumulo_user_manual/images/failure_handling.png
deleted file mode 100644
index c131de6..0000000
Binary files a/docs/src/main/latex/accumulo_user_manual/images/failure_handling.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 78af9c5..2b92402 100644
--- a/pom.xml
+++ b/pom.xml
@@ -230,7 +230,7 @@
<artifactId>accumulo-docs</artifactId>
<version>${project.version}</version>
<classifier>user-manual</classifier>
- <type>pdf</type>
+ <type>html</type>
</dependency>
<dependency>
<groupId>org.apache.accumulo</groupId>
@@ -594,6 +594,11 @@
</configuration>
</plugin>
<plugin>
+ <groupId>org.asciidoctor</groupId>
+ <artifactId>asciidoctor-maven-plugin</artifactId>
+ <version>0.1.4</version>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<version>1.8</version>
@@ -621,11 +626,6 @@
<version>1.2.1</version>
</plugin>
<plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>latex-maven-plugin</artifactId>
- <version>1.1</version>
- </plugin>
- <plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
<version>1.0.0</version>
[19/50] [abbrv] ACCUMULO-2583 Change up the thrift and stub out a
service to choose a random tserver to service replication work.
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java
new file mode 100644
index 0000000..3d07b34
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java
@@ -0,0 +1,1010 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.accumulo.core.replication.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class RemoteReplicationCoordinator {
+
+ public interface Iface {
+
+ public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, org.apache.thrift.TException;
+
+ }
+
+ public interface AsyncIface {
+
+ public void getServicerAddress(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.getServicerAddress_call> resultHandler) throws org.apache.thrift.TException;
+
+ }
+
+ public static class Client extends org.apache.thrift.TServiceClient implements Iface {
+ public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
+ public Factory() {}
+ public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
+ return new Client(prot);
+ }
+ public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ return new Client(iprot, oprot);
+ }
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol prot)
+ {
+ super(prot, prot);
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ super(iprot, oprot);
+ }
+
+ public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, org.apache.thrift.TException
+ {
+ send_getServicerAddress(remoteTableId);
+ return recv_getServicerAddress();
+ }
+
+ public void send_getServicerAddress(int remoteTableId) throws org.apache.thrift.TException
+ {
+ getServicerAddress_args args = new getServicerAddress_args();
+ args.setRemoteTableId(remoteTableId);
+ sendBase("getServicerAddress", args);
+ }
+
+ public String recv_getServicerAddress() throws NoServersAvailableException, org.apache.thrift.TException
+ {
+ getServicerAddress_result result = new getServicerAddress_result();
+ receiveBase(result, "getServicerAddress");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getServicerAddress failed: unknown result");
+ }
+
+ }
+ public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
+ public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
+ private org.apache.thrift.async.TAsyncClientManager clientManager;
+ private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
+ public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
+ this.clientManager = clientManager;
+ this.protocolFactory = protocolFactory;
+ }
+ public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
+ return new AsyncClient(protocolFactory, clientManager, transport);
+ }
+ }
+
+ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
+ super(protocolFactory, clientManager, transport);
+ }
+
+ public void getServicerAddress(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<getServicerAddress_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ getServicerAddress_call method_call = new getServicerAddress_call(remoteTableId, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class getServicerAddress_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int remoteTableId;
+ public getServicerAddress_call(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<getServicerAddress_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.remoteTableId = remoteTableId;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getServicerAddress", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ getServicerAddress_args args = new getServicerAddress_args();
+ args.setRemoteTableId(remoteTableId);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public String getResult() throws NoServersAvailableException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_getServicerAddress();
+ }
+ }
+
+ }
+
+ public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
+ public Processor(I iface) {
+ super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
+ }
+
+ protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ processMap.put("getServicerAddress", new getServicerAddress());
+ return processMap;
+ }
+
+ public static class getServicerAddress<I extends Iface> extends org.apache.thrift.ProcessFunction<I, getServicerAddress_args> {
+ public getServicerAddress() {
+ super("getServicerAddress");
+ }
+
+ public getServicerAddress_args getEmptyArgsInstance() {
+ return new getServicerAddress_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public getServicerAddress_result getResult(I iface, getServicerAddress_args args) throws org.apache.thrift.TException {
+ getServicerAddress_result result = new getServicerAddress_result();
+ try {
+ result.success = iface.getServicerAddress(args.remoteTableId);
+ } catch (NoServersAvailableException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ }
+
+ public static class getServicerAddress_args implements org.apache.thrift.TBase<getServicerAddress_args, getServicerAddress_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getServicerAddress_args");
+
+ private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new getServicerAddress_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new getServicerAddress_argsTupleSchemeFactory());
+ }
+
+ public int remoteTableId; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ REMOTE_TABLE_ID((short)1, "remoteTableId");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // REMOTE_TABLE_ID
+ return REMOTE_TABLE_ID;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __REMOTETABLEID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getServicerAddress_args.class, metaDataMap);
+ }
+
+ public getServicerAddress_args() {
+ }
+
+ public getServicerAddress_args(
+ int remoteTableId)
+ {
+ this();
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public getServicerAddress_args(getServicerAddress_args other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.remoteTableId = other.remoteTableId;
+ }
+
+ public getServicerAddress_args deepCopy() {
+ return new getServicerAddress_args(this);
+ }
+
+ @Override
+ public void clear() {
+ setRemoteTableIdIsSet(false);
+ this.remoteTableId = 0;
+ }
+
+ public int getRemoteTableId() {
+ return this.remoteTableId;
+ }
+
+ public getServicerAddress_args setRemoteTableId(int remoteTableId) {
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ return this;
+ }
+
+ public void unsetRemoteTableId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
+ public boolean isSetRemoteTableId() {
+ return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ public void setRemoteTableIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ if (value == null) {
+ unsetRemoteTableId();
+ } else {
+ setRemoteTableId((Integer)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return Integer.valueOf(getRemoteTableId());
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return isSetRemoteTableId();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof getServicerAddress_args)
+ return this.equals((getServicerAddress_args)that);
+ return false;
+ }
+
+ public boolean equals(getServicerAddress_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_remoteTableId = true;
+ boolean that_present_remoteTableId = true;
+ if (this_present_remoteTableId || that_present_remoteTableId) {
+ if (!(this_present_remoteTableId && that_present_remoteTableId))
+ return false;
+ if (this.remoteTableId != that.remoteTableId)
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(getServicerAddress_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ getServicerAddress_args typedOther = (getServicerAddress_args)other;
+
+ lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRemoteTableId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("getServicerAddress_args(");
+ boolean first = true;
+
+ sb.append("remoteTableId:");
+ sb.append(this.remoteTableId);
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class getServicerAddress_argsStandardSchemeFactory implements SchemeFactory {
+ public getServicerAddress_argsStandardScheme getScheme() {
+ return new getServicerAddress_argsStandardScheme();
+ }
+ }
+
+ private static class getServicerAddress_argsStandardScheme extends StandardScheme<getServicerAddress_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // REMOTE_TABLE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
+ oprot.writeI32(struct.remoteTableId);
+ oprot.writeFieldEnd();
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class getServicerAddress_argsTupleSchemeFactory implements SchemeFactory {
+ public getServicerAddress_argsTupleScheme getScheme() {
+ return new getServicerAddress_argsTupleScheme();
+ }
+ }
+
+ private static class getServicerAddress_argsTupleScheme extends TupleScheme<getServicerAddress_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRemoteTableId()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetRemoteTableId()) {
+ oprot.writeI32(struct.remoteTableId);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class getServicerAddress_result implements org.apache.thrift.TBase<getServicerAddress_result, getServicerAddress_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getServicerAddress_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new getServicerAddress_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new getServicerAddress_resultTupleSchemeFactory());
+ }
+
+ public String success; // required
+ public NoServersAvailableException e; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success"),
+ E((short)1, "e");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ case 1: // E
+ return E;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getServicerAddress_result.class, metaDataMap);
+ }
+
+ public getServicerAddress_result() {
+ }
+
+ public getServicerAddress_result(
+ String success,
+ NoServersAvailableException e)
+ {
+ this();
+ this.success = success;
+ this.e = e;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public getServicerAddress_result(getServicerAddress_result other) {
+ if (other.isSetSuccess()) {
+ this.success = other.success;
+ }
+ if (other.isSetE()) {
+ this.e = new NoServersAvailableException(other.e);
+ }
+ }
+
+ public getServicerAddress_result deepCopy() {
+ return new getServicerAddress_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ this.e = null;
+ }
+
+ public String getSuccess() {
+ return this.success;
+ }
+
+ public getServicerAddress_result setSuccess(String success) {
+ this.success = success;
+ return this;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ public NoServersAvailableException getE() {
+ return this.e;
+ }
+
+ public getServicerAddress_result setE(NoServersAvailableException e) {
+ this.e = e;
+ return this;
+ }
+
+ public void unsetE() {
+ this.e = null;
+ }
+
+ /** Returns true if field e is set (has been assigned a value) and false otherwise */
+ public boolean isSetE() {
+ return this.e != null;
+ }
+
+ public void setEIsSet(boolean value) {
+ if (!value) {
+ this.e = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((String)value);
+ }
+ break;
+
+ case E:
+ if (value == null) {
+ unsetE();
+ } else {
+ setE((NoServersAvailableException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ case E:
+ return getE();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ case E:
+ return isSetE();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof getServicerAddress_result)
+ return this.equals((getServicerAddress_result)that);
+ return false;
+ }
+
+ public boolean equals(getServicerAddress_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ boolean this_present_e = true && this.isSetE();
+ boolean that_present_e = true && that.isSetE();
+ if (this_present_e || that_present_e) {
+ if (!(this_present_e && that_present_e))
+ return false;
+ if (!this.e.equals(that.e))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(getServicerAddress_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ getServicerAddress_result typedOther = (getServicerAddress_result)other;
+
+ lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetE()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("getServicerAddress_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.success);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("e:");
+ if (this.e == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.e);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class getServicerAddress_resultStandardSchemeFactory implements SchemeFactory {
+ public getServicerAddress_resultStandardScheme getScheme() {
+ return new getServicerAddress_resultStandardScheme();
+ }
+ }
+
+ private static class getServicerAddress_resultStandardScheme extends StandardScheme<getServicerAddress_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.success = iprot.readString();
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 1: // E
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.e = new NoServersAvailableException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ oprot.writeString(struct.success);
+ oprot.writeFieldEnd();
+ }
+ if (struct.e != null) {
+ oprot.writeFieldBegin(E_FIELD_DESC);
+ struct.e.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class getServicerAddress_resultTupleSchemeFactory implements SchemeFactory {
+ public getServicerAddress_resultTupleScheme getScheme() {
+ return new getServicerAddress_resultTupleScheme();
+ }
+ }
+
+ private static class getServicerAddress_resultTupleScheme extends TupleScheme<getServicerAddress_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ if (struct.isSetE()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetSuccess()) {
+ oprot.writeString(struct.success);
+ }
+ if (struct.isSetE()) {
+ struct.e.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.success = iprot.readString();
+ struct.setSuccessIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.e = new NoServersAvailableException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ }
+ }
+ }
+
+ }
+
+}
[47/50] [abbrv] git commit: ACCUMULO-2575 Fix bug in GC where files
were being prematurely deleted and add tests to catch
Posted by el...@apache.org.
ACCUMULO-2575 Fix bug in GC where files were being prematurely deleted and add tests to catch
After realizing the deadlock potential when not having tservers write the initial
replication entry to the metadata table, the garbage collection code was not updated
for the shift in data layout. On a clean system, when first beginning replication,
the replication table will not yet exist, but replication entries will exist in the
metadata table.
The GC code still assuming that the lack of a replication table implied that
there was no data pending replication and short-circuited early. We need to check
the metadata table always, and, when present, also the replication table.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/53fc90fc
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/53fc90fc
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/53fc90fc
Branch: refs/heads/ACCUMULO-378
Commit: 53fc90fcc783a37df666915cec45f3da3a290c7f
Parents: 440d7fc
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 16:16:33 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 16:16:33 2014 -0400
----------------------------------------------------------------------
.../gc/GarbageCollectWriteAheadLogs.java | 70 ++++++++++++++------
.../gc/GarbageCollectWriteAheadLogsTest.java | 68 ++++++++++++++++++-
.../replication/ReplicationWorkAssigner.java | 2 +
.../test/replication/ReplicationIT.java | 6 +-
.../test/replication/ReplicationWithGCIT.java | 60 ++++-------------
test/src/test/resources/log4j.properties | 2 +
6 files changed, 135 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fc90fc/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index 6e34e43..950bc12 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -66,16 +66,18 @@ import org.apache.accumulo.trace.instrument.Trace;
import org.apache.accumulo.trace.instrument.Tracer;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Iterables;
import com.google.common.net.HostAndPort;
import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.TextFormat;
public class GarbageCollectWriteAheadLogs {
- private static final Logger log = Logger.getLogger(GarbageCollectWriteAheadLogs.class);
+ private static final Logger log = LoggerFactory.getLogger(GarbageCollectWriteAheadLogs.class);
private final Instance instance;
private final VolumeManager fs;
@@ -191,7 +193,7 @@ public class GarbageCollectWriteAheadLogs {
} catch (KeeperException.NoNodeException ex) {
return false;
} catch (Exception ex) {
- log.debug(ex, ex);
+ log.debug(ex.toString(), ex);
return true;
}
}
@@ -349,13 +351,15 @@ public class GarbageCollectWriteAheadLogs {
Entry<String,Path> wal = walIter.next();
String fullPath = wal.getValue().toString();
if (neededByReplication(conn, fullPath)) {
- log.debug("Removing WAL from candidate deletion as it is still needed for replication: " + fullPath);
+ log.debug("Removing WAL from candidate deletion as it is still needed for replication: {} ", fullPath);
// If we haven't already removed it, check to see if this WAL is
// "in use" by replication (needed for replication purposes)
status.currentLog.inUse++;
walIter.remove();
sortedWALogs.remove(wal.getKey());
+ } else {
+ log.debug("WAL not needed for replication {}", fullPath);
}
count++;
}
@@ -370,13 +374,25 @@ public class GarbageCollectWriteAheadLogs {
*/
protected boolean neededByReplication(Connector conn, String wal) {
log.info("Checking replication table for " + wal);
- Iterable<Entry<Key,Value>> iter;
- try {
- iter = getReplicationStatusForFile(conn, wal);
- } catch (TableNotFoundException e) {
- log.trace("Replication table was not found");
- return false;
- }
+
+ // try {
+ // log.info("Current state of Metadata table");
+ // for (Entry<Key,Value> entry : conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
+ // log.info(entry.getKey().toStringNoTruncate() + "=" + TextFormat.shortDebugString(Status.parseFrom(entry.getValue().get())));
+ // }
+ // } catch (Exception e) {
+ // log.error("Could not read metadata table");
+ // }
+ // try {
+ // log.info("Current state of replication table");
+ // for (Entry<Key,Value> entry : conn.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
+ // log.info(entry.getKey().toStringNoTruncate() + "=" + TextFormat.shortDebugString(Status.parseFrom(entry.getValue().get())));
+ // }
+ // } catch (Exception e) {
+ // log.error("Could not read replication table");
+ // }
+
+ Iterable<Entry<Key,Value>> iter = getReplicationStatusForFile(conn, wal);
// TODO Push down this filter to the tserver to only return records
// that are not completely replicated and convert this loop into a
@@ -384,6 +400,7 @@ public class GarbageCollectWriteAheadLogs {
for (Entry<Key,Value> entry : iter) {
try {
Status status = Status.parseFrom(entry.getValue().get());
+ log.info("Checking if {} is safe for removal with {}", wal, TextFormat.shortDebugString(status));
if (!StatusUtil.isSafeForRemoval(status)) {
return true;
}
@@ -395,21 +412,34 @@ public class GarbageCollectWriteAheadLogs {
return false;
}
- protected Iterable<Entry<Key,Value>> getReplicationStatusForFile(Connector conn, String wal) throws TableNotFoundException {
- Scanner replScanner = ReplicationTable.getScanner(conn);
-
- // Scan only the Status records
- StatusSection.limit(replScanner);
- // Only look for this specific WAL
- replScanner.setRange(Range.exact(wal));
+ protected Iterable<Entry<Key,Value>> getReplicationStatusForFile(Connector conn, String wal) {
+ Scanner metaScanner;
+ try {
+ metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ } catch (TableNotFoundException e) {
+ throw new RuntimeException(e);
+ }
- Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
// Need to add in the replication section prefix
metaScanner.setRange(Range.exact(ReplicationSection.getRowPrefix() + wal));
// Limit the column family to be sure
metaScanner.fetchColumnFamily(ReplicationSection.COLF);
- return Iterables.concat(replScanner, metaScanner);
+ try {
+ Scanner replScanner = ReplicationTable.getScanner(conn);
+
+ // Scan only the Status records
+ StatusSection.limit(replScanner);
+
+ // Only look for this specific WAL
+ replScanner.setRange(Range.exact(wal));
+
+ return Iterables.concat(metaScanner, replScanner);
+ } catch (TableNotFoundException e) {
+ // do nothing
+ }
+
+ return metaScanner;
}
private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fc90fc/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
index 749fc96..1a3995b 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
@@ -29,17 +29,19 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.UUID;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.BatchWriterConfig;
import org.apache.accumulo.core.client.Connector;
import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.mock.MockInstance;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Key;
@@ -59,13 +61,13 @@ import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
public class GarbageCollectWriteAheadLogsTest {
@@ -313,7 +315,7 @@ public class GarbageCollectWriteAheadLogsTest {
}
@Override
- protected Iterable<Entry<Key,Value>> getReplicationStatusForFile(Connector conn, String wal) throws TableNotFoundException {
+ protected Iterable<Entry<Key,Value>> getReplicationStatusForFile(Connector conn, String wal) {
return this.replData;
}
}
@@ -434,4 +436,64 @@ public class GarbageCollectWriteAheadLogsTest {
// Both should have been deleted
Assert.assertEquals(0, nameToFileMap.size());
}
+
+ @Test
+ public void noReplicationTableDoesntLimitMetatdataResults() throws Exception {
+ Instance inst = new MockInstance(testName.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+
+ String wal = "hdfs://localhost:8020/accumulo/wal/tserver+port/123456-1234-1234-12345678";
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ Mutation m = new Mutation(ReplicationSection.getRowPrefix() + wal);
+ m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.newFileValue());
+ bw.addMutation(m);
+ bw.close();
+
+ GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(inst, volMgr, false);
+
+ Iterable<Entry<Key,Value>> data = gcWALs.getReplicationStatusForFile(conn, wal);
+ Entry<Key,Value> entry = Iterables.getOnlyElement(data);
+
+ Assert.assertEquals(ReplicationSection.getRowPrefix() + wal, entry.getKey().getRow().toString());
+ }
+
+ @Test
+ public void fetchesReplicationEntriesFromMetadataAndReplicationTables() throws Exception {
+ Instance inst = new MockInstance(testName.getMethodName());
+ Connector conn = inst.getConnector("root", new PasswordToken(""));
+ ReplicationTable.create(conn);
+
+ String wal = "hdfs://localhost:8020/accumulo/wal/tserver+port/123456-1234-1234-12345678";
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ Mutation m = new Mutation(ReplicationSection.getRowPrefix() + wal);
+ m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.newFileValue());
+ bw.addMutation(m);
+ bw.close();
+
+ bw = ReplicationTable.getBatchWriter(conn);
+ m = new Mutation(wal);
+ StatusSection.add(m, new Text("1"), StatusUtil.newFileValue());
+ bw.addMutation(m);
+ bw.close();
+
+ GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(inst, volMgr, false);
+
+ Iterable<Entry<Key,Value>> iter = gcWALs.getReplicationStatusForFile(conn, wal);
+ Map<Key,Value> data = new HashMap<>();
+ for (Entry<Key,Value> e : iter) {
+ data.put(e.getKey(), e.getValue());
+ }
+
+ Assert.assertEquals(2, data.size());
+
+ // Should get one element from each table (metadata and replication)
+ for (Key k : data.keySet()) {
+ String row = k.getRow().toString();
+ if (row.startsWith(ReplicationSection.getRowPrefix())) {
+ Assert.assertTrue(row.endsWith(wal));
+ } else {
+ Assert.assertEquals(wal, row);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fc90fc/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
index a6607e5..24842a9 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -177,6 +177,8 @@ public class ReplicationWorkAssigner extends Daemon {
// Keep the state of the work we queued correct
cleanupFinishedWork();
+
+ UtilWaitThread.sleep(conf.getTimeInMillis(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fc90fc/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
index d592716..0d4099c 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@ -108,11 +108,11 @@ public class ReplicationIT extends ConfigurableMacIT {
Scanner s = ReplicationTable.getScanner(connMaster);
for (Entry<Key,Value> e : s) {
- System.out.println(e.getKey().toStringNoTruncate() + " " + TextFormat.shortDebugString(Status.parseFrom(e.getValue().get())));
+ log.info(e.getKey().toStringNoTruncate() + " " + TextFormat.shortDebugString(Status.parseFrom(e.getValue().get())));
}
- System.out.println();
- System.out.println();
+ log.info("");
+ log.info("");
Thread.sleep(1000);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fc90fc/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
index 92f6fb3..b20a3bd 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationWithGCIT.java
@@ -55,6 +55,7 @@ import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.accumulo.tserver.TabletServer;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.io.Text;
@@ -62,6 +63,7 @@ import org.junit.Assert;
import org.junit.Test;
import com.google.common.collect.Iterables;
+import com.google.protobuf.TextFormat;
/**
*
@@ -205,18 +207,6 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
}
bw.close();
- // System.out.println("**** WALs from metadata");
- // for (String metadataWal : metadataWals) {
- // System.out.println(metadataWal);
- // }
- //
- // System.out.println("**** WALs from replication");
- // s = ReplicationTable.getScanner(conn);
- // StatusSection.limit(s);
- // for (Entry<Key,Value> entry : s) {
- // System.out.println(entry.getKey().toStringNoTruncate() + " " + Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
- // }
-
s = ReplicationTable.getScanner(conn);
StatusSection.limit(s);
bw = conn.createBatchWriter(ReplicationTable.NAME, new BatchWriterConfig());
@@ -231,13 +221,6 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
}
bw.close();
- // System.out.println("**** WALs from replication");
- // s = ReplicationTable.getScanner(conn);
- // StatusSection.limit(s);
- // for (Entry<Key,Value> entry : s) {
- // System.out.println(entry.getKey().toStringNoTruncate() + " " + Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
- // }
-
// Kill the tserver(s) and restart them
// to ensure that the WALs we previously observed all move to closed.
for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
@@ -254,24 +237,6 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
Entry<Key,Value> entry : s) {}
}
- // for (int i = 0; i < 5; i++) {
- // s = conn.createScanner(MetadataTable.NAME, new Authorizations());
- // s.setRange(ReplicationSection.getRange());
- // System.out.println("**** Metadata");
- // for (Entry<Key,Value> entry : s) {
- // System.out.println(entry.getKey().toStringNoTruncate() + " " + Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
- // }
- //
- // s = ReplicationTable.getScanner(conn);
- // StatusSection.limit(s);
- // System.out.println("**** Replication status");
- // for (Entry<Key,Value> entry : s) {
- // System.out.println(entry.getKey().toStringNoTruncate() + " " + Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
- // }
- //
- // Thread.sleep(1000);
- // }
-
try {
boolean allClosed = true;
for (int i = 0; i < 10; i++) {
@@ -306,18 +271,10 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
gc.waitFor();
}
- // System.out.println("****** Replication table iterators");
- // System.out.println(conn.tableOperations().listIterators(ReplicationTable.NAME));
- // for (Entry<String,String> entry : conn.tableOperations().getProperties(ReplicationTable.NAME)) {
- // System.out.println(entry.getKey()+ "=" + entry.getValue());
- // }
- // System.out.println();
-
- System.out.println("****** Final Replication logs before failure");
s = ReplicationTable.getScanner(conn);
StatusSection.limit(s);
for (Entry<Key,Value> entry : s) {
- System.out.println(entry.getKey().toStringNoTruncate() + " " + Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
+ log.info(entry.getKey().toStringNoTruncate() + " " + TextFormat.shortDebugString(Status.parseFrom(entry.getValue().get())));
}
Assert.fail("Expected all replication records to be closed");
}
@@ -325,6 +282,7 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
@Test
public void replicatedStatusEntriesAreDeleted() throws Exception {
Connector conn = getConnector();
+ FileSystem fs = FileSystem.getLocal(new Configuration());
String table1 = "table1";
// replication shouldn't exist when we begin
@@ -469,7 +427,15 @@ public class ReplicationWithGCIT extends ConfigurableMacIT {
Thread.sleep(1000);
}
- Assert.assertTrue("Did nto find any replication entries in the replication table", foundResults);
+ Assert.assertTrue("Did not find any replication entries in the replication table", foundResults);
+
+ s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ s.setRange(ReplicationSection.getRange());
+ for (Entry<Key,Value> entry : s) {
+ String row = entry.getKey().getRow().toString();
+ Path file = new Path(row.substring(ReplicationSection.getRowPrefix().length()));
+ Assert.assertTrue(file + " did not exist when it should", fs.exists(file));
+ }
/**
* After recovery completes, we should have unreplicated, closed Status messages. The close happens at the beginning of log recovery.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/53fc90fc/test/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/test/src/test/resources/log4j.properties b/test/src/test/resources/log4j.properties
index 71c06e9..dd382f7 100644
--- a/test/src/test/resources/log4j.properties
+++ b/test/src/test/resources/log4j.properties
@@ -34,3 +34,5 @@ log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO
log4j.logger.org.apache.accumulo.server.util.ReplicationTableUtil=TRACE
log4j.logger.org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator=INFO
log4j.logger.org.apache.accumulo.core.client.impl.ThriftScanner=INFO
+log4j.logger.org.apache.accumulo.server.zookeeper.DistributedWorkQueue=INFO
+log4j.logger.org.apache.accumulo.fate.zookeeper.DistributedReadWriteLock=WARN
\ No newline at end of file
[02/50] [abbrv] git commit: ACCUMULO-2760 just wait for the tserver
to die if the wait was long; timeout means it failed to die
Posted by el...@apache.org.
ACCUMULO-2760 just wait for the tserver to die if the wait was long; timeout means it failed to die
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/b3c89367
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/b3c89367
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/b3c89367
Branch: refs/heads/ACCUMULO-378
Commit: b3c893671ff039836251fc6c33b9522690a4dda5
Parents: 15e4f00
Author: Eric C. Newton <er...@gmail.com>
Authored: Wed Apr 30 12:15:38 2014 -0400
Committer: Eric C. Newton <er...@gmail.com>
Committed: Wed Apr 30 12:15:38 2014 -0400
----------------------------------------------------------------------
.../accumulo/test/functional/HalfDeadTServerIT.java | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/b3c89367/test/src/test/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java b/test/src/test/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
index d235545..af7b329 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/HalfDeadTServerIT.java
@@ -97,7 +97,7 @@ public class HalfDeadTServerIT extends ConfigurableMacIT {
@Test
public void testTimeout() throws Exception {
- String results = test(40);
+ String results = test(20);
if (results != null) {
if (!results.contains("Session expired")) {
System.out.println("Failed to find Session expired in");
@@ -164,21 +164,24 @@ public class HalfDeadTServerIT extends ConfigurableMacIT {
VerifyIngest.verifyIngest(c, vopts, SOPTS);
} else {
UtilWaitThread.sleep(5 * 1000);
+ tserver.waitFor();
+ t.join();
+ tserver = null;
}
// verify the process was blocked
String results = t.toString();
assertTrue(results.contains("sleeping\nsleeping\nsleeping\n"));
- assertTrue(results.contains("Zookeeper error, will retry"));
return results;
} finally {
if (ingest != null) {
ingest.destroy();
ingest.waitFor();
}
- tserver.destroy();
- tserver.waitFor();
- t.join();
- UtilWaitThread.sleep(1000);
+ if (tserver != null) {
+ tserver.destroy();
+ tserver.waitFor();
+ t.join();
+ }
}
}
[46/50] [abbrv] git commit: ACCUMULO-378 Tune down some aggressive
logging
Posted by el...@apache.org.
ACCUMULO-378 Tune down some aggressive logging
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/440d7fcd
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/440d7fcd
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/440d7fcd
Branch: refs/heads/ACCUMULO-378
Commit: 440d7fcd498eb48a5e665dcb73ad795f3ba547cc
Parents: b18e0ee
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 16:15:35 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 16:15:35 2014 -0400
----------------------------------------------------------------------
.../accumulo/gc/replication/RemoveCompleteReplicationRecords.java | 2 +-
.../accumulo/master/replication/ReplicationWorkAssigner.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/440d7fcd/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java b/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
index b46416c..6287aef 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
@@ -77,7 +77,7 @@ public class RemoveCompleteReplicationRecords implements Runnable {
metaBw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
replBw = conn.createBatchWriter(ReplicationTable.NAME, new BatchWriterConfig());
} catch (TableNotFoundException e) {
- log.error("Replication table was deleted", e);
+ log.debug("Not attempting to remove complete replication records as the replication table doesn't yet exist");
return;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/440d7fcd/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
index dddbf58..a6607e5 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -227,7 +227,7 @@ public class ReplicationWorkAssigner extends Daemon {
if (!queuedWork.contains(key)) {
queueWork(key, file);
} else {
- log.debug("Not re-queueing work for {}", key);
+ log.trace("Not re-queueing work for {}", key);
}
} else {
log.debug("Not queueing work for {} because [{}] doesn't need replication", file, TextFormat.shortDebugString(status));
[13/50] [abbrv] git commit: Merge remote-tracking branch
'origin/1.6.0-SNAPSHOT' into 1.6.1-SNAPSHOT
Posted by el...@apache.org.
Merge remote-tracking branch 'origin/1.6.0-SNAPSHOT' into 1.6.1-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4d5ba9da
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4d5ba9da
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4d5ba9da
Branch: refs/heads/ACCUMULO-378
Commit: 4d5ba9daf0eedbd14df0e8d49a0fd3e38a8fa48f
Parents: 18c0e8e 1c38957
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri May 2 18:18:34 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri May 2 18:18:34 2014 -0400
----------------------------------------------------------------------
pom.xml | 3 ++
.../test/functional/HalfDeadTServerIT.java | 15 +++---
.../accumulo/test/randomwalk/FrameworkTest.java | 48 ++++++++++----------
3 files changed, 36 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/4d5ba9da/pom.xml
----------------------------------------------------------------------
[44/50] [abbrv] git commit: ACCUMULO-2575 Also check metadata table
to account for status updates that haven't made it to replication table yet
Posted by el...@apache.org.
ACCUMULO-2575 Also check metadata table to account for status updates that haven't made it to replication table yet
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/2372c71d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/2372c71d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/2372c71d
Branch: refs/heads/ACCUMULO-378
Commit: 2372c71df8ea7aa5299a93481d6b84088cc85049
Parents: b5cd35a
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 13:26:09 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 13:26:09 2014 -0400
----------------------------------------------------------------------
.../gc/GarbageCollectWriteAheadLogs.java | 18 +++--
.../gc/GarbageCollectWriteAheadLogsTest.java | 76 ++++++++++++++++++--
2 files changed, 86 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2372c71d/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
index c373dd7..6e34e43 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogs.java
@@ -41,9 +41,12 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.gc.thrift.GCStatus;
import org.apache.accumulo.core.gc.thrift.GcCycleStats;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
@@ -67,6 +70,7 @@ import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.zookeeper.KeeperException;
+import com.google.common.collect.Iterables;
import com.google.common.net.HostAndPort;
import com.google.protobuf.InvalidProtocolBufferException;
@@ -392,14 +396,20 @@ public class GarbageCollectWriteAheadLogs {
}
protected Iterable<Entry<Key,Value>> getReplicationStatusForFile(Connector conn, String wal) throws TableNotFoundException {
- Scanner s = ReplicationTable.getScanner(conn);
+ Scanner replScanner = ReplicationTable.getScanner(conn);
// Scan only the Status records
- StatusSection.limit(s);
+ StatusSection.limit(replScanner);
// Only look for this specific WAL
- s.setRange(Range.exact(wal));
+ replScanner.setRange(Range.exact(wal));
- return s;
+ Scanner metaScanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
+ // Need to add in the replication section prefix
+ metaScanner.setRange(Range.exact(ReplicationSection.getRowPrefix() + wal));
+ // Limit the column family to be sure
+ metaScanner.fetchColumnFamily(ReplicationSection.COLF);
+
+ return Iterables.concat(replScanner, metaScanner);
}
private int scanServers(Map<Path,String> fileToServerMap, Map<String,Path> nameToFileMap) throws Exception {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/2372c71d/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
----------------------------------------------------------------------
diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
index 11390f5..749fc96 100644
--- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
+++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectWriteAheadLogsTest.java
@@ -46,6 +46,9 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.gc.thrift.GCStatus;
+import org.apache.accumulo.core.gc.thrift.GcCycleStats;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.ReplicationSection;
import org.apache.accumulo.core.protobuf.ProtobufUtil;
import org.apache.accumulo.core.replication.ReplicationSchema.StatusSection;
import org.apache.accumulo.core.replication.StatusUtil;
@@ -56,9 +59,12 @@ import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import com.google.common.collect.Maps;
@@ -76,7 +82,9 @@ public class GarbageCollectWriteAheadLogsTest {
private VolumeManager volMgr;
private GarbageCollectWriteAheadLogs gcwal;
private long modTime;
- private GCStatus status;
+
+ @Rule
+ public TestName testName = new TestName();
@Before
public void setUp() throws Exception {
@@ -84,7 +92,6 @@ public class GarbageCollectWriteAheadLogsTest {
volMgr = createMock(VolumeManager.class);
gcwal = new GarbageCollectWriteAheadLogs(instance, volMgr, false);
modTime = System.currentTimeMillis();
- status = createMock(GCStatus.class);
}
@Test
@@ -290,6 +297,7 @@ public class GarbageCollectWriteAheadLogsTest {
assertFalse(GarbageCollectWriteAheadLogs.isUUID(null));
}
+ // It was easier to do this than get the mocking working for me
private static class ReplicationGCWAL extends GarbageCollectWriteAheadLogs {
private List<Entry<Key,Value>> replData;
@@ -345,9 +353,11 @@ public class GarbageCollectWriteAheadLogsTest {
public void removeReplicationEntries() throws Exception {
String file1 = UUID.randomUUID().toString(), file2 = UUID.randomUUID().toString();
- Instance inst = new MockInstance("removeReplicationEntries");
+ Instance inst = new MockInstance(testName.getMethodName());
Credentials creds = new Credentials("root", new PasswordToken(""));
Connector conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+
+ GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(inst, volMgr, false);
ReplicationTable.create(conn);
@@ -359,11 +369,69 @@ public class GarbageCollectWriteAheadLogsTest {
StatusSection.add(m, new Text("1"), StatusUtil.newFileValue());
bw.addMutation(m);
+ // These WALs are potential candidates for deletion from fs
Map<String,Path> nameToFileMap = new HashMap<>();
+ nameToFileMap.put(file1, new Path("/wals/" + file1));
+ nameToFileMap.put(file2, new Path("/wals/" + file2));
+
Map<String,Path> sortedWALogs = Collections.emptyMap();
- gcwal.removeReplicationEntries(nameToFileMap, sortedWALogs, this.status, creds);
+ // Make the GCStatus and GcCycleStats
+ GCStatus status = new GCStatus();
+ GcCycleStats cycleStats = new GcCycleStats();
+ status.currentLog = cycleStats;
+
+ // We should iterate over two entries
+ Assert.assertEquals(2, gcWALs.removeReplicationEntries(nameToFileMap, sortedWALogs, status, creds));
+
+ // We should have noted that two files were still in use
+ Assert.assertEquals(2l, cycleStats.inUse);
+
+ // Both should have been deleted
+ Assert.assertEquals(0, nameToFileMap.size());
+ }
+
+ @Test
+ public void replicationEntriesOnlyInMetaPreventGC() throws Exception {
+ String file1 = UUID.randomUUID().toString(), file2 = UUID.randomUUID().toString();
+
+ Instance inst = new MockInstance(testName.getMethodName());
+ Credentials creds = new Credentials("root", new PasswordToken(""));
+ Connector conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+
+ GarbageCollectWriteAheadLogs gcWALs = new GarbageCollectWriteAheadLogs(inst, volMgr, false);
+
+ ReplicationTable.create(conn);
+
+ // Write some records to the metadata table, we haven't yet written status records to the replication table
+ BatchWriter bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ Mutation m = new Mutation(ReplicationSection.getRowPrefix() + "/wals/" + file1);
+ m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.newFileValue());
+ bw.addMutation(m);
+
+ m = new Mutation(ReplicationSection.getRowPrefix() + "/wals/" + file2);
+ m.put(ReplicationSection.COLF, new Text("1"), StatusUtil.newFileValue());
+ bw.addMutation(m);
+
+ // These WALs are potential candidates for deletion from fs
+ Map<String,Path> nameToFileMap = new HashMap<>();
+ nameToFileMap.put(file1, new Path("/wals/" + file1));
+ nameToFileMap.put(file2, new Path("/wals/" + file2));
+
+ Map<String,Path> sortedWALogs = Collections.emptyMap();
+
+ // Make the GCStatus and GcCycleStats objects
+ GCStatus status = new GCStatus();
+ GcCycleStats cycleStats = new GcCycleStats();
+ status.currentLog = cycleStats;
+
+ // We should iterate over two entries
+ Assert.assertEquals(2, gcWALs.removeReplicationEntries(nameToFileMap, sortedWALogs, status, creds));
+
+ // We should have noted that two files were still in use
+ Assert.assertEquals(2l, cycleStats.inUse);
+ // Both should have been deleted
Assert.assertEquals(0, nameToFileMap.size());
}
}
[20/50] [abbrv] git commit: ACCUMULO-2583 Change up the thrift and
stub out a service to choose a random tserver to service replication work.
Posted by el...@apache.org.
ACCUMULO-2583 Change up the thrift and stub out a service to choose a random tserver to service replication work.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3e21a41d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3e21a41d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3e21a41d
Branch: refs/heads/ACCUMULO-378
Commit: 3e21a41d4e4b4248e2765b789566e61a85b427dc
Parents: e9a75d1
Author: Josh Elser <el...@apache.org>
Authored: Sun May 4 00:55:18 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sun May 4 00:55:18 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 2 +
.../thrift/NoServersAvailableException.java | 500 +++++
.../replication/thrift/RemoteReplication.java | 1917 ------------------
.../thrift/RemoteReplicationCoordinator.java | 1010 +++++++++
.../thrift/RemoteReplicationServicer.java | 1917 ++++++++++++++++++
core/src/main/thrift/replication.thrift | 11 +-
.../java/org/apache/accumulo/master/Master.java | 2 +
.../MasterReplicationCoordinator.java | 76 +
.../MasterReplicationCoordinatorTest.java | 70 +
9 files changed, 3587 insertions(+), 1918 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index db810be..30835b9 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -460,6 +460,8 @@ public enum Property {
REPLICATION_WORK_ASSIGNMENT_SLEEP("replication.work.assignment.sleep", "30s", PropertyType.TIMEDURATION, "Amount of time to sleep between replication work assignment"),
@Experimental
REPLICATION_WORKER_THREADS("replication.worker.threads", "4", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
+ @Experimental
+ REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "1001", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/core/src/main/java/org/apache/accumulo/core/replication/thrift/NoServersAvailableException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/NoServersAvailableException.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/NoServersAvailableException.java
new file mode 100644
index 0000000..3d2b559
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/replication/thrift/NoServersAvailableException.java
@@ -0,0 +1,500 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.accumulo.core.replication.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class NoServersAvailableException extends TException implements org.apache.thrift.TBase<NoServersAvailableException, NoServersAvailableException._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("NoServersAvailableException");
+
+ private static final org.apache.thrift.protocol.TField CODE_FIELD_DESC = new org.apache.thrift.protocol.TField("code", org.apache.thrift.protocol.TType.I32, (short)1);
+ private static final org.apache.thrift.protocol.TField REASON_FIELD_DESC = new org.apache.thrift.protocol.TField("reason", org.apache.thrift.protocol.TType.STRING, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new NoServersAvailableExceptionStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new NoServersAvailableExceptionTupleSchemeFactory());
+ }
+
+ public int code; // required
+ public String reason; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ CODE((short)1, "code"),
+ REASON((short)2, "reason");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // CODE
+ return CODE;
+ case 2: // REASON
+ return REASON;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __CODE_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.CODE, new org.apache.thrift.meta_data.FieldMetaData("code", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.REASON, new org.apache.thrift.meta_data.FieldMetaData("reason", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(NoServersAvailableException.class, metaDataMap);
+ }
+
+ public NoServersAvailableException() {
+ }
+
+ public NoServersAvailableException(
+ int code,
+ String reason)
+ {
+ this();
+ this.code = code;
+ setCodeIsSet(true);
+ this.reason = reason;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public NoServersAvailableException(NoServersAvailableException other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.code = other.code;
+ if (other.isSetReason()) {
+ this.reason = other.reason;
+ }
+ }
+
+ public NoServersAvailableException deepCopy() {
+ return new NoServersAvailableException(this);
+ }
+
+ @Override
+ public void clear() {
+ setCodeIsSet(false);
+ this.code = 0;
+ this.reason = null;
+ }
+
+ public int getCode() {
+ return this.code;
+ }
+
+ public NoServersAvailableException setCode(int code) {
+ this.code = code;
+ setCodeIsSet(true);
+ return this;
+ }
+
+ public void unsetCode() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __CODE_ISSET_ID);
+ }
+
+ /** Returns true if field code is set (has been assigned a value) and false otherwise */
+ public boolean isSetCode() {
+ return EncodingUtils.testBit(__isset_bitfield, __CODE_ISSET_ID);
+ }
+
+ public void setCodeIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __CODE_ISSET_ID, value);
+ }
+
+ public String getReason() {
+ return this.reason;
+ }
+
+ public NoServersAvailableException setReason(String reason) {
+ this.reason = reason;
+ return this;
+ }
+
+ public void unsetReason() {
+ this.reason = null;
+ }
+
+ /** Returns true if field reason is set (has been assigned a value) and false otherwise */
+ public boolean isSetReason() {
+ return this.reason != null;
+ }
+
+ public void setReasonIsSet(boolean value) {
+ if (!value) {
+ this.reason = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case CODE:
+ if (value == null) {
+ unsetCode();
+ } else {
+ setCode((Integer)value);
+ }
+ break;
+
+ case REASON:
+ if (value == null) {
+ unsetReason();
+ } else {
+ setReason((String)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case CODE:
+ return Integer.valueOf(getCode());
+
+ case REASON:
+ return getReason();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case CODE:
+ return isSetCode();
+ case REASON:
+ return isSetReason();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof NoServersAvailableException)
+ return this.equals((NoServersAvailableException)that);
+ return false;
+ }
+
+ public boolean equals(NoServersAvailableException that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_code = true;
+ boolean that_present_code = true;
+ if (this_present_code || that_present_code) {
+ if (!(this_present_code && that_present_code))
+ return false;
+ if (this.code != that.code)
+ return false;
+ }
+
+ boolean this_present_reason = true && this.isSetReason();
+ boolean that_present_reason = true && that.isSetReason();
+ if (this_present_reason || that_present_reason) {
+ if (!(this_present_reason && that_present_reason))
+ return false;
+ if (!this.reason.equals(that.reason))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(NoServersAvailableException other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ NoServersAvailableException typedOther = (NoServersAvailableException)other;
+
+ lastComparison = Boolean.valueOf(isSetCode()).compareTo(typedOther.isSetCode());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetCode()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.code, typedOther.code);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetReason()).compareTo(typedOther.isSetReason());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetReason()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.reason, typedOther.reason);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("NoServersAvailableException(");
+ boolean first = true;
+
+ sb.append("code:");
+ sb.append(this.code);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("reason:");
+ if (this.reason == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.reason);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class NoServersAvailableExceptionStandardSchemeFactory implements SchemeFactory {
+ public NoServersAvailableExceptionStandardScheme getScheme() {
+ return new NoServersAvailableExceptionStandardScheme();
+ }
+ }
+
+ private static class NoServersAvailableExceptionStandardScheme extends StandardScheme<NoServersAvailableException> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, NoServersAvailableException struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // CODE
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.code = iprot.readI32();
+ struct.setCodeIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // REASON
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.reason = iprot.readString();
+ struct.setReasonIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, NoServersAvailableException struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(CODE_FIELD_DESC);
+ oprot.writeI32(struct.code);
+ oprot.writeFieldEnd();
+ if (struct.reason != null) {
+ oprot.writeFieldBegin(REASON_FIELD_DESC);
+ oprot.writeString(struct.reason);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class NoServersAvailableExceptionTupleSchemeFactory implements SchemeFactory {
+ public NoServersAvailableExceptionTupleScheme getScheme() {
+ return new NoServersAvailableExceptionTupleScheme();
+ }
+ }
+
+ private static class NoServersAvailableExceptionTupleScheme extends TupleScheme<NoServersAvailableException> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, NoServersAvailableException struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetCode()) {
+ optionals.set(0);
+ }
+ if (struct.isSetReason()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetCode()) {
+ oprot.writeI32(struct.code);
+ }
+ if (struct.isSetReason()) {
+ oprot.writeString(struct.reason);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, NoServersAvailableException struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.code = iprot.readI32();
+ struct.setCodeIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.reason = iprot.readString();
+ struct.setReasonIsSet(true);
+ }
+ }
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplication.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplication.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplication.java
deleted file mode 100644
index e82240f..0000000
--- a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplication.java
+++ /dev/null
@@ -1,1917 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.accumulo.core.replication.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings("all") public class RemoteReplication {
-
- public interface Iface {
-
- public void replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException;
-
- public void replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateLog_call> resultHandler) throws org.apache.thrift.TException;
-
- public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public void replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException
- {
- send_replicateLog(remoteTableId, data);
- recv_replicateLog();
- }
-
- public void send_replicateLog(int remoteTableId, WalEdits data) throws org.apache.thrift.TException
- {
- replicateLog_args args = new replicateLog_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- sendBase("replicateLog", args);
- }
-
- public void recv_replicateLog() throws RemoteReplicationException, org.apache.thrift.TException
- {
- replicateLog_result result = new replicateLog_result();
- receiveBase(result, "replicateLog");
- if (result.e != null) {
- throw result.e;
- }
- return;
- }
-
- public void replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException
- {
- send_replicateKeyValues(remoteTableId, data);
- recv_replicateKeyValues();
- }
-
- public void send_replicateKeyValues(int remoteTableId, KeyValues data) throws org.apache.thrift.TException
- {
- replicateKeyValues_args args = new replicateKeyValues_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- sendBase("replicateKeyValues", args);
- }
-
- public void recv_replicateKeyValues() throws RemoteReplicationException, org.apache.thrift.TException
- {
- replicateKeyValues_result result = new replicateKeyValues_result();
- receiveBase(result, "replicateKeyValues");
- if (result.e != null) {
- throw result.e;
- }
- return;
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- replicateLog_call method_call = new replicateLog_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class replicateLog_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int remoteTableId;
- private WalEdits data;
- public replicateLog_call(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.remoteTableId = remoteTableId;
- this.data = data;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateLog", org.apache.thrift.protocol.TMessageType.CALL, 0));
- replicateLog_args args = new replicateLog_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws RemoteReplicationException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_replicateLog();
- }
- }
-
- public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- replicateKeyValues_call method_call = new replicateKeyValues_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class replicateKeyValues_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int remoteTableId;
- private KeyValues data;
- public replicateKeyValues_call(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.remoteTableId = remoteTableId;
- this.data = data;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateKeyValues", org.apache.thrift.protocol.TMessageType.CALL, 0));
- replicateKeyValues_args args = new replicateKeyValues_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws RemoteReplicationException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_replicateKeyValues();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("replicateLog", new replicateLog());
- processMap.put("replicateKeyValues", new replicateKeyValues());
- return processMap;
- }
-
- public static class replicateLog<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateLog_args> {
- public replicateLog() {
- super("replicateLog");
- }
-
- public replicateLog_args getEmptyArgsInstance() {
- return new replicateLog_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public replicateLog_result getResult(I iface, replicateLog_args args) throws org.apache.thrift.TException {
- replicateLog_result result = new replicateLog_result();
- try {
- iface.replicateLog(args.remoteTableId, args.data);
- } catch (RemoteReplicationException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- public static class replicateKeyValues<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateKeyValues_args> {
- public replicateKeyValues() {
- super("replicateKeyValues");
- }
-
- public replicateKeyValues_args getEmptyArgsInstance() {
- return new replicateKeyValues_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public replicateKeyValues_result getResult(I iface, replicateKeyValues_args args) throws org.apache.thrift.TException {
- replicateKeyValues_result result = new replicateKeyValues_result();
- try {
- iface.replicateKeyValues(args.remoteTableId, args.data);
- } catch (RemoteReplicationException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- }
-
- public static class replicateLog_args implements org.apache.thrift.TBase<replicateLog_args, replicateLog_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_args");
-
- private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateLog_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateLog_argsTupleSchemeFactory());
- }
-
- public int remoteTableId; // required
- public WalEdits data; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- REMOTE_TABLE_ID((short)1, "remoteTableId"),
- DATA((short)2, "data");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // REMOTE_TABLE_ID
- return REMOTE_TABLE_ID;
- case 2: // DATA
- return DATA;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REMOTETABLEID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, WalEdits.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_args.class, metaDataMap);
- }
-
- public replicateLog_args() {
- }
-
- public replicateLog_args(
- int remoteTableId,
- WalEdits data)
- {
- this();
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- this.data = data;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateLog_args(replicateLog_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.remoteTableId = other.remoteTableId;
- if (other.isSetData()) {
- this.data = new WalEdits(other.data);
- }
- }
-
- public replicateLog_args deepCopy() {
- return new replicateLog_args(this);
- }
-
- @Override
- public void clear() {
- setRemoteTableIdIsSet(false);
- this.remoteTableId = 0;
- this.data = null;
- }
-
- public int getRemoteTableId() {
- return this.remoteTableId;
- }
-
- public replicateLog_args setRemoteTableId(int remoteTableId) {
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- return this;
- }
-
- public void unsetRemoteTableId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
- public boolean isSetRemoteTableId() {
- return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- public void setRemoteTableIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
- }
-
- public WalEdits getData() {
- return this.data;
- }
-
- public replicateLog_args setData(WalEdits data) {
- this.data = data;
- return this;
- }
-
- public void unsetData() {
- this.data = null;
- }
-
- /** Returns true if field data is set (has been assigned a value) and false otherwise */
- public boolean isSetData() {
- return this.data != null;
- }
-
- public void setDataIsSet(boolean value) {
- if (!value) {
- this.data = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case REMOTE_TABLE_ID:
- if (value == null) {
- unsetRemoteTableId();
- } else {
- setRemoteTableId((Integer)value);
- }
- break;
-
- case DATA:
- if (value == null) {
- unsetData();
- } else {
- setData((WalEdits)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case REMOTE_TABLE_ID:
- return Integer.valueOf(getRemoteTableId());
-
- case DATA:
- return getData();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case REMOTE_TABLE_ID:
- return isSetRemoteTableId();
- case DATA:
- return isSetData();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateLog_args)
- return this.equals((replicateLog_args)that);
- return false;
- }
-
- public boolean equals(replicateLog_args that) {
- if (that == null)
- return false;
-
- boolean this_present_remoteTableId = true;
- boolean that_present_remoteTableId = true;
- if (this_present_remoteTableId || that_present_remoteTableId) {
- if (!(this_present_remoteTableId && that_present_remoteTableId))
- return false;
- if (this.remoteTableId != that.remoteTableId)
- return false;
- }
-
- boolean this_present_data = true && this.isSetData();
- boolean that_present_data = true && that.isSetData();
- if (this_present_data || that_present_data) {
- if (!(this_present_data && that_present_data))
- return false;
- if (!this.data.equals(that.data))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateLog_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateLog_args typedOther = (replicateLog_args)other;
-
- lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRemoteTableId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetData()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateLog_args(");
- boolean first = true;
-
- sb.append("remoteTableId:");
- sb.append(this.remoteTableId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("data:");
- if (this.data == null) {
- sb.append("null");
- } else {
- sb.append(this.data);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (data != null) {
- data.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateLog_argsStandardSchemeFactory implements SchemeFactory {
- public replicateLog_argsStandardScheme getScheme() {
- return new replicateLog_argsStandardScheme();
- }
- }
-
- private static class replicateLog_argsStandardScheme extends StandardScheme<replicateLog_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // REMOTE_TABLE_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // DATA
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.data = new WalEdits();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
- oprot.writeI32(struct.remoteTableId);
- oprot.writeFieldEnd();
- if (struct.data != null) {
- oprot.writeFieldBegin(DATA_FIELD_DESC);
- struct.data.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateLog_argsTupleSchemeFactory implements SchemeFactory {
- public replicateLog_argsTupleScheme getScheme() {
- return new replicateLog_argsTupleScheme();
- }
- }
-
- private static class replicateLog_argsTupleScheme extends TupleScheme<replicateLog_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetRemoteTableId()) {
- optionals.set(0);
- }
- if (struct.isSetData()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetRemoteTableId()) {
- oprot.writeI32(struct.remoteTableId);
- }
- if (struct.isSetData()) {
- struct.data.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- }
- if (incoming.get(1)) {
- struct.data = new WalEdits();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateLog_result implements org.apache.thrift.TBase<replicateLog_result, replicateLog_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_result");
-
- private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateLog_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateLog_resultTupleSchemeFactory());
- }
-
- public RemoteReplicationException e; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- E((short)1, "e");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // E
- return E;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_result.class, metaDataMap);
- }
-
- public replicateLog_result() {
- }
-
- public replicateLog_result(
- RemoteReplicationException e)
- {
- this();
- this.e = e;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateLog_result(replicateLog_result other) {
- if (other.isSetE()) {
- this.e = new RemoteReplicationException(other.e);
- }
- }
-
- public replicateLog_result deepCopy() {
- return new replicateLog_result(this);
- }
-
- @Override
- public void clear() {
- this.e = null;
- }
-
- public RemoteReplicationException getE() {
- return this.e;
- }
-
- public replicateLog_result setE(RemoteReplicationException e) {
- this.e = e;
- return this;
- }
-
- public void unsetE() {
- this.e = null;
- }
-
- /** Returns true if field e is set (has been assigned a value) and false otherwise */
- public boolean isSetE() {
- return this.e != null;
- }
-
- public void setEIsSet(boolean value) {
- if (!value) {
- this.e = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case E:
- if (value == null) {
- unsetE();
- } else {
- setE((RemoteReplicationException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case E:
- return getE();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case E:
- return isSetE();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateLog_result)
- return this.equals((replicateLog_result)that);
- return false;
- }
-
- public boolean equals(replicateLog_result that) {
- if (that == null)
- return false;
-
- boolean this_present_e = true && this.isSetE();
- boolean that_present_e = true && that.isSetE();
- if (this_present_e || that_present_e) {
- if (!(this_present_e && that_present_e))
- return false;
- if (!this.e.equals(that.e))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateLog_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateLog_result typedOther = (replicateLog_result)other;
-
- lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetE()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateLog_result(");
- boolean first = true;
-
- sb.append("e:");
- if (this.e == null) {
- sb.append("null");
- } else {
- sb.append(this.e);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateLog_resultStandardSchemeFactory implements SchemeFactory {
- public replicateLog_resultStandardScheme getScheme() {
- return new replicateLog_resultStandardScheme();
- }
- }
-
- private static class replicateLog_resultStandardScheme extends StandardScheme<replicateLog_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // E
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.e != null) {
- oprot.writeFieldBegin(E_FIELD_DESC);
- struct.e.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateLog_resultTupleSchemeFactory implements SchemeFactory {
- public replicateLog_resultTupleScheme getScheme() {
- return new replicateLog_resultTupleScheme();
- }
- }
-
- private static class replicateLog_resultTupleScheme extends TupleScheme<replicateLog_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetE()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetE()) {
- struct.e.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateKeyValues_args implements org.apache.thrift.TBase<replicateKeyValues_args, replicateKeyValues_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_args");
-
- private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateKeyValues_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateKeyValues_argsTupleSchemeFactory());
- }
-
- public int remoteTableId; // required
- public KeyValues data; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- REMOTE_TABLE_ID((short)1, "remoteTableId"),
- DATA((short)2, "data");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // REMOTE_TABLE_ID
- return REMOTE_TABLE_ID;
- case 2: // DATA
- return DATA;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REMOTETABLEID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeyValues.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_args.class, metaDataMap);
- }
-
- public replicateKeyValues_args() {
- }
-
- public replicateKeyValues_args(
- int remoteTableId,
- KeyValues data)
- {
- this();
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- this.data = data;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateKeyValues_args(replicateKeyValues_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.remoteTableId = other.remoteTableId;
- if (other.isSetData()) {
- this.data = new KeyValues(other.data);
- }
- }
-
- public replicateKeyValues_args deepCopy() {
- return new replicateKeyValues_args(this);
- }
-
- @Override
- public void clear() {
- setRemoteTableIdIsSet(false);
- this.remoteTableId = 0;
- this.data = null;
- }
-
- public int getRemoteTableId() {
- return this.remoteTableId;
- }
-
- public replicateKeyValues_args setRemoteTableId(int remoteTableId) {
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- return this;
- }
-
- public void unsetRemoteTableId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
- public boolean isSetRemoteTableId() {
- return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- public void setRemoteTableIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
- }
-
- public KeyValues getData() {
- return this.data;
- }
-
- public replicateKeyValues_args setData(KeyValues data) {
- this.data = data;
- return this;
- }
-
- public void unsetData() {
- this.data = null;
- }
-
- /** Returns true if field data is set (has been assigned a value) and false otherwise */
- public boolean isSetData() {
- return this.data != null;
- }
-
- public void setDataIsSet(boolean value) {
- if (!value) {
- this.data = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case REMOTE_TABLE_ID:
- if (value == null) {
- unsetRemoteTableId();
- } else {
- setRemoteTableId((Integer)value);
- }
- break;
-
- case DATA:
- if (value == null) {
- unsetData();
- } else {
- setData((KeyValues)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case REMOTE_TABLE_ID:
- return Integer.valueOf(getRemoteTableId());
-
- case DATA:
- return getData();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case REMOTE_TABLE_ID:
- return isSetRemoteTableId();
- case DATA:
- return isSetData();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateKeyValues_args)
- return this.equals((replicateKeyValues_args)that);
- return false;
- }
-
- public boolean equals(replicateKeyValues_args that) {
- if (that == null)
- return false;
-
- boolean this_present_remoteTableId = true;
- boolean that_present_remoteTableId = true;
- if (this_present_remoteTableId || that_present_remoteTableId) {
- if (!(this_present_remoteTableId && that_present_remoteTableId))
- return false;
- if (this.remoteTableId != that.remoteTableId)
- return false;
- }
-
- boolean this_present_data = true && this.isSetData();
- boolean that_present_data = true && that.isSetData();
- if (this_present_data || that_present_data) {
- if (!(this_present_data && that_present_data))
- return false;
- if (!this.data.equals(that.data))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateKeyValues_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateKeyValues_args typedOther = (replicateKeyValues_args)other;
-
- lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRemoteTableId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetData()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateKeyValues_args(");
- boolean first = true;
-
- sb.append("remoteTableId:");
- sb.append(this.remoteTableId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("data:");
- if (this.data == null) {
- sb.append("null");
- } else {
- sb.append(this.data);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (data != null) {
- data.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateKeyValues_argsStandardSchemeFactory implements SchemeFactory {
- public replicateKeyValues_argsStandardScheme getScheme() {
- return new replicateKeyValues_argsStandardScheme();
- }
- }
-
- private static class replicateKeyValues_argsStandardScheme extends StandardScheme<replicateKeyValues_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // REMOTE_TABLE_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // DATA
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.data = new KeyValues();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
- oprot.writeI32(struct.remoteTableId);
- oprot.writeFieldEnd();
- if (struct.data != null) {
- oprot.writeFieldBegin(DATA_FIELD_DESC);
- struct.data.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateKeyValues_argsTupleSchemeFactory implements SchemeFactory {
- public replicateKeyValues_argsTupleScheme getScheme() {
- return new replicateKeyValues_argsTupleScheme();
- }
- }
-
- private static class replicateKeyValues_argsTupleScheme extends TupleScheme<replicateKeyValues_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetRemoteTableId()) {
- optionals.set(0);
- }
- if (struct.isSetData()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetRemoteTableId()) {
- oprot.writeI32(struct.remoteTableId);
- }
- if (struct.isSetData()) {
- struct.data.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- }
- if (incoming.get(1)) {
- struct.data = new KeyValues();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateKeyValues_result implements org.apache.thrift.TBase<replicateKeyValues_result, replicateKeyValues_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_result");
-
- private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateKeyValues_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateKeyValues_resultTupleSchemeFactory());
- }
-
- public RemoteReplicationException e; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- E((short)1, "e");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // E
- return E;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_result.class, metaDataMap);
- }
-
- public replicateKeyValues_result() {
- }
-
- public replicateKeyValues_result(
- RemoteReplicationException e)
- {
- this();
- this.e = e;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateKeyValues_result(replicateKeyValues_result other) {
- if (other.isSetE()) {
- this.e = new RemoteReplicationException(other.e);
- }
- }
-
- public replicateKeyValues_result deepCopy() {
- return new replicateKeyValues_result(this);
- }
-
- @Override
- public void clear() {
- this.e = null;
- }
-
- public RemoteReplicationException getE() {
- return this.e;
- }
-
- public replicateKeyValues_result setE(RemoteReplicationException e) {
- this.e = e;
- return this;
- }
-
- public void unsetE() {
- this.e = null;
- }
-
- /** Returns true if field e is set (has been assigned a value) and false otherwise */
- public boolean isSetE() {
- return this.e != null;
- }
-
- public void setEIsSet(boolean value) {
- if (!value) {
- this.e = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case E:
- if (value == null) {
- unsetE();
- } else {
- setE((RemoteReplicationException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case E:
- return getE();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case E:
- return isSetE();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateKeyValues_result)
- return this.equals((replicateKeyValues_result)that);
- return false;
- }
-
- public boolean equals(replicateKeyValues_result that) {
- if (that == null)
- return false;
-
- boolean this_present_e = true && this.isSetE();
- boolean that_present_e = true && that.isSetE();
- if (this_present_e || that_present_e) {
- if (!(this_present_e && that_present_e))
- return false;
- if (!this.e.equals(that.e))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateKeyValues_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateKeyValues_result typedOther = (replicateKeyValues_result)other;
-
- lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetE()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateKeyValues_result(");
- boolean first = true;
-
- sb.append("e:");
- if (this.e == null) {
- sb.append("null");
- } else {
- sb.append(this.e);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateKeyValues_resultStandardSchemeFactory implements SchemeFactory {
- public replicateKeyValues_resultStandardScheme getScheme() {
- return new replicateKeyValues_resultStandardScheme();
- }
- }
-
- private static class replicateKeyValues_resultStandardScheme extends StandardScheme<replicateKeyValues_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // E
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.e != null) {
- oprot.writeFieldBegin(E_FIELD_DESC);
- struct.e.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateKeyValues_resultTupleSchemeFactory implements SchemeFactory {
- public replicateKeyValues_resultTupleScheme getScheme() {
- return new replicateKeyValues_resultTupleScheme();
- }
- }
-
- private static class replicateKeyValues_resultTupleScheme extends TupleScheme<replicateKeyValues_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetE()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetE()) {
- struct.e.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- }
- }
- }
-
- }
-
-}
[25/50] [abbrv] git commit: Merge branch '1.6.1-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.1-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ab453545
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ab453545
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ab453545
Branch: refs/heads/ACCUMULO-378
Commit: ab453545961c526f06dfd48b0056990c4fcdd5a8
Parents: 428ed9a cdcc1d4
Author: Bill Havanki <bh...@cloudera.com>
Authored: Mon May 5 09:39:33 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon May 5 09:39:33 2014 -0400
----------------------------------------------------------------------
conf/templates/generic_logger.properties | 58 ++++++++++++++++++++
conf/templates/monitor_logger.properties | 44 +++++++++++++++
.../org/apache/accumulo/server/Accumulo.java | 39 +++++++++++--
.../server/watcher/MonitorLog4jWatcher.java | 19 ++++++-
.../apache/accumulo/server/AccumuloTest.java | 51 +++++++++++++++++
.../server/watcher/MonitorLog4jWatcherTest.java | 50 +++++++++++++++++
6 files changed, 252 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ab453545/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ab453545/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
[24/50] [abbrv] git commit: ACCUMULO-2383 Add log4j properties support
Posted by el...@apache.org.
ACCUMULO-2383 Add log4j properties support
Accumulo now looks for either XML or properties files for the Log4J configuration. It will
also heed the log4j.configuration isystem property if it is set. Its MonitorLog4jWatcher
can now load either properties or XML.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cdcc1d4d
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cdcc1d4d
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cdcc1d4d
Branch: refs/heads/ACCUMULO-378
Commit: cdcc1d4d41bf16e628b87893814a0ae9f627bcdb
Parents: 5a72c37
Author: Bill Havanki <bh...@cloudera.com>
Authored: Tue Apr 22 13:44:55 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon May 5 09:31:54 2014 -0400
----------------------------------------------------------------------
conf/templates/generic_logger.properties | 58 ++++++++++++++++++++
conf/templates/monitor_logger.properties | 44 +++++++++++++++
.../org/apache/accumulo/server/Accumulo.java | 39 +++++++++++--
.../server/watcher/MonitorLog4jWatcher.java | 19 ++++++-
.../apache/accumulo/server/AccumuloTest.java | 51 +++++++++++++++++
.../server/watcher/MonitorLog4jWatcherTest.java | 50 +++++++++++++++++
6 files changed, 252 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdcc1d4d/conf/templates/generic_logger.properties
----------------------------------------------------------------------
diff --git a/conf/templates/generic_logger.properties b/conf/templates/generic_logger.properties
new file mode 100644
index 0000000..06e28df
--- /dev/null
+++ b/conf/templates/generic_logger.properties
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Write out everything at the DEBUG level to the debug log
+log4j.appender.A2=org.apache.log4j.RollingFileAppender
+log4j.appender.A2.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.debug.log
+log4j.appender.A2.MaxFileSize=1000MB
+log4j.appender.A2.MaxBackupIndex=10
+log4j.appender.A2.Threshold=DEBUG
+log4j.appender.A2.layout=org.apache.log4j.PatternLayout
+log4j.appender.A2.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+
+# Write out INFO and higher to the regular log
+log4j.appender.A3=org.apache.log4j.RollingFileAppender
+log4j.appender.A3.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.log
+log4j.appender.A3.MaxFileSize=1000MB
+log4j.appender.A3.MaxBackupIndex=10
+log4j.appender.A3.Threshold=INFO
+log4j.appender.A3.layout=org.apache.log4j.PatternLayout
+log4j.appender.A3.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %m%n
+
+# Send all logging data to a centralized logger
+# If the centralized logger is down, buffer the log events, but drop them if it stays down
+log4j.appender.ASYNC=org.apache.accumulo.core.util.AsyncSocketAppender
+log4j.appender.ASYNC.RemoteHost=${org.apache.accumulo.core.host.log}
+log4j.appender.ASYNC.Port=4560
+log4j.appender.ASYNC.Application=${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}
+log4j.appender.ASYNC.Threshold=WARN
+
+# Log accumulo events to the debug, normal and remote logs.
+log4j.logger.org.apache.accumulo=DEBUG, A2, A3, ASYNC
+log4j.additivity.org.apache.accumulo=false
+
+# change to INFO for authorization events
+log4j.logger.org.apache.accumulo.server.security.Auditor=WARN
+
+log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO
+
+log4j.logger.org.apache.accumulo.examples.wikisearch=INFO
+
+log4j.logger.org.mortbay.log=WARN
+
+log4j.logger.com.yahoo.zookeeper=ERROR
+
+# Log non-accumulo events to the debug and normal logs
+log4j.rootLogger=INFO, A2, A3
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdcc1d4d/conf/templates/monitor_logger.properties
----------------------------------------------------------------------
diff --git a/conf/templates/monitor_logger.properties b/conf/templates/monitor_logger.properties
new file mode 100644
index 0000000..4dada2d
--- /dev/null
+++ b/conf/templates/monitor_logger.properties
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Write out everything at the DEBUG level to the debug log
+log4j.appender.A2=org.apache.log4j.RollingFileAppender
+log4j.appender.A2.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.debug.log
+log4j.appender.A2.MaxFileSize=100MB
+log4j.appender.A2.MaxBackupIndex=10
+log4j.appender.A2.Threshold=DEBUG
+log4j.appender.A2.layout=org.apache.log4j.PatternLayout
+log4j.appender.A2.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n
+
+# Write out INFO and higher to the regular log
+log4j.appender.A3=org.apache.log4j.RollingFileAppender
+log4j.appender.A3.File=${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}_fromprops.log
+log4j.appender.A3.MaxFileSize=100MB
+log4j.appender.A3.MaxBackupIndex=10
+log4j.appender.A3.Threshold=INFO
+log4j.appender.A3.layout=org.apache.log4j.PatternLayout
+log4j.appender.A3.layout.ConversionPattern=%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n
+
+# Keep the last few log messages for display to the user
+log4j.appender.GUI=org.apache.accumulo.server.monitor.LogService
+log4j.appender.GUI.Keep=50
+log4j.appender.GUI.Threshold=WARN
+
+# Log accumulo messages to debug, normal and GUI
+log4j.logger.org.apache.accumulo=DEBUG, A2, A3, GUI
+log4j.additivity.org.apache.accumulo=false
+
+# Log non-accumulo messages to debug, normal logs.
+log4j.rootLogger=INFO, A2, A3
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdcc1d4d/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
index 4e1eb35..68d862d 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/Accumulo.java
@@ -110,6 +110,35 @@ public class Accumulo {
}
}
+ /**
+ * Finds the best log4j configuration file. A generic file is used only if an
+ * application-specific file is not available. An XML file is preferred over
+ * a properties file, if possible.
+ *
+ * @param confDir directory where configuration files should reside
+ * @param application application name for configuration file name
+ * @return configuration file name
+ */
+ static String locateLogConfig(String confDir, String application) {
+ String explicitConfigFile = System.getProperty("log4j.configuration");
+ if (explicitConfigFile != null) {
+ return explicitConfigFile;
+ }
+ String[] configFiles = {
+ String.format("%s/%s_logger.xml", confDir, application),
+ String.format("%s/%s_logger.properties", confDir, application),
+ String.format("%s/generic_logger.xml", confDir),
+ String.format("%s/generic_logger.properties", confDir)
+ };
+ String defaultConfigFile = configFiles[2]; // generic_logger.xml
+ for (String f : configFiles) {
+ if (new File(f).exists()) {
+ return f;
+ }
+ }
+ return defaultConfigFile;
+ }
+
public static void init(VolumeManager fs, ServerConfiguration config, String application) throws UnknownHostException {
System.setProperty("org.apache.accumulo.core.application", application);
@@ -126,11 +155,7 @@ public class Accumulo {
System.setProperty("org.apache.accumulo.core.host.log.port", Integer.toString(logPort));
// Use a specific log config, if it exists
- String logConfig = String.format("%s/%s_logger.xml", System.getenv("ACCUMULO_CONF_DIR"), application);
- if (!new File(logConfig).exists()) {
- // otherwise, use the generic config
- logConfig = String.format("%s/generic_logger.xml", System.getenv("ACCUMULO_CONF_DIR"));
- }
+ String logConfig = locateLogConfig(System.getenv("ACCUMULO_CONF_DIR"), application);
// Turn off messages about not being able to reach the remote logger... we protect against that.
LogLog.setQuietMode(true);
@@ -140,7 +165,9 @@ public class Accumulo {
DOMConfigurator.configureAndWatch(auditConfig, 5000);
// Configure logging using information advertised in zookeeper by the monitor
- new MonitorLog4jWatcher(config.getInstance().getInstanceID(), logConfig, 5000).start();
+ MonitorLog4jWatcher logConfigWatcher = new MonitorLog4jWatcher(config.getInstance().getInstanceID(), logConfig);
+ logConfigWatcher.setDelay(5000L);
+ logConfigWatcher.start();
log.info(application + " starting");
log.info("Instance " + config.getInstance().getInstanceID());
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdcc1d4d/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
index ac3426e..bc5e99f 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcher.java
@@ -22,6 +22,7 @@ import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.log4j.Appender;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.apache.log4j.PropertyConfigurator;
import org.apache.log4j.helpers.FileWatchdog;
import org.apache.log4j.xml.DOMConfigurator;
import org.apache.zookeeper.KeeperException.NoNodeException;
@@ -40,16 +41,24 @@ public class MonitorLog4jWatcher extends FileWatchdog implements Watcher {
private static final String PORT_PROPERTY_NAME = "org.apache.accumulo.core.host.log.port";
private final Object lock;
+ private final boolean usingProperties;
private boolean loggingDisabled = false;
protected String path;
- public MonitorLog4jWatcher(String instance, String filename, int delay) {
+ public MonitorLog4jWatcher(String instance, String filename) {
super(filename);
- setDelay(delay);
+ usingProperties = (filename != null && filename.endsWith(".properties"));
this.path = ZooUtil.getRoot(instance) + Constants.ZMONITOR_LOG4J_ADDR;
this.lock = new Object();
}
+ boolean isUsingProperties() {
+ return usingProperties;
+ }
+ String getPath() {
+ return path;
+ }
+
@Override
public void run() {
try {
@@ -85,7 +94,11 @@ public class MonitorLog4jWatcher extends FileWatchdog implements Watcher {
private void resetLogger() {
// Force a reset on the logger's configuration
LogManager.resetConfiguration();
- new DOMConfigurator().doConfigure(filename, LogManager.getLoggerRepository());
+ if (usingProperties) {
+ new PropertyConfigurator().doConfigure(filename, LogManager.getLoggerRepository());
+ } else {
+ new DOMConfigurator().doConfigure(filename, LogManager.getLoggerRepository());
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdcc1d4d/server/base/src/test/java/org/apache/accumulo/server/AccumuloTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/AccumuloTest.java b/server/base/src/test/java/org/apache/accumulo/server/AccumuloTest.java
index 9366163..a090b40 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/AccumuloTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/AccumuloTest.java
@@ -16,7 +16,9 @@
*/
package org.apache.accumulo.server;
+import java.io.File;
import java.io.FileNotFoundException;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -24,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.easymock.EasyMock.createMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
@@ -82,4 +85,52 @@ public class AccumuloTest {
assertEquals(-1, Accumulo.getAccumuloPersistentVersion(fs, path));
}
+ @Test
+ public void testLocateLogConfig() throws Exception {
+ File confDir = new File(FileUtils.getTempDirectory(), "AccumuloTest" + System.currentTimeMillis());
+ String confDirName = confDir.getAbsolutePath();
+ assertTrue("Failed to make test configuration directory", confDir.mkdir());
+ try {
+ File appProps = new File(confDir, "flogger_logger.properties");
+ String appPropsName = String.format("%s/flogger_logger.properties", confDirName);
+ FileUtils.touch(appProps);
+ File genericXml = new File(confDir, "generic_logger.xml");
+ String genericXmlName = String.format("%s/generic_logger.xml", confDirName);
+ FileUtils.touch(genericXml);
+
+ assertEquals(appPropsName, Accumulo.locateLogConfig(confDirName, "flogger"));
+ assertEquals(genericXmlName, Accumulo.locateLogConfig(confDirName, "flagger"));
+
+ assertTrue("Failed to delete log configuration file", appProps.delete());
+ assertEquals(genericXmlName, Accumulo.locateLogConfig(confDirName, "flogger"));
+ } finally {
+ FileUtils.deleteDirectory(confDir);
+ }
+ }
+ @Test
+ public void testLocateLogConfig_Default() throws Exception {
+ File confDir = new File(FileUtils.getTempDirectory(), "AccumuloTest" + System.currentTimeMillis());
+ String confDirName = confDir.getAbsolutePath();
+ assertTrue("Failed to make test configuration directory", confDir.mkdir());
+ try {
+ File genericXml = new File(confDir, "generic_logger.xml");
+ String genericXmlName = String.format("%s/generic_logger.xml", confDirName);
+
+ assertEquals(genericXmlName, Accumulo.locateLogConfig(confDirName, "flogger"));
+ } finally {
+ FileUtils.deleteDirectory(confDir);
+ }
+ }
+ @Test
+ public void testLocateLogConfig_Explicit() throws Exception {
+ File confDir = new File(FileUtils.getTempDirectory(), "AccumuloTest" + System.currentTimeMillis());
+ String confDirName = confDir.getAbsolutePath();
+ System.setProperty("log4j.configuration", "myconfig.xml");
+ try {
+ assertEquals("myconfig.xml", Accumulo.locateLogConfig(confDirName, "flogger"));
+ } finally {
+ FileUtils.deleteDirectory(confDir);
+ System.clearProperty("log4j.configuration");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cdcc1d4d/server/base/src/test/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcherTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcherTest.java b/server/base/src/test/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcherTest.java
new file mode 100644
index 0000000..448e0ac
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/watcher/MonitorLog4jWatcherTest.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.watcher;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.easymock.Capture;
+import static org.easymock.EasyMock.*;
+
+public class MonitorLog4jWatcherTest {
+ private static final String INSTANCE = "instance";
+ private static final String FILENAME = "something_logger.xml";
+
+ private MonitorLog4jWatcher w;
+
+ @Before
+ public void setUp() throws Exception {
+ w = new MonitorLog4jWatcher(INSTANCE, FILENAME);
+ }
+
+ @Test
+ public void testGetters() {
+ assertFalse(w.isUsingProperties());
+ assertEquals(ZooUtil.getRoot(INSTANCE) + Constants.ZMONITOR_LOG4J_ADDR, w.getPath());
+ }
+
+ @Test
+ public void testPropertyDetection() {
+ w = new MonitorLog4jWatcher(INSTANCE, FILENAME + ".properties");
+ assertTrue(w.isUsingProperties());
+ }
+}
[21/50] [abbrv] git commit: ACCUMULO-2583 Missed an extra zero in the
default port
Posted by el...@apache.org.
ACCUMULO-2583 Missed an extra zero in the default port
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/102ec71a
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/102ec71a
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/102ec71a
Branch: refs/heads/ACCUMULO-378
Commit: 102ec71ae2e628b8e5f4da26ed8c70e9914f26d6
Parents: 3e21a41
Author: Josh Elser <el...@apache.org>
Authored: Sun May 4 12:20:36 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sun May 4 12:20:36 2014 -0400
----------------------------------------------------------------------
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/102ec71a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 30835b9..6700736 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -461,7 +461,7 @@ public enum Property {
@Experimental
REPLICATION_WORKER_THREADS("replication.worker.threads", "4", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
@Experimental
- REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "1001", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
+ REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "10001", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
;
[35/50] [abbrv] git commit: ACCUMULO-2212 Fix UTF8 constant in
ZooReaderWriterFactory
Posted by el...@apache.org.
ACCUMULO-2212 Fix UTF8 constant in ZooReaderWriterFactory
The merge of ACCUMULO-2212 work from the 1.6.1-SNAPSHOT branch introduced the use of
Constants.UTF8, which does not exist in the master branch. This commit replaces that
constant with a locally defined one.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/cd045729
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/cd045729
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/cd045729
Branch: refs/heads/ACCUMULO-378
Commit: cd045729e44173ab145b25e40942e6c0b5e694b0
Parents: 4ba7d3d
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed May 7 14:01:34 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Wed May 7 14:01:34 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/cd045729/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
index 3c8d143..1682cda 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/zookeeper/ZooReaderWriterFactory.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.server.zookeeper;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
+import java.nio.charset.Charset;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
@@ -31,6 +32,7 @@ import org.apache.accumulo.server.conf.ServerConfiguration;
* A factory for {@link ZooReaderWriter} objects.
*/
public class ZooReaderWriterFactory {
+ private static final Charset UTF8 = Charset.forName("UTF-8");
private static final String SCHEME = "digest";
private static final String USER = "accumulo";
private static IZooReaderWriter instance = null;
@@ -48,7 +50,7 @@ public class ZooReaderWriterFactory {
* @return reader/writer
*/
public IZooReaderWriter getZooReaderWriter(String string, int timeInMillis, String secret) {
- return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(Constants.UTF8));
+ return new ZooReaderWriter(string, timeInMillis, SCHEME, (USER + ":" + secret).getBytes(UTF8));
}
/**
[14/50] [abbrv] git commit: Merge branch '1.6.1-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.1-SNAPSHOT'
Conflicts:
assemble/pom.xml
core/pom.xml
docs/pom.xml
examples/simple/pom.xml
fate/pom.xml
maven-plugin/pom.xml
minicluster/pom.xml
pom.xml
proxy/pom.xml
server/base/pom.xml
server/gc/pom.xml
server/master/pom.xml
server/monitor/pom.xml
server/native/pom.xml
server/tracer/pom.xml
server/tserver/pom.xml
start/pom.xml
test/pom.xml
trace/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/49b1f08c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/49b1f08c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/49b1f08c
Branch: refs/heads/ACCUMULO-378
Commit: 49b1f08c3fd711d021758841e5556b06e93cc931
Parents: 1dae029 4d5ba9d
Author: Christopher Tubbs <ct...@apache.org>
Authored: Fri May 2 18:38:51 2014 -0400
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Fri May 2 18:38:51 2014 -0400
----------------------------------------------------------------------
----------------------------------------------------------------------
[34/50] [abbrv] git commit: Merge branch '1.6.1-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.1-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/4ba7d3d1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/4ba7d3d1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/4ba7d3d1
Branch: refs/heads/ACCUMULO-378
Commit: 4ba7d3d198bb5266143a8b6e6756dffd6630e7f2
Parents: ab45354 484491d
Author: Bill Havanki <bh...@cloudera.com>
Authored: Wed May 7 11:41:46 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Wed May 7 11:41:46 2014 -0400
----------------------------------------------------------------------
.../zookeeper/RetryingInvocationHandler.java | 63 ++++++++++++++
.../fate/zookeeper/ZooReaderWriter.java | 24 +-----
.../RetryingInvocationHandlerTest.java | 87 ++++++++++++++++++++
.../zookeeper/ZooReaderWriterFactory.java | 86 +++++++++++++++++++
4 files changed, 238 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
[40/50] [abbrv] git commit: ACCUMULO-1327 converted latex manual to
asciidoc
Posted by el...@apache.org.
ACCUMULO-1327 converted latex manual to asciidoc
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/900d6abb
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/900d6abb
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/900d6abb
Branch: refs/heads/ACCUMULO-378
Commit: 900d6abb04dd1f5c87d16b55624055de6d11a6ab
Parents: cd04572
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed May 7 17:54:28 2014 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed May 7 17:54:28 2014 -0700
----------------------------------------------------------------------
README | 2 +-
assemble/pom.xml | 2 +-
assemble/src/main/assemblies/component.xml | 2 +-
.../accumulo/core/conf/ConfigurationDocGen.java | 72 +-
.../core/conf/DefaultConfiguration.java | 4 +-
.../accumulo/core/conf/config-header.html | 2 -
.../apache/accumulo/core/conf/config-header.tex | 99 ---
.../apache/accumulo/core/conf/config-header.txt | 93 +++
docs/pom.xml | 67 +-
.../main/asciidoc/accumulo_user_manual.asciidoc | 62 ++
.../main/asciidoc/chapters/administration.txt | 396 +++++++++
docs/src/main/asciidoc/chapters/analytics.txt | 229 ++++++
docs/src/main/asciidoc/chapters/clients.txt | 320 ++++++++
docs/src/main/asciidoc/chapters/design.txt | 179 +++++
.../asciidoc/chapters/development_clients.txt | 107 +++
.../asciidoc/chapters/high_speed_ingest.txt | 125 +++
.../src/main/asciidoc/chapters/introduction.txt | 25 +
docs/src/main/asciidoc/chapters/multivolume.txt | 82 ++
docs/src/main/asciidoc/chapters/security.txt | 176 ++++
docs/src/main/asciidoc/chapters/shell.txt | 129 +++
.../asciidoc/chapters/table_configuration.txt | 635 +++++++++++++++
.../src/main/asciidoc/chapters/table_design.txt | 336 ++++++++
.../main/asciidoc/chapters/troubleshooting.txt | 758 ++++++++++++++++++
docs/src/main/asciidoc/images/accumulo-logo.png | Bin 0 -> 11427 bytes
.../main/asciidoc/images/data_distribution.png | Bin 0 -> 86937 bytes
.../main/asciidoc/images/failure_handling.png | Bin 0 -> 48905 bytes
.../accumulo_user_manual.tex | 55 --
.../chapters/administration.tex | 431 ----------
.../accumulo_user_manual/chapters/analytics.tex | 248 ------
.../accumulo_user_manual/chapters/clients.tex | 331 --------
.../accumulo_user_manual/chapters/design.tex | 186 -----
.../chapters/development_clients.tex | 109 ---
.../chapters/high_speed_ingest.tex | 133 ----
.../chapters/introduction.tex | 27 -
.../chapters/multivolume.tex | 85 --
.../accumulo_user_manual/chapters/security.tex | 179 -----
.../accumulo_user_manual/chapters/shell.tex | 138 ----
.../chapters/table_configuration.tex | 663 ----------------
.../chapters/table_design.tex | 343 --------
.../chapters/troubleshooting.tex | 794 -------------------
.../images/data_distribution.png | Bin 86937 -> 0 bytes
.../images/failure_handling.png | Bin 48905 -> 0 bytes
pom.xml | 12 +-
43 files changed, 3754 insertions(+), 3882 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/README
----------------------------------------------------------------------
diff --git a/README b/README
index a2d758b..e806dff 100644
--- a/README
+++ b/README
@@ -72,7 +72,7 @@ library into the $ACCUMULO_HOME/lib/native/map.
Building Documentation
-Use the following command to build the User Manual (docs/target/accumulo_user_manual.pdf)
+Use the following command to build the User Manual (docs/target/generated-docs/accumulo_user_manual.html)
and the configuration HTML page (docs/target/config.html)
mvn package -P docs -DskipTests
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/assemble/pom.xml
----------------------------------------------------------------------
diff --git a/assemble/pom.xml b/assemble/pom.xml
index d30c265..bbd409b 100644
--- a/assemble/pom.xml
+++ b/assemble/pom.xml
@@ -227,7 +227,7 @@
<groupId>org.apache.accumulo</groupId>
<artifactId>accumulo-docs</artifactId>
<classifier>user-manual</classifier>
- <type>pdf</type>
+ <type>html</type>
</dependency>
</dependencies>
</profile>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/assemble/src/main/assemblies/component.xml
----------------------------------------------------------------------
diff --git a/assemble/src/main/assemblies/component.xml b/assemble/src/main/assemblies/component.xml
index 747444b..5ef9914 100644
--- a/assemble/src/main/assemblies/component.xml
+++ b/assemble/src/main/assemblies/component.xml
@@ -47,7 +47,7 @@
<outputFileNameMapping>${artifactId}_user_manual.${artifact.extension}</outputFileNameMapping>
<useTransitiveDependencies>false</useTransitiveDependencies>
<includes>
- <include>${groupId}:${artifactId}-docs:pdf:user-manual</include>
+ <include>${groupId}:${artifactId}-docs:html:user-manual</include>
</includes>
</dependencySet>
</dependencySets>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 8ed5506..6cf9c77 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@ -39,6 +39,7 @@ class ConfigurationDocGen {
pageHeader();
beginSection("Available Properties");
+ propertyQuickLinks();
for (Property prefix : prefixes) {
if (!prefix.isExperimental()) {
prefixSection(prefix);
@@ -75,6 +76,8 @@ class ConfigurationDocGen {
abstract void propertyTypeDescriptions();
+ abstract void propertyQuickLinks();
+
abstract String sanitize(String str);
}
@@ -130,8 +133,7 @@ class ConfigurationDocGen {
}
@Override
- void pageHeader() {
- super.pageHeader();
+ void propertyQuickLinks() {
doc.println("<p>Jump to: ");
String delimiter = "";
for (Property prefix : prefixes) {
@@ -210,22 +212,37 @@ class ConfigurationDocGen {
}
- private class LaTeX extends Format {
+ private class Asciidoc extends Format {
@Override
void beginSection(String section) {
- doc.println("\\section{" + section + "}");
+ doc.println("=== " + section);
}
@Override
String getExt() {
- return "tex";
+ return "txt";
+ }
+
+ @Override
+ void propertyQuickLinks() {
+ doc.println("Jump to: ");
+ String delimiter = "";
+ for (Property prefix : prefixes) {
+ if (!prefix.isExperimental()) {
+ doc.print(delimiter + "<<" + prefix.name() + ">>");
+ delimiter = " | ";
+ }
+ }
+ doc.println();
+ doc.println();
}
@Override
void prefixSection(Property prefix) {
boolean depr = prefix.isDeprecated();
- doc.println("\\subsection{" + prefix.getKey() + "*}" + (depr ? " (Deprecated)" : ""));
- doc.println((depr ? "\\sout{\\textit{Deprecated.} " : "") + sanitize(prefix.getDescription()) + (depr ? "}" : ""));
+ doc.println("[[" + prefix.name() + "]]");
+ doc.println("==== " + prefix.getKey() + "*" + (depr ? " (Deprecated)" : ""));
+ doc.println(strike((depr ? "_Deprecated._ " : "") + sanitize(prefix.getDescription()), depr));
doc.println();
}
@@ -233,18 +250,28 @@ class ConfigurationDocGen {
void property(Property prefix, Property prop) {
boolean depr = prefix.isDeprecated() || prop.isDeprecated();
if (prop.getKey().startsWith(prefix.getKey())) {
- doc.println("\\subsubsection{" + prop.getKey() + "}");
- doc.println(strike((depr ? "\\textit{Deprecated.} " : "") + sanitize(prop.getDescription()), depr));
+ doc.println("===== " + prop.getKey());
+ doc.println(strike((depr ? "_Deprecated._ " : "") + sanitize(prop.getDescription()), depr));
doc.println();
- doc.println(strike("\\textit{Type:} " + prop.getType().name() + "\\\\", depr));
- doc.println(strike("\\textit{ZooKeeper Mutable:} " + isZooKeeperMutable(prop) + "\\\\", depr));
- doc.println(strike("\\textit{Default Value:} " + sanitize(prop.getRawDefaultValue()), depr));
+ doc.println(strike("_Type:_ " + prop.getType().name(), depr) + " +");
+ doc.println(strike("_Zookeeper Mutable:_ " + isZooKeeperMutable(prop), depr) + " +");
+ String defaultValue = sanitize(prop.getRawDefaultValue()).trim();
+ if (defaultValue.length() == 0) {
+ // need a placeholder or the asciidoc line break won't work
+ defaultValue = strike("_Default Value:_ _empty_", depr);
+ } else if (defaultValue.contains("\n")) {
+ // deal with multi-line values, skip strikethrough of value
+ defaultValue = strike("_Default Value:_ ", depr) + "\n----\n" + defaultValue + "\n----\n";
+ } else {
+ defaultValue = strike("_Default Value:_ " + "`" + defaultValue + "`", depr);
+ }
+ doc.println(defaultValue);
doc.println();
}
}
private String strike(String s, boolean isDeprecated) {
- return (isDeprecated ? "\\sout{" : "") + s + (isDeprecated ? "}" : "");
+ return (isDeprecated ? "[line-through]#" : "") + s + (isDeprecated ? "#" : "");
}
@Override
@@ -252,7 +279,7 @@ class ConfigurationDocGen {
for (PropertyType type : PropertyType.values()) {
if (type == PropertyType.PREFIX)
continue;
- doc.println("\\subsection{" + sanitize(type.toString()) + "}");
+ doc.println("==== " + sanitize(type.toString()));
doc.println(sanitize(type.getFormatDescription()));
doc.println();
}
@@ -260,19 +287,6 @@ class ConfigurationDocGen {
@Override
String sanitize(String str) {
- str = str.replace("\\", "\\textbackslash{}");
- str = str.replace("~", "\\textasciitilde{}");
- str = str.replace("^", "\\textasciicircum");
-
- str = str.replace("&", "\\&");
- str = str.replace("%", "\\%");
- str = str.replace("$", "\\$");
- str = str.replace("#", "\\#");
- str = str.replace("_", "\\_");
- str = str.replace("{", "\\{");
- str = str.replace("}", "\\}");
-
- str = str.replaceAll("(?:\r\n|\r|\n)", "\\\\\\\\\n");
return str;
}
}
@@ -334,8 +348,8 @@ class ConfigurationDocGen {
new HTML().generate();
}
- void generateLaTeX() {
- new LaTeX().generate();
+ void generateAsciidoc() {
+ new Asciidoc().generate();
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
index b14d924..f3f43b9 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/DefaultConfiguration.java
@@ -79,8 +79,8 @@ public class DefaultConfiguration extends AccumuloConfiguration {
public static void main(String[] args) throws FileNotFoundException, UnsupportedEncodingException {
if (args.length == 2 && args[0].equals("--generate-html")) {
new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateHtml();
- } else if (args.length == 2 && args[0].equals("--generate-latex")) {
- new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateLaTeX();
+ } else if (args.length == 2 && args[0].equals("--generate-asciidoc")) {
+ new ConfigurationDocGen(new PrintStream(args[1], StandardCharsets.UTF_8.name())).generateAsciidoc();
} else {
throw new IllegalArgumentException("Usage: " + DefaultConfiguration.class.getName() + " --generate-html <filename> | --generate-latex <filename>");
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
index 8270ad2..9c770b1 100644
--- a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
+++ b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.html
@@ -86,5 +86,3 @@
default | table.failures.ignore ..................... | false
</pre>
- <h1>Configuration Properties</h1>
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex
deleted file mode 100644
index a1e1dc0..0000000
--- a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.tex
+++ /dev/null
@@ -1,99 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Configuration Management}
-\label{app:config}
-
-\section{Configuration Overview}
-
-All accumulo properties have a default value in the source code. Properties can also be set
-in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
-accumulo will choose the property with the highest precedence. This order of precedence is described
-below (from highest to lowest):
-
-\subsection{Zookeeper table properties}
-Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml
-
-Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- config -t TABLE -s PROPERTY=VALUE
-\end{verbatim}\endgroup
-
-\subsection{Zookeeper system properties}
-
-System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a `yes' in the `Zookeeper Mutable' column in the table below. They are set with the following shell command:
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- config -s PROPERTY=VALUE
-\end{verbatim}\endgroup
-
-If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).
-
-While most system properties take effect immediately, some require a restart of the process which is indicated in `Zookeeper Mutable'.
-
-\subsection{accumulo-site.xml}
-
-Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.
-
-Certain properties (indicated by a `no' in `Zookeeper Mutable') cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.
-
-\subsection{Default Values}
-
-All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.
-
-While the default value is usually optimal, there are cases where a change can increase query and ingest performance.
-
-\section{Configuration in the Shell}
-
-The `config' command in the shell allows you to view the current system configuration. You can also use the `-t' option to view a table's configuration as below:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-
- $ ./bin/accumulo shell -u root
- Enter current password for 'root'@'ac14': ******
-
- Shell - Apache Accumulo Interactive Shell
- -
- - version: 1.5.0
- - instance name: ac14
- - instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
- -
- - type 'help' for a list of available commands
- -
- root@ac13> config -t foo
- ---------+---------------------------------------------+------------------------------------------------------
- SCOPE | NAME | VALUE
- ---------+---------------------------------------------+------------------------------------------------------
- default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
- default | table.bloom.enabled ....................... | false
- default | table.bloom.error.rate .................... | 0.5%
- default | table.bloom.hash.type ..................... | murmur
- default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
- default | table.bloom.load.threshold ................ | 1
- default | table.bloom.size .......................... | 1048576
- default | table.cache.block.enable .................. | false
- default | table.cache.index.enable .................. | false
- default | table.compaction.major.everything.at ...... | 19700101000000GMT
- default | table.compaction.major.everything.idle .... | 1h
- default | table.compaction.major.ratio .............. | 1.3
- site | @override .............................. | 1.4
- system | @override .............................. | 1.5
- table | @override .............................. | 1.6
- default | table.compaction.minor.idle ............... | 5m
- default | table.compaction.minor.logs.threshold ..... | 3
- default | table.failures.ignore ..................... | false
-
-\end{verbatim}\endgroup
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt
----------------------------------------------------------------------
diff --git a/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt
new file mode 100644
index 0000000..d278882
--- /dev/null
+++ b/core/src/main/resources/org/apache/accumulo/core/conf/config-header.txt
@@ -0,0 +1,93 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+[[configuration]]
+== Configuration Management
+
+=== Configuration Overview
+
+All accumulo properties have a default value in the source code. Properties can also be set
+in accumulo-site.xml and in zookeeper on per-table or system-wide basis. If properties are set in more than one location,
+accumulo will choose the property with the highest precedence. This order of precedence is described
+below (from highest to lowest):
+
+==== Zookeeper table properties
+Table properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. While table properties take precedent over system properties, both will override properties set in accumulo-site.xml
+
+Table properties consist of all properties with the table.* prefix. Table properties are configured on a per-table basis using the following shell commmand:
+
+ config -t TABLE -s PROPERTY=VALUE
+
+==== Zookeeper system properties
+
+System properties are applied to the entire cluster when set in zookeeper using the accumulo API or shell. System properties consist of all properties with a `yes` in the _Zookeeper Mutable_ column in the table below. They are set with the following shell command:
+
+ config -s PROPERTY=VALUE
+
+If a table.* property is set using this method, the value will apply to all tables except those configured on per-table basis (which have higher precedence).
+
+While most system properties take effect immediately, some require a restart of the process which is indicated in _Zookeeper Mutable_.
+
+==== accumulo-site.xml
+
+Accumulo processes (master, tserver, etc) read their local accumulo-site.xml on start up. Therefore, changes made to accumulo-site.xml must rsynced across the cluster and processes must be restarted to apply changes.
+
+Certain properties (indicated by a `no` in _Zookeeper Mutable_) cannot be set in zookeeper and only set in this file. The accumulo-site.xml also allows you to configure tablet servers with different settings.
+
+==== Default Values
+
+All properties have a default value in the source code. This value has the lowest precedence and is overriden if set in accumulo-site.xml or zookeeper.
+
+While the default value is usually optimal, there are cases where a change can increase query and ingest performance.
+
+=== Configuration in the Shell
+
+The `config` command in the shell allows you to view the current system configuration. You can also use the `-t` option to view a table's configuration as below:
+
+----
+$ ./bin/accumulo shell -u root
+Enter current password for 'root'@'accumulo-instance': ******
+
+Shell - Apache Accumulo Interactive Shell
+-
+- version: 1.6.0
+- instance name: accumulo-instance
+- instance id: 4f48fa03-f692-43ce-ae03-94c9ea8b7181
+-
+- type 'help' for a list of available commands
+-
+root@accumulo-instance> config -t foo
+---------+---------------------------------------------+------------------------------------------------------
+SCOPE | NAME | VALUE
+---------+---------------------------------------------+------------------------------------------------------
+default | table.balancer ............................ | org.apache.accumulo.server.master.balancer.DefaultLoadBalancer
+default | table.bloom.enabled ....................... | false
+default | table.bloom.error.rate .................... | 0.5%
+default | table.bloom.hash.type ..................... | murmur
+default | table.bloom.key.functor ................... | org.apache.accumulo.core.file.keyfunctor.RowFunctor
+default | table.bloom.load.threshold ................ | 1
+default | table.bloom.size .......................... | 1048576
+default | table.cache.block.enable .................. | false
+default | table.cache.index.enable .................. | false
+default | table.compaction.major.everything.at ...... | 19700101000000GMT
+default | table.compaction.major.everything.idle .... | 1h
+default | table.compaction.major.ratio .............. | 1.3
+site | @override .............................. | 1.4
+system | @override .............................. | 1.5
+table | @override .............................. | 1.6
+default | table.compaction.minor.idle ............... | 5m
+default | table.compaction.minor.logs.threshold ..... | 3
+default | table.failures.ignore ..................... | false
+----
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/pom.xml
----------------------------------------------------------------------
diff --git a/docs/pom.xml b/docs/pom.xml
index c438eb0..4cc75a9 100644
--- a/docs/pom.xml
+++ b/docs/pom.xml
@@ -38,6 +38,48 @@
<build>
<plugins>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-asciidoc</id>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <outputDirectory>${project.build.directory}/asciidoc</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/asciidoc</directory>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.asciidoctor</groupId>
+ <artifactId>asciidoctor-maven-plugin</artifactId>
+ <configuration>
+ <backend>html</backend>
+ <doctype>book</doctype>
+ <embedAssets>true</embedAssets>
+ <imagesDir>${project.build.directory}/asciidoc/images</imagesDir>
+ <sourceDirectory>${project.build.directory}/asciidoc</sourceDirectory>
+ <sourceHighlighter>highlightjs</sourceHighlighter>
+ </configuration>
+ <executions>
+ <execution>
+ <id>output-html</id>
+ <goals>
+ <goal>process-asciidoc</goal>
+ </goals>
+ <phase>prepare-package</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<executions>
@@ -51,7 +93,7 @@
<executable>mkdir</executable>
<arguments>
<argument>-p</argument>
- <argument>${project.build.directory}/latex/accumulo_user_manual/appendices</argument>
+ <argument>${project.build.directory}/asciidoc/appendices</argument>
</arguments>
</configuration>
</execution>
@@ -80,8 +122,8 @@
<mainClass>org.apache.accumulo.core.conf.DefaultConfiguration</mainClass>
<classpathScope>compile</classpathScope>
<arguments>
- <argument>--generate-latex</argument>
- <argument>${project.build.directory}/latex/accumulo_user_manual/appendices/config.tex</argument>
+ <argument>--generate-asciidoc</argument>
+ <argument>${project.build.directory}/asciidoc/appendices/config.txt</argument>
</arguments>
</configuration>
</execution>
@@ -89,31 +131,18 @@
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
- <artifactId>latex-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>build-pdf-manuals</id>
- <goals>
- <goal>latex</goal>
- </goals>
- <phase>prepare-package</phase>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
- <id>attach-user-manual-pdf</id>
+ <id>attach-user-manual-html</id>
<goals>
<goal>attach-artifact</goal>
</goals>
<configuration>
<artifacts>
<artifact>
- <file>${project.build.directory}/accumulo_user_manual.pdf</file>
- <type>pdf</type>
+ <file>${project.build.directory}/generated-docs/accumulo_user_manual.html</file>
+ <type>html</type>
<classifier>user-manual</classifier>
</artifact>
</artifacts>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/accumulo_user_manual.asciidoc b/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
new file mode 100644
index 0000000..e3f9d4b
--- /dev/null
+++ b/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+Apache Accumulo User Manual Version 1.6
+=======================================
+:author: Apache Accumulo Project
+:email: dev@accumulo.apache.org
+//:theme: flask
+:toc2:
+:toclevels: 4
+:toc-title: Apache Accumulo 1.6
+:numbered:
+//:max-width: 50em
+:website: http://accumulo.apache.org/
+
+image::accumulo-logo.png[]
+
+Copyright © 2011-2014 The Apache Software Foundation, Licensed under the Apache
+License, Version 2.0. Apache Accumulo, Accumulo, Apache, and the Apache
+Accumulo project logo are trademarks of the Apache Software Foundation.
+
+include::chapters/introduction.txt[]
+
+include::chapters/design.txt[]
+
+include::chapters/shell.txt[]
+
+include::chapters/clients.txt[]
+
+include::chapters/development_clients.txt[]
+
+include::chapters/table_configuration.txt[]
+
+include::chapters/table_design.txt[]
+
+include::chapters/high_speed_ingest.txt[]
+
+include::chapters/analytics.txt[]
+
+include::chapters/security.txt[]
+
+include::chapters/administration.txt[]
+
+include::chapters/multivolume.txt[]
+
+include::chapters/troubleshooting.txt[]
+
+[appendix]
+include::appendices/config.txt[]
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/administration.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/administration.txt b/docs/src/main/asciidoc/chapters/administration.txt
new file mode 100644
index 0000000..4dbcd1b
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/administration.txt
@@ -0,0 +1,396 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Administration
+
+=== Hardware
+
+Because we are running essentially two or three systems simultaneously layered
+across the cluster: HDFS, Accumulo and MapReduce, it is typical for hardware to
+consist of 4 to 8 cores, and 8 to 32 GB RAM. This is so each running process can have
+at least one core and 2 - 4 GB each.
+
+One core running HDFS can typically keep 2 to 4 disks busy, so each machine may
+typically have as little as 2 x 300GB disks and as much as 4 x 1TB or 2TB disks.
+
+It is possible to do with less than this, such as with 1u servers with 2 cores and 4GB
+each, but in this case it is recommended to only run up to two processes per
+machine -- i.e. DataNode and TabletServer or DataNode and MapReduce worker but
+not all three. The constraint here is having enough available heap space for all the
+processes on a machine.
+
+=== Network
+
+Accumulo communicates via remote procedure calls over TCP/IP for both passing
+data and control messages. In addition, Accumulo uses HDFS clients to
+communicate with HDFS. To achieve good ingest and query performance, sufficient
+network bandwidth must be available between any two machines.
+
+In addition to needing access to ports associated with HDFS and ZooKeeper, Accumulo will
+use the following default ports. Please make sure that they are open, or change
+their value in conf/accumulo-site.xml.
+
+.Accumulo default ports
+[width="75%",cols=">,^2,^2"]
+[options="header"]
+|====
+|Port | Description | Property Name
+|4445 | Shutdown Port (Accumulo MiniCluster) | n/a
+|4560 | Accumulo monitor (for centralized log display) | monitor.port.log4j
+|9997 | Tablet Server | tserver.port.client
+|9999 | Master Server | master.port.client
+|12234 | Accumulo Tracer | trace.port.client
+|42424 | Accumulo Proxy Server | n/a
+|50091 | Accumulo GC | gc.port.client
+|50095 | Accumulo HTTP monitor | monitor.port.client
+|====
+
+In addition, the user can provide +0+ and an ephemeral port will be chosen instead. This
+ephemeral port is likely to be unique and not already bound. Thus, configuring ports to
+use +0+ instead of an explicit value, should, in most cases, work around any issues of
+running multiple distinct Accumulo instances (or any other process which tries to use the
+same default ports) on the same hardware.
+
+=== Installation
+Choose a directory for the Accumulo installation. This directory will be referenced
+by the environment variable +$ACCUMULO_HOME+. Run the following:
+
+ $ tar xzf accumulo-1.6.0-bin.tar.gz # unpack to subdirectory
+ $ mv accumulo-1.6.0 $ACCUMULO_HOME # move to desired location
+
+Repeat this step at each machine within the cluster. Usually all machines have the
+same +$ACCUMULO_HOME+.
+
+=== Dependencies
+Accumulo requires HDFS and ZooKeeper to be configured and running
+before starting. Password-less SSH should be configured between at least the
+Accumulo master and TabletServer machines. It is also a good idea to run Network
+Time Protocol (NTP) within the cluster to ensure nodes' clocks don't get too out of
+sync, which can cause problems with automatically timestamped data.
+
+=== Configuration
+
+Accumulo is configured by editing several Shell and XML files found in
++$ACCUMULO_HOME/conf+. The structure closely resembles Hadoop's configuration
+files.
+
+==== Edit conf/accumulo-env.sh
+
+Accumulo needs to know where to find the software it depends on. Edit accumulo-env.sh
+and specify the following:
+
+. Enter the location of the installation directory of Accumulo for +$ACCUMULO_HOME+
+. Enter your system's Java home for +$JAVA_HOME+
+. Enter the location of Hadoop for +$HADOOP_PREFIX+
+. Choose a location for Accumulo logs and enter it for +$ACCUMULO_LOG_DIR+
+. Enter the location of ZooKeeper for +$ZOOKEEPER_HOME+
+
+By default Accumulo TabletServers are set to use 1GB of memory. You may change
+this by altering the value of +$ACCUMULO_TSERVER_OPTS+. Note the syntax is that of
+the Java JVM command line options. This value should be less than the physical
+memory of the machines running TabletServers.
+
+There are similar options for the master's memory usage and the garbage collector
+process. Reduce these if they exceed the physical RAM of your hardware and
+increase them, within the bounds of the physical RAM, if a process fails because of
+insufficient memory.
+
+Note that you will be specifying the Java heap space in accumulo-env.sh. You should
+make sure that the total heap space used for the Accumulo tserver and the Hadoop
+DataNode and TaskTracker is less than the available memory on each slave node in
+the cluster. On large clusters, it is recommended that the Accumulo master, Hadoop
+NameNode, secondary NameNode, and Hadoop JobTracker all be run on separate
+machines to allow them to use more heap space. If you are running these on the
+same machine on a small cluster, likewise make sure their heap space settings fit
+within the available memory.
+
+==== Native Map
+
+The tablet server uses a data structure called a MemTable to store sorted key/value
+pairs in memory when they are first received from the client. When a minor compaction
+occurs, this data structure is written to HDFS. The MemTable will default to using
+memory in the JVM but a JNI version, called the native map, can be used to significantly
+speed up performance by utilizing the memory space of the native operating system. The
+native map also avoids the performance implications brought on by garbage collection
+in the JVM by causing it to pause much less frequently.
+
+32-bit and 64-bit Linux versions of the native map ship with the Accumulo dist package.
+For other operating systems, the native map can be built from the codebase in two ways-
+from maven or from the Makefile.
+
+. Build from maven using the following command: +mvn clean package -Pnative+.
+. Build from the c++ source by running +make+ in the `$ACCUMULO_HOME/server/src/main/c++` directory.
+
+After building the native map from the source, you will find the artifact in
++$ACCUMULO_HOME/lib/native+. Upon starting up, the tablet server will look
+in this directory for the map library. If the file is renamed or moved from its
+target directory, the tablet server may not be able to find it.
+
+==== Cluster Specification
+
+On the machine that will serve as the Accumulo master:
+
+. Write the IP address or domain name of the Accumulo Master to the +$ACCUMULO_HOME/conf/masters+ file.
+. Write the IP addresses or domain name of the machines that will be TabletServers in +$ACCUMULO_HOME/conf/slaves+, one per line.
+
+Note that if using domain names rather than IP addresses, DNS must be configured
+properly for all machines participating in the cluster. DNS can be a confusing source
+of errors.
+
+==== Accumulo Settings
+Specify appropriate values for the following settings in
++$ACCUMULO_HOME/conf/accumulo-site.xml+ :
+
+[source,xml]
+<property>
+ <name>instance.zookeeper.host</name>
+ <value>zooserver-one:2181,zooserver-two:2181</value>
+ <description>list of zookeeper servers</description>
+</property>
+
+This enables Accumulo to find ZooKeeper. Accumulo uses ZooKeeper to coordinate
+settings between processes and helps finalize TabletServer failure.
+
+[source,xml]
+<property>
+ <name>instance.secret</name>
+ <value>DEFAULT</value>
+</property>
+
+The instance needs a secret to enable secure communication between servers. Configure your
+secret and make sure that the +accumulo-site.xml+ file is not readable to other users.
+
+Some settings can be modified via the Accumulo shell and take effect immediately, but
+some settings require a process restart to take effect. See the configuration documentation
+(available in the docs directory of the tarball and in <<configuration>>) for details.
+
+==== Deploy Configuration
+
+Copy the masters, slaves, accumulo-env.sh, and if necessary, accumulo-site.xml
+from the +$ACCUMULO_HOME/conf/+ directory on the master to all the machines
+specified in the slaves file.
+
+=== Initialization
+
+Accumulo must be initialized to create the structures it uses internally to locate
+data across the cluster. HDFS is required to be configured and running before
+Accumulo can be initialized.
+
+Once HDFS is started, initialization can be performed by executing
++$ACCUMULO_HOME/bin/accumulo init+ . This script will prompt for a name
+for this instance of Accumulo. The instance name is used to identify a set of tables
+and instance-specific settings. The script will then write some information into
+HDFS so Accumulo can start properly.
+
+The initialization script will prompt you to set a root password. Once Accumulo is
+initialized it can be started.
+
+=== Running
+
+==== Starting Accumulo
+
+Make sure Hadoop is configured on all of the machines in the cluster, including
+access to a shared HDFS instance. Make sure HDFS and ZooKeeper are running.
+Make sure ZooKeeper is configured and running on at least one machine in the
+cluster.
+Start Accumulo using the +bin/start-all.sh+ script.
+
+To verify that Accumulo is running, check the Status page as described under
+_Monitoring_. In addition, the Shell can provide some information about the status of
+tables via reading the metadata tables.
+
+==== Stopping Accumulo
+
+To shutdown cleanly, run +bin/stop-all.sh+ and the master will orchestrate the
+shutdown of all the tablet servers. Shutdown waits for all minor compactions to finish, so it may
+take some time for particular configurations.
+
+==== Adding a Node
+
+Update your +$ACCUMULO_HOME/conf/slaves+ (or +$ACCUMULO_CONF_DIR/slaves+) file to account for the addition.
+
+ $ACCUMULO_HOME/bin/accumulo admin start <host(s)> {<host> ...}
+
+Alternatively, you can ssh to each of the hosts you want to add and run:
+
+ $ACCUMULO_HOME/bin/start-here.sh
+
+Make sure the host in question has the new configuration, or else the tablet
+server won't start; at a minimum this needs to be on the host(s) being added,
+but in practice it's good to ensure consistent configuration across all nodes.
+
+==== Decomissioning a Node
+
+If you need to take a node out of operation, you can trigger a graceful shutdown of a tablet
+server. Accumulo will automatically rebalance the tablets across the available tablet servers.
+
+ $ACCUMULO_HOME/bin/accumulo admin stop <host(s)> {<host> ...}
+
+Alternatively, you can ssh to each of the hosts you want to remove and run:
+
+ $ACCUMULO_HOME/bin/stop-here.sh
+
+Be sure to update your +$ACCUMULO_HOME/conf/slaves+ (or +$ACCUMULO_CONF_DIR/slaves+) file to
+account for the removal of these hosts. Bear in mind that the monitor will not re-read the
+slaves file automatically, so it will report the decomissioned servers as down; it's
+recommended that you restart the monitor so that the node list is up to date.
+
+=== Monitoring
+
+The Accumulo Master provides an interface for monitoring the status and health of
+Accumulo components. This interface can be accessed by pointing a web browser to
++http://accumulomaster:50095/status+
+
+=== Tracing
+It can be difficult to determine why some operations are taking longer
+than expected. For example, you may be looking up items with very low
+latency, but sometimes the lookups take much longer. Determining the
+cause of the delay is difficult because the system is distributed, and
+the typical lookup is fast.
+
+Accumulo has been instrumented to record the time that various
+operations take when tracing is turned on. The fact that tracing is
+enabled follows all the requests made on behalf of the user throughout
+the distributed infrastructure of accumulo, and across all threads of
+execution.
+
+These time spans will be inserted into the +trace+ table in
+Accumulo. You can browse recent traces from the Accumulo monitor
+page. You can also read the +trace+ table directly like any
+other table.
+
+The design of Accumulo's distributed tracing follows that of
+http://research.google.com/pubs/pub36356.html[Google's Dapper].
+
+==== Tracers
+To collect traces, Accumulo needs at least one server listed in
+ +$ACCUMULO_HOME/conf/tracers+. The server collects traces
+from clients and writes them to the +trace+ table. The Accumulo
+user that the tracer connects to Accumulo with can be configured with
+the following properties
+
+ trace.user
+ trace.token.property.password
+
+==== Instrumenting a Client
+Tracing can be used to measure a client operation, such as a scan, as
+the operation traverses the distributed system. To enable tracing for
+your application call
+
+[source,java]
+DistributedTrace.enable(instance, new ZooReader(instance), hostname, "myApplication");
+
+Once tracing has been enabled, a client can wrap an operation in a trace.
+
+[source,java]
+Trace.on("Client Scan");
+BatchScanner scanner = conn.createBatchScanner(...);
+// Configure your scanner
+for (Entry entry : scanner) {
+}
+Trace.off();
+
+Additionally, the user can create additional Spans within a Trace.
+
+[source,java]
+Trace.on("Client Update");
+...
+Span readSpan = Trace.start("Read");
+...
+readSpan.stop();
+...
+Span writeSpan = Trace.start("Write");
+...
+writeSpan.stop();
+Trace.off();
+
+Like Dapper, Accumulo tracing supports user defined annotations to associate additional data with a Trace.
+
+[source,java]
+...
+int numberOfEntriesRead = 0;
+Span readSpan = Trace.start("Read");
+// Do the read, update the counter
+...
+readSpan.data("Number of Entries Read", String.valueOf(numberOfEntriesRead));
+
+Some client operations may have a high volume within your
+application. As such, you may wish to only sample a percentage of
+operations for tracing. As seen below, the CountSampler can be used to
+help enable tracing for 1-in-1000 operations
+
+[source,java]
+Sampler sampler = new CountSampler(1000);
+...
+if (sampler.next()) {
+ Trace.on("Read");
+}
+...
+Trace.offNoFlush();
+
+It should be noted that it is safe to turn off tracing even if it
+isn't currently active. The +Trace.offNoFlush()+ should be used if the
+user does not wish to have +Trace.off()+ block while flushing trace
+data.
+
+==== Viewing Collected Traces
+To view collected traces, use the "Recent Traces" link on the Monitor
+UI. You can also programmatically access and print traces using the
++TraceDump+ class.
+
+==== Tracing from the Shell
+You can enable tracing for operations run from the shell by using the
++trace on+ and +trace off+ commands.
+
+----
+root@test test> trace on
+
+root@test test> scan
+a b:c [] d
+
+root@test test> trace off
+Waiting for trace information
+Waiting for trace information
+Trace started at 2013/08/26 13:24:08.332
+Time Start Service@Location Name
+ 3628+0 shell@localhost shell:root
+ 8+1690 shell@localhost scan
+ 7+1691 shell@localhost scan:location
+ 6+1692 tserver@localhost startScan
+ 5+1692 tserver@localhost tablet read ahead 6
+----
+
+=== Logging
+Accumulo processes each write to a set of log files. By default these are found under
++$ACCUMULO/logs/+.
+
+=== Recovery
+
+In the event of TabletServer failure or error on shutting Accumulo down, some
+mutations may not have been minor compacted to HDFS properly. In this case,
+Accumulo will automatically reapply such mutations from the write-ahead log
+either when the tablets from the failed server are reassigned by the Master (in the
+case of a single TabletServer failure) or the next time Accumulo starts (in the event of
+failure during shutdown).
+
+Recovery is performed by asking a tablet server to sort the logs so that tablets can easily find their missing
+updates. The sort status of each file is displayed on
+Accumulo monitor status page. Once the recovery is complete any
+tablets involved should return to an ``online'' state. Until then those tablets will be
+unavailable to clients.
+
+The Accumulo client library is configured to retry failed mutations and in many
+cases clients will be able to continue processing after the recovery process without
+throwing an exception.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/analytics.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/analytics.txt b/docs/src/main/asciidoc/chapters/analytics.txt
new file mode 100644
index 0000000..00e0403
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/analytics.txt
@@ -0,0 +1,229 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Analytics
+
+Accumulo supports more advanced data processing than simply keeping keys
+sorted and performing efficient lookups. Analytics can be developed by using
+MapReduce and Iterators in conjunction with Accumulo tables.
+
+=== MapReduce
+
+Accumulo tables can be used as the source and destination of MapReduce jobs. To
+use an Accumulo table with a MapReduce job (specifically with the new Hadoop API
+as of version 0.20), configure the job parameters to use the AccumuloInputFormat
+and AccumuloOutputFormat. Accumulo specific parameters can be set via these
+two format classes to do the following:
+
+* Authenticate and provide user credentials for the input
+* Restrict the scan to a range of rows
+* Restrict the input to a subset of available columns
+
+==== Mapper and Reducer classes
+
+To read from an Accumulo table create a Mapper with the following class
+parameterization and be sure to configure the AccumuloInputFormat.
+
+[source,java]
+class MyMapper extends Mapper<Key,Value,WritableComparable,Writable> {
+ public void map(Key k, Value v, Context c) {
+ // transform key and value data here
+ }
+}
+
+To write to an Accumulo table, create a Reducer with the following class
+parameterization and be sure to configure the AccumuloOutputFormat. The key
+emitted from the Reducer identifies the table to which the mutation is sent. This
+allows a single Reducer to write to more than one table if desired. A default table
+can be configured using the AccumuloOutputFormat, in which case the output table
+name does not have to be passed to the Context object within the Reducer.
+
+[source,java]
+class MyReducer extends Reducer<WritableComparable, Writable, Text, Mutation> {
+ public void reduce(WritableComparable key, Iterable<Text> values, Context c) {
+ Mutation m;
+ // create the mutation based on input key and value
+ c.write(new Text("output-table"), m);
+ }
+}
+
+The Text object passed as the output should contain the name of the table to which
+this mutation should be applied. The Text can be null in which case the mutation
+will be applied to the default table name specified in the AccumuloOutputFormat
+options.
+
+==== AccumuloInputFormat options
+
+[source,java]
+----
+Job job = new Job(getConf());
+AccumuloInputFormat.setInputInfo(job,
+ "user",
+ "passwd".getBytes(),
+ "table",
+ new Authorizations());
+
+AccumuloInputFormat.setZooKeeperInstance(job, "myinstance",
+ "zooserver-one,zooserver-two");
+----
+
+*Optional Settings:*
+
+To restrict Accumulo to a set of row ranges:
+
+[source,java]
+ArrayList<Range> ranges = new ArrayList<Range>();
+// populate array list of row ranges ...
+AccumuloInputFormat.setRanges(job, ranges);
+
+To restrict Accumulo to a list of columns:
+
+[source,java]
+ArrayList<Pair<Text,Text>> columns = new ArrayList<Pair<Text,Text>>();
+// populate list of columns
+AccumuloInputFormat.fetchColumns(job, columns);
+
+To use a regular expression to match row IDs:
+
+[source,java]
+IteratorSetting is = new IteratorSetting(30, RexExFilter.class);
+RegExFilter.setRegexs(is, ".*suffix", null, null, null, true);
+AccumuloInputFormat.addIterator(job, is);
+
+==== AccumuloMultiTableInputFormat options
+
+The AccumuloMultiTableInputFormat allows the scanning over multiple tables
+in a single MapReduce job. Separate ranges, columns, and iterators can be
+used for each table.
+
+[source,java]
+InputTableConfig tableOneConfig = new InputTableConfig();
+InputTableConfig tableTwoConfig = new InputTableConfig();
+
+To set the configuration objects on the job:
+
+[source,java]
+Map<String, InputTableConfig> configs = new HashMap<String,InputTableConfig>();
+configs.put("table1", tableOneConfig);
+configs.put("table2", tableTwoConfig);
+AccumuloMultiTableInputFormat.setInputTableConfigs(job, configs);
+
+*Optional settings:*
+
+To restrict to a set of ranges:
+
+[source,java]
+ArrayList<Range> tableOneRanges = new ArrayList<Range>();
+ArrayList<Range> tableTwoRanges = new ArrayList<Range>();
+// populate array lists of row ranges for tables...
+tableOneConfig.setRanges(tableOneRanges);
+tableTwoConfig.setRanges(tableTwoRanges);
+
+To restrict Accumulo to a list of columns:
+
+[source,java]
+ArrayList<Pair<Text,Text>> tableOneColumns = new ArrayList<Pair<Text,Text>>();
+ArrayList<Pair<Text,Text>> tableTwoColumns = new ArrayList<Pair<Text,Text>>();
+// populate lists of columns for each of the tables ...
+tableOneConfig.fetchColumns(tableOneColumns);
+tableTwoConfig.fetchColumns(tableTwoColumns);
+
+To set scan iterators:
+
+[source,java]
+List<IteratorSetting> tableOneIterators = new ArrayList<IteratorSetting>();
+List<IteratorSetting> tableTwoIterators = new ArrayList<IteratorSetting>();
+// populate the lists of iterator settings for each of the tables ...
+tableOneConfig.setIterators(tableOneIterators);
+tableTwoConfig.setIterators(tableTwoIterators);
+
+
+The name of the table can be retrieved from the input split:
+
+[source,java]
+class MyMapper extends Mapper<Key,Value,WritableComparable,Writable> {
+ public void map(Key k, Value v, Context c) {
+ RangeInputSplit split = (RangeInputSplit)c.getInputSplit();
+ String tableName = split.getTableName();
+ // do something with table name
+ }
+}
+
+
+==== AccumuloOutputFormat options
+
+[source,java]
+----
+boolean createTables = true;
+String defaultTable = "mytable";
+
+AccumuloOutputFormat.setOutputInfo(job,
+ "user",
+ "passwd".getBytes(),
+ createTables,
+ defaultTable);
+
+AccumuloOutputFormat.setZooKeeperInstance(job, "myinstance",
+ "zooserver-one,zooserver-two");
+----
+
+*Optional Settings:*
+
+[source,java]
+AccumuloOutputFormat.setMaxLatency(job, 300000); // milliseconds
+AccumuloOutputFormat.setMaxMutationBufferSize(job, 50000000); // bytes
+
+An example of using MapReduce with Accumulo can be found at
++accumulo/docs/examples/README.mapred+.
+
+=== Combiners
+
+Many applications can benefit from the ability to aggregate values across common
+keys. This can be done via Combiner iterators and is similar to the Reduce step in
+MapReduce. This provides the ability to define online, incrementally updated
+analytics without the overhead or latency associated with batch-oriented
+MapReduce jobs.
+
+All that is needed to aggregate values of a table is to identify the fields over which
+values will be grouped, insert mutations with those fields as the key, and configure
+the table with a combining iterator that supports the summarizing operation
+desired.
+
+The only restriction on an combining iterator is that the combiner developer
+should not assume that all values for a given key have been seen, since new
+mutations can be inserted at anytime. This precludes using the total number of
+values in the aggregation such as when calculating an average, for example.
+
+==== Feature Vectors
+
+An interesting use of combining iterators within an Accumulo table is to store
+feature vectors for use in machine learning algorithms. For example, many
+algorithms such as k-means clustering, support vector machines, anomaly detection,
+etc. use the concept of a feature vector and the calculation of distance metrics to
+learn a particular model. The columns in an Accumulo table can be used to efficiently
+store sparse features and their weights to be incrementally updated via the use of an
+combining iterator.
+
+=== Statistical Modeling
+
+Statistical models that need to be updated by many machines in parallel could be
+similarly stored within an Accumulo table. For example, a MapReduce job that is
+iteratively updating a global statistical model could have each map or reduce worker
+reference the parts of the model to be read and updated through an embedded
+Accumulo client.
+
+Using Accumulo this way enables efficient and fast lookups and updates of small
+pieces of information in a random access pattern, which is complementary to
+MapReduce's sequential access model.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/clients.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/clients.txt b/docs/src/main/asciidoc/chapters/clients.txt
new file mode 100644
index 0000000..3e0b845
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/clients.txt
@@ -0,0 +1,320 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Writing Accumulo Clients
+
+=== Running Client Code
+
+There are multiple ways to run Java code that uses Accumulo. Below is a list
+of the different ways to execute client code.
+
+* using java executable
+* using the accumulo script
+* using the tool script
+
+In order to run client code written to run against Accumulo, you will need to
+include the jars that Accumulo depends on in your classpath. Accumulo client
+code depends on Hadoop and Zookeeper. For Hadoop add the hadoop client jar, all
+of the jars in the Hadoop lib directory, and the conf directory to the
+classpath. For Zookeeper 3.3 you only need to add the Zookeeper jar, and not
+what is in the Zookeeper lib directory. You can run the following command on a
+configured Accumulo system to see what its using for its classpath.
+
+ $ACCUMULO_HOME/bin/accumulo classpath
+
+Another option for running your code is to put a jar file in
++$ACCUMULO_HOME/lib/ext+. After doing this you can use the accumulo
+script to execute your code. For example if you create a jar containing the
+class +com.foo.Client+ and placed that in +lib/ext+, then you could use the command
++$ACCUMULO_HOME/bin/accumulo com.foo.Client+ to execute your code.
+
+If you are writing map reduce job that access Accumulo, then you can use the
+bin/tool.sh script to run those jobs. See the map reduce example.
+
+=== Connecting
+
+All clients must first identify the Accumulo instance to which they will be
+communicating. Code to do this is as follows:
+
+[source,java]
+----
+String instanceName = "myinstance";
+String zooServers = "zooserver-one,zooserver-two"
+Instance inst = new ZooKeeperInstance(instanceName, zooServers);
+
+Connector conn = inst.getConnector("user", new PasswordToken("passwd"));
+----
+
+=== Writing Data
+
+Data are written to Accumulo by creating Mutation objects that represent all the
+changes to the columns of a single row. The changes are made atomically in the
+TabletServer. Clients then add Mutations to a BatchWriter which submits them to
+the appropriate TabletServers.
+
+Mutations can be created thus:
+
+[source,java]
+----
+Text rowID = new Text("row1");
+Text colFam = new Text("myColFam");
+Text colQual = new Text("myColQual");
+ColumnVisibility colVis = new ColumnVisibility("public");
+long timestamp = System.currentTimeMillis();
+
+Value value = new Value("myValue".getBytes());
+
+Mutation mutation = new Mutation(rowID);
+mutation.put(colFam, colQual, colVis, timestamp, value);
+----
+
+==== BatchWriter
+The BatchWriter is highly optimized to send Mutations to multiple TabletServers
+and automatically batches Mutations destined for the same TabletServer to
+amortize network overhead. Care must be taken to avoid changing the contents of
+any Object passed to the BatchWriter since it keeps objects in memory while
+batching.
+
+Mutations are added to a BatchWriter thus:
+
+[source,java]
+----
+// BatchWriterConfig has reasonable defaults
+BatchWriterConfig config = new BatchWriterConfig();
+config.setMaxMemory(10000000L); // bytes available to batchwriter for buffering mutations
+
+BatchWriter writer = conn.createBatchWriter("table", config)
+
+writer.add(mutation);
+
+writer.close();
+----
+
+An example of using the batch writer can be found at
++accumulo/docs/examples/README.batch+.
+
+==== ConditionalWriter
+The ConditionalWriter enables efficient, atomic read-modify-write operations on
+rows. The ConditionalWriter writes special Mutations which have a list of per
+column conditions that must all be met before the mutation is applied. The
+conditions are checked in the tablet server while a row lock is
+held\footnote{Mutations written by the BatchWriter will not obtain a row
+lock.}. The conditions that can be checked for a column are equality and
+absence. For example a conditional mutation can require that column A is
+absent inorder to be applied. Iterators can be applied when checking
+conditions. Using iterators, many other operations besides equality and
+absence can be checked. For example, using an iterator that converts values
+less than 5 to 0 and everything else to 1, its possible to only apply a
+mutation when a column is less than 5.
+
+In the case when a tablet server dies after a client sent a conditional
+mutation, its not known if the mutation was applied or not. When this happens
+the ConditionalWriter reports a status of UNKNOWN for the ConditionalMutation.
+In many cases this situation can be dealt with by simply reading the row again
+and possibly sending another conditional mutation. If this is not sufficient,
+then a higher level of abstraction can be built by storing transactional
+information within a row.
+
+An example of using the batch writer can be found at
++accumulo/docs/examples/README.reservations+.
+
+=== Reading Data
+
+Accumulo is optimized to quickly retrieve the value associated with a given key, and
+to efficiently return ranges of consecutive keys and their associated values.
+
+==== Scanner
+
+To retrieve data, Clients use a Scanner, which acts like an Iterator over
+keys and values. Scanners can be configured to start and stop at particular keys, and
+to return a subset of the columns available.
+
+[source,java]
+----
+// specify which visibilities we are allowed to see
+Authorizations auths = new Authorizations("public");
+
+Scanner scan =
+ conn.createScanner("table", auths);
+
+scan.setRange(new Range("harry","john"));
+scan.fetchFamily("attributes");
+
+for(Entry<Key,Value> entry : scan) {
+ String row = entry.getKey().getRow();
+ Value value = entry.getValue();
+}
+----
+
+==== Isolated Scanner
+
+Accumulo supports the ability to present an isolated view of rows when
+scanning. There are three possible ways that a row could change in Accumulo :
+
+* a mutation applied to a table
+* iterators executed as part of a minor or major compaction
+* bulk import of new files
+
+Isolation guarantees that either all or none of the changes made by these
+operations on a row are seen. Use the IsolatedScanner to obtain an isolated
+view of an Accumulo table. When using the regular scanner it is possible to see
+a non isolated view of a row. For example if a mutation modifies three
+columns, it is possible that you will only see two of those modifications.
+With the isolated scanner either all three of the changes are seen or none.
+
+The IsolatedScanner buffers rows on the client side so a large row will not
+crash a tablet server. By default rows are buffered in memory, but the user
+can easily supply their own buffer if they wish to buffer to disk when rows are
+large.
+
+For an example, look at the following
+
+ examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
+
+==== BatchScanner
+
+For some types of access, it is more efficient to retrieve several ranges
+simultaneously. This arises when accessing a set of rows that are not consecutive
+whose IDs have been retrieved from a secondary index, for example.
+
+The BatchScanner is configured similarly to the Scanner; it can be configured to
+retrieve a subset of the columns available, but rather than passing a single Range,
+BatchScanners accept a set of Ranges. It is important to note that the keys returned
+by a BatchScanner are not in sorted order since the keys streamed are from multiple
+TabletServers in parallel.
+
+[source,java]
+----
+ArrayList<Range> ranges = new ArrayList<Range>();
+// populate list of ranges ...
+
+BatchScanner bscan =
+ conn.createBatchScanner("table", auths, 10);
+bscan.setRanges(ranges);
+bscan.fetchFamily("attributes");
+
+for(Entry<Key,Value> entry : scan) {
+ System.out.println(entry.getValue());
+}
+----
+
+An example of the BatchScanner can be found at
++accumulo/docs/examples/README.batch+.
+
+=== Proxy
+
+The proxy API allows the interaction with Accumulo with languages other than Java.
+A proxy server is provided in the codebase and a client can further be generated.
+
+==== Prequisites
+
+The proxy server can live on any node in which the basic client API would work. That
+means it must be able to communicate with the Master, ZooKeepers, NameNode, and the
+DataNodes. A proxy client only needs the ability to communicate with the proxy server.
+
+
+==== Configuration
+
+The configuration options for the proxy server live inside of a properties file. At
+the very least, you need to supply the following properties:
+
+ protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
+ tokenClass=org.apache.accumulo.core.client.security.tokens.PasswordToken
+ port=42424
+ instance=test
+ zookeepers=localhost:2181
+
+You can find a sample configuration file in your distribution:
+
+ $ACCUMULO_HOME/proxy/proxy.properties.
+
+This sample configuration file further demonstrates an ability to back the proxy server
+by MockAccumulo or the MiniAccumuloCluster.
+
+==== Running the Proxy Server
+
+After the properties file holding the configuration is created, the proxy server
+can be started using the following command in the Accumulo distribution (assuming
+your properties file is named +config.properties+):
+
+ $ACCUMULO_HOME/bin/accumulo proxy -p config.properties
+
+==== Creating a Proxy Client
+
+Aside from installing the Thrift compiler, you will also need the language-specific library
+for Thrift installed to generate client code in that language. Typically, your operating
+system's package manager will be able to automatically install these for you in an expected
+location such as +/usr/lib/python/site-packages/thrift+.
+
+You can find the thrift file for generating the client:
+
+ $ACCUMULO_HOME/proxy/proxy.thrift.
+
+After a client is generated, the port specified in the configuration properties above will be
+used to connect to the server.
+
+==== Using a Proxy Client
+
+The following examples have been written in Java and the method signatures may be
+slightly different depending on the language specified when generating client with
+the Thrift compiler. After initiating a connection to the Proxy (see Apache Thrift's
+documentation for examples of connecting to a Thrift service), the methods on the
+proxy client will be available. The first thing to do is log in:
+
+[source,java]
+Map password = new HashMap<String,String>();
+password.put("password", "secret");
+ByteBuffer token = client.login("root", password);
+
+Once logged in, the token returned will be used for most subsequent calls to the client.
+Let's create a table, add some data, scan the table, and delete it.
+
+
+First, create a table.
+
+[source,java]
+client.createTable(token, "myTable", true, TimeType.MILLIS);
+
+
+Next, add some data:
+
+[source,java]
+----
+// first, create a writer on the server
+String writer = client.createWriter(token, "myTable", new WriterOptions());
+
+// build column updates
+Map<ByteBuffer, List<ColumnUpdate> cells> cellsToUpdate = //...
+
+// send updates to the server
+client.updateAndFlush(writer, "myTable", cellsToUpdate);
+
+client.closeWriter(writer);
+----
+
+
+Scan for the data and batch the return of the results on the server:
+
+[source,java]
+----
+String scanner = client.createScanner(token, "myTable", new ScanOptions());
+ScanResult results = client.nextK(scanner, 100);
+
+for(KeyValue keyValue : results.getResultsIterator()) {
+ // do something with results
+}
+
+client.closeScanner(scanner);
+----
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/design.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/design.txt b/docs/src/main/asciidoc/chapters/design.txt
new file mode 100644
index 0000000..f57afc3
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/design.txt
@@ -0,0 +1,179 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Accumulo Design
+
+=== Data Model
+
+Accumulo provides a richer data model than simple key-value stores, but is not a
+fully relational database. Data is represented as key-value pairs, where the key and
+value are comprised of the following elements:
+
+[width="75%",cols="^,^,^,^,^,^"]
+|===========================================================================
+ 5+|Key .3+^.^|Value
+.2+^.^|Row ID 3+|Column .2+^.^|Timestamp
+ |Family |Qualifier |Visibility
+|===========================================================================
+
+All elements of the Key and the Value are represented as byte arrays except for
+Timestamp, which is a Long. Accumulo sorts keys by element and lexicographically
+in ascending order. Timestamps are sorted in descending order so that later
+versions of the same Key appear first in a sequential scan. Tables consist of a set of
+sorted key-value pairs.
+
+=== Architecture
+
+Accumulo is a distributed data storage and retrieval system and as such consists of
+several architectural components, some of which run on many individual servers.
+Much of the work Accumulo does involves maintaining certain properties of the
+data, such as organization, availability, and integrity, across many commodity-class
+machines.
+
+=== Components
+
+An instance of Accumulo includes many TabletServers, one Garbage Collector process,
+one Master server and many Clients.
+
+==== Tablet Server
+
+The TabletServer manages some subset of all the tablets (partitions of tables). This includes receiving writes from clients, persisting writes to a
+write-ahead log, sorting new key-value pairs in memory, periodically
+flushing sorted key-value pairs to new files in HDFS, and responding
+to reads from clients, forming a merge-sorted view of all keys and
+values from all the files it has created and the sorted in-memory
+store.
+
+TabletServers also perform recovery of a tablet
+that was previously on a server that failed, reapplying any writes
+found in the write-ahead log to the tablet.
+
+==== Garbage Collector
+
+Accumulo processes will share files stored in HDFS. Periodically, the Garbage
+Collector will identify files that are no longer needed by any process, and
+delete them. Multiple garbage collectors can be run to provide hot-standby support.
+They will perform leader election among themselves to choose a single active instance.
+
+==== Master
+
+The Accumulo Master is responsible for detecting and responding to TabletServer
+failure. It tries to balance the load across TabletServer by assigning tablets carefully
+and instructing TabletServers to unload tablets when necessary. The Master ensures all
+tablets are assigned to one TabletServer each, and handles table creation, alteration,
+and deletion requests from clients. The Master also coordinates startup, graceful
+shutdown and recovery of changes in write-ahead logs when Tablet servers fail.
+
+Multiple masters may be run. The masters will choose among themselves a single master,
+and the others will become backups if the master should fail.
+
+==== Tracer
+
+The Accumulo Tracer process supports the distributed timing API provided by Accumulo.
+One to many of these processes can be run on a cluster which will write the timing
+information to a given Accumulo table for future reference. Seeing the section on
+Tracing for more information on this support.
+
+==== Monitor
+
+The Accumulo Monitor is a web application that provides a wealth of information about
+the state of an instance. The Monitor shows graphs and tables which contain information
+about read/write rates, cache hit/miss rates, and Accumulo table information such as scan
+rate and active/queued compactions. Additionally, the Monitor should always be the first
+point of entry when attempting to debug an Accumulo problem as it will show high-level problems
+in addition to aggregated errors from all nodes in the cluster. See the section on Monitoring
+for more information.
+
+Multiple Monitors can be run to provide hot-standby support in the face of failure. Due to the
+forwarding of logs from remote hosts to the Monitor, only one Monitor process should be active
+at one time. Leader election will be performed internally to choose the active Monitor.
+
+==== Client
+
+Accumulo includes a client library that is linked to every application. The client
+library contains logic for finding servers managing a particular tablet, and
+communicating with TabletServers to write and retrieve key-value pairs.
+
+=== Data Management
+
+Accumulo stores data in tables, which are partitioned into tablets. Tablets are
+partitioned on row boundaries so that all of the columns and values for a particular
+row are found together within the same tablet. The Master assigns Tablets to one
+TabletServer at a time. This enables row-level transactions to take place without
+using distributed locking or some other complicated synchronization mechanism. As
+clients insert and query data, and as machines are added and removed from the
+cluster, the Master migrates tablets to ensure they remain available and that the
+ingest and query load is balanced across the cluster.
+
+image::data_distribution.png[width=500]
+
+=== Tablet Service
+
+
+When a write arrives at a TabletServer it is written to a Write-Ahead Log and
+then inserted into a sorted data structure in memory called a MemTable. When the
+MemTable reaches a certain size the TabletServer writes out the sorted key-value
+pairs to a file in HDFS called Indexed Sequential Access Method (ISAM)
+file. This process is called a minor compaction. A new MemTable is then created
+and the fact of the compaction is recorded in the Write-Ahead Log.
+
+When a request to read data arrives at a TabletServer, the TabletServer does a
+binary search across the MemTable as well as the in-memory indexes associated
+with each ISAM file to find the relevant values. If clients are performing a
+scan, several key-value pairs are returned to the client in order from the
+MemTable and the set of ISAM files by performing a merge-sort as they are read.
+
+=== Compactions
+
+In order to manage the number of files per tablet, periodically the TabletServer
+performs Major Compactions of files within a tablet, in which some set of ISAM
+files are combined into one file. The previous files will eventually be removed
+by the Garbage Collector. This also provides an opportunity to permanently
+remove deleted key-value pairs by omitting key-value pairs suppressed by a
+delete entry when the new file is created.
+
+=== Splitting
+
+When a table is created it has one tablet. As the table grows its initial
+tablet eventually splits into two tablets. Its likely that one of these
+tablets will migrate to another tablet server. As the table continues to grow,
+its tablets will continue to split and be migrated. The decision to
+automatically split a tablet is based on the size of a tablets files. The
+size threshold at which a tablet splits is configurable per table. In addition
+to automatic splitting, a user can manually add split points to a table to
+create new tablets. Manually splitting a new table can parallelize reads and
+writes giving better initial performance without waiting for automatic
+splitting.
+
+As data is deleted from a table, tablets may shrink. Over time this can lead
+to small or empty tablets. To deal with this, merging of tablets was
+introduced in Accumulo 1.4. This is discussed in more detail later.
+
+=== Fault-Tolerance
+
+If a TabletServer fails, the Master detects it and automatically reassigns the tablets
+assigned from the failed server to other servers. Any key-value pairs that were in
+memory at the time the TabletServer fails are automatically reapplied from the Write-Ahead
+Log(WAL) to prevent any loss of data.
+
+Tablet servers write their WALs directly to HDFS so the logs are available to all tablet
+servers for recovery. To make the recovery process efficient, the updates within a log are
+grouped by tablet. TabletServers can quickly apply the mutations from the sorted logs
+that are destined for the tablets they have now been assigned.
+
+TabletServer failures are noted on the Master's monitor page, accessible via
++http://master-address:50095/monitor+.
+
+image::failure_handling.png[width=500]
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/development_clients.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/development_clients.txt b/docs/src/main/asciidoc/chapters/development_clients.txt
new file mode 100644
index 0000000..fe15c7b
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/development_clients.txt
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+== Development Clients
+
+Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of
+Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If
+you want to write a unit test that uses Accumulo, you need a lot of infrastructure
+in place before your test can run.
+
+=== Mock Accumulo
+
+Mock Accumulo supplies mock implementations for much of the client API. It presently
+does not enforce users, logins, permissions, etc. It does support Iterators and Combiners.
+Note that MockAccumulo holds all data in memory, and will not retain any data or
+settings between runs.
+
+While normal interaction with the Accumulo client looks like this:
+
+[source,java]
+Instance instance = new ZooKeeperInstance(...);
+Connector conn = instance.getConnector(user, passwordToken);
+
+To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance:
+
+[source,java]
+Instance instance = new MockInstance();
+
+In fact, you can use the +--fake+ option to the Accumulo shell and interact with
+MockAccumulo:
+
+----
+$ ./bin/accumulo shell --fake -u root -p ''
+
+Shell - Apache Accumulo Interactive Shell
+-
+- version: 1.6
+- instance name: fake
+- instance id: mock-instance-id
+-
+- type 'help' for a list of available commands
+-
+
+root@fake> createtable test
+
+root@fake test> insert row1 cf cq value
+root@fake test> insert row2 cf cq value2
+root@fake test> insert row3 cf cq value3
+
+root@fake test> scan
+row1 cf:cq [] value
+row2 cf:cq [] value2
+row3 cf:cq [] value3
+
+root@fake test> scan -b row2 -e row2
+row2 cf:cq [] value2
+
+root@fake test>
+----
+
+When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat
+and AccumuloOutputFormat classes:
+
+[source,java]
+// ... set up job configuration
+AccumuloInputFormat.setMockInstance(job, "mockInstance");
+AccumuloOutputFormat.setMockInstance(job, "mockInstance");
+
+=== Mini Accumulo Cluster
+
+While the Mock Accumulo provides a lightweight implementation of the client API for unit
+testing, it is often necessary to write more realistic end-to-end integration tests that
+take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by
+configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well
+as some Tablet Servers. It runs against the local filesystem instead of having to start
+up HDFS.
+
+To start it up, you will need to supply an empty directory and a root password as arguments:
+
+[source,java]
+File tempDirectory = // JUnit and Guava supply mechanisms for creating temp directories
+MiniAccumuloCluster accumulo = new MiniAccumuloCluster(tempDirectory, "password");
+accumulo.start();
+
+Once we have our mini cluster running, we will want to interact with the Accumulo client API:
+
+[source,java]
+Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
+Connector conn = instance.getConnector("root", new PasswordToken("password"));
+
+Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster:
+
+[source,java]
+accumulo.stop();
+// delete your temporary folder
[41/50] [abbrv] ACCUMULO-2573 ACCUMULO-2583 Initial stub test to see
WAL get transmitted to the peer.
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
new file mode 100644
index 0000000..d592716
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.replication;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloClusterImpl;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.test.functional.ConfigurableMacIT;
+import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.RawLocalFileSystem;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ *
+ */
+public class ReplicationIT extends ConfigurableMacIT {
+
+ @Override
+ public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
+ cfg.setNumTservers(1);
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1M");
+ cfg.setProperty(Property.GC_CYCLE_START, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
+ }
+
+ @Test//(timeout = 60 * 1000)
+ public void test() throws Exception {
+ MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
+ ROOT_PASSWORD);
+ peerCfg.setNumTservers(1);
+ peerCfg.setInstanceName("peer");
+ peerCfg.setProperty(Property.REPLICATION_RECEIPT_SERVICE_PORT, "10002");
+ MiniAccumuloClusterImpl peerCluster = peerCfg.build();
+
+ peerCluster.start();
+
+ Connector connMaster = getConnector();
+ Connector connPeer = peerCluster.getConnector("root", ROOT_PASSWORD);
+
+ String peerClusterName = "peer";
+
+ // ...peer = AccumuloReplicaSystem,instanceName,zookeepers
+ connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName,
+ AccumuloReplicaSystem.class.getName() + "," + peerCluster.getInstanceName() + "," + peerCluster.getZooKeepers());
+
+ String masterTable = "master", peerTable = "peer";
+
+ connMaster.tableOperations().create(masterTable);
+ String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
+ Assert.assertNotNull(masterTableId);
+
+ connPeer.tableOperations().create(peerTable);
+ String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
+ Assert.assertNotNull(peerTableId);
+
+ // Replicate this table to the peerClusterName in a table with the peerTableId table id
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
+ connMaster.tableOperations().setProperty(masterTable, Property.TABLE_REPLICATION_TARGETS.getKey() + peerClusterName, peerTableId);
+
+ // Write some data to table1
+ BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
+ for (int rows = 0; rows < 250; rows++) {
+ Mutation m = new Mutation(Integer.toString(rows));
+ for (int cols = 0; cols < 500; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
+ }
+
+ bw.close();
+
+ while (!connMaster.tableOperations().exists(ReplicationTable.NAME)) {
+ Thread.sleep(500);
+ }
+
+ for (int i = 0; i < 10; i++) {
+
+ Scanner s = ReplicationTable.getScanner(connMaster);
+ for (Entry<Key,Value> e : s) {
+ System.out.println(e.getKey().toStringNoTruncate() + " " + TextFormat.shortDebugString(Status.parseFrom(e.getValue().get())));
+ }
+
+ System.out.println();
+ System.out.println();
+
+ Thread.sleep(1000);
+ }
+
+ peerCluster.stop();
+ }
+
+}
[31/50] [abbrv] ACCUMULO-2583 First stab at setting up the actual
wire transfer to the replication peer
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java
deleted file mode 100644
index d828d7d..0000000
--- a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java
+++ /dev/null
@@ -1,1917 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.accumulo.core.replication.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings("all") public class RemoteReplicationServicer {
-
- public interface Iface {
-
- public void replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException;
-
- public void replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateLog_call> resultHandler) throws org.apache.thrift.TException;
-
- public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public void replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException
- {
- send_replicateLog(remoteTableId, data);
- recv_replicateLog();
- }
-
- public void send_replicateLog(int remoteTableId, WalEdits data) throws org.apache.thrift.TException
- {
- replicateLog_args args = new replicateLog_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- sendBase("replicateLog", args);
- }
-
- public void recv_replicateLog() throws RemoteReplicationException, org.apache.thrift.TException
- {
- replicateLog_result result = new replicateLog_result();
- receiveBase(result, "replicateLog");
- if (result.e != null) {
- throw result.e;
- }
- return;
- }
-
- public void replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException
- {
- send_replicateKeyValues(remoteTableId, data);
- recv_replicateKeyValues();
- }
-
- public void send_replicateKeyValues(int remoteTableId, KeyValues data) throws org.apache.thrift.TException
- {
- replicateKeyValues_args args = new replicateKeyValues_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- sendBase("replicateKeyValues", args);
- }
-
- public void recv_replicateKeyValues() throws RemoteReplicationException, org.apache.thrift.TException
- {
- replicateKeyValues_result result = new replicateKeyValues_result();
- receiveBase(result, "replicateKeyValues");
- if (result.e != null) {
- throw result.e;
- }
- return;
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- replicateLog_call method_call = new replicateLog_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class replicateLog_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int remoteTableId;
- private WalEdits data;
- public replicateLog_call(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.remoteTableId = remoteTableId;
- this.data = data;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateLog", org.apache.thrift.protocol.TMessageType.CALL, 0));
- replicateLog_args args = new replicateLog_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws RemoteReplicationException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_replicateLog();
- }
- }
-
- public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- replicateKeyValues_call method_call = new replicateKeyValues_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class replicateKeyValues_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int remoteTableId;
- private KeyValues data;
- public replicateKeyValues_call(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.remoteTableId = remoteTableId;
- this.data = data;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateKeyValues", org.apache.thrift.protocol.TMessageType.CALL, 0));
- replicateKeyValues_args args = new replicateKeyValues_args();
- args.setRemoteTableId(remoteTableId);
- args.setData(data);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public void getResult() throws RemoteReplicationException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- (new Client(prot)).recv_replicateKeyValues();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("replicateLog", new replicateLog());
- processMap.put("replicateKeyValues", new replicateKeyValues());
- return processMap;
- }
-
- public static class replicateLog<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateLog_args> {
- public replicateLog() {
- super("replicateLog");
- }
-
- public replicateLog_args getEmptyArgsInstance() {
- return new replicateLog_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public replicateLog_result getResult(I iface, replicateLog_args args) throws org.apache.thrift.TException {
- replicateLog_result result = new replicateLog_result();
- try {
- iface.replicateLog(args.remoteTableId, args.data);
- } catch (RemoteReplicationException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- public static class replicateKeyValues<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateKeyValues_args> {
- public replicateKeyValues() {
- super("replicateKeyValues");
- }
-
- public replicateKeyValues_args getEmptyArgsInstance() {
- return new replicateKeyValues_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public replicateKeyValues_result getResult(I iface, replicateKeyValues_args args) throws org.apache.thrift.TException {
- replicateKeyValues_result result = new replicateKeyValues_result();
- try {
- iface.replicateKeyValues(args.remoteTableId, args.data);
- } catch (RemoteReplicationException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- }
-
- public static class replicateLog_args implements org.apache.thrift.TBase<replicateLog_args, replicateLog_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_args");
-
- private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateLog_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateLog_argsTupleSchemeFactory());
- }
-
- public int remoteTableId; // required
- public WalEdits data; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- REMOTE_TABLE_ID((short)1, "remoteTableId"),
- DATA((short)2, "data");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // REMOTE_TABLE_ID
- return REMOTE_TABLE_ID;
- case 2: // DATA
- return DATA;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REMOTETABLEID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, WalEdits.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_args.class, metaDataMap);
- }
-
- public replicateLog_args() {
- }
-
- public replicateLog_args(
- int remoteTableId,
- WalEdits data)
- {
- this();
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- this.data = data;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateLog_args(replicateLog_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.remoteTableId = other.remoteTableId;
- if (other.isSetData()) {
- this.data = new WalEdits(other.data);
- }
- }
-
- public replicateLog_args deepCopy() {
- return new replicateLog_args(this);
- }
-
- @Override
- public void clear() {
- setRemoteTableIdIsSet(false);
- this.remoteTableId = 0;
- this.data = null;
- }
-
- public int getRemoteTableId() {
- return this.remoteTableId;
- }
-
- public replicateLog_args setRemoteTableId(int remoteTableId) {
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- return this;
- }
-
- public void unsetRemoteTableId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
- public boolean isSetRemoteTableId() {
- return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- public void setRemoteTableIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
- }
-
- public WalEdits getData() {
- return this.data;
- }
-
- public replicateLog_args setData(WalEdits data) {
- this.data = data;
- return this;
- }
-
- public void unsetData() {
- this.data = null;
- }
-
- /** Returns true if field data is set (has been assigned a value) and false otherwise */
- public boolean isSetData() {
- return this.data != null;
- }
-
- public void setDataIsSet(boolean value) {
- if (!value) {
- this.data = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case REMOTE_TABLE_ID:
- if (value == null) {
- unsetRemoteTableId();
- } else {
- setRemoteTableId((Integer)value);
- }
- break;
-
- case DATA:
- if (value == null) {
- unsetData();
- } else {
- setData((WalEdits)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case REMOTE_TABLE_ID:
- return Integer.valueOf(getRemoteTableId());
-
- case DATA:
- return getData();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case REMOTE_TABLE_ID:
- return isSetRemoteTableId();
- case DATA:
- return isSetData();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateLog_args)
- return this.equals((replicateLog_args)that);
- return false;
- }
-
- public boolean equals(replicateLog_args that) {
- if (that == null)
- return false;
-
- boolean this_present_remoteTableId = true;
- boolean that_present_remoteTableId = true;
- if (this_present_remoteTableId || that_present_remoteTableId) {
- if (!(this_present_remoteTableId && that_present_remoteTableId))
- return false;
- if (this.remoteTableId != that.remoteTableId)
- return false;
- }
-
- boolean this_present_data = true && this.isSetData();
- boolean that_present_data = true && that.isSetData();
- if (this_present_data || that_present_data) {
- if (!(this_present_data && that_present_data))
- return false;
- if (!this.data.equals(that.data))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateLog_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateLog_args typedOther = (replicateLog_args)other;
-
- lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRemoteTableId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetData()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateLog_args(");
- boolean first = true;
-
- sb.append("remoteTableId:");
- sb.append(this.remoteTableId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("data:");
- if (this.data == null) {
- sb.append("null");
- } else {
- sb.append(this.data);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (data != null) {
- data.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateLog_argsStandardSchemeFactory implements SchemeFactory {
- public replicateLog_argsStandardScheme getScheme() {
- return new replicateLog_argsStandardScheme();
- }
- }
-
- private static class replicateLog_argsStandardScheme extends StandardScheme<replicateLog_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // REMOTE_TABLE_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // DATA
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.data = new WalEdits();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
- oprot.writeI32(struct.remoteTableId);
- oprot.writeFieldEnd();
- if (struct.data != null) {
- oprot.writeFieldBegin(DATA_FIELD_DESC);
- struct.data.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateLog_argsTupleSchemeFactory implements SchemeFactory {
- public replicateLog_argsTupleScheme getScheme() {
- return new replicateLog_argsTupleScheme();
- }
- }
-
- private static class replicateLog_argsTupleScheme extends TupleScheme<replicateLog_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetRemoteTableId()) {
- optionals.set(0);
- }
- if (struct.isSetData()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetRemoteTableId()) {
- oprot.writeI32(struct.remoteTableId);
- }
- if (struct.isSetData()) {
- struct.data.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- }
- if (incoming.get(1)) {
- struct.data = new WalEdits();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateLog_result implements org.apache.thrift.TBase<replicateLog_result, replicateLog_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_result");
-
- private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateLog_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateLog_resultTupleSchemeFactory());
- }
-
- public RemoteReplicationException e; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- E((short)1, "e");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // E
- return E;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_result.class, metaDataMap);
- }
-
- public replicateLog_result() {
- }
-
- public replicateLog_result(
- RemoteReplicationException e)
- {
- this();
- this.e = e;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateLog_result(replicateLog_result other) {
- if (other.isSetE()) {
- this.e = new RemoteReplicationException(other.e);
- }
- }
-
- public replicateLog_result deepCopy() {
- return new replicateLog_result(this);
- }
-
- @Override
- public void clear() {
- this.e = null;
- }
-
- public RemoteReplicationException getE() {
- return this.e;
- }
-
- public replicateLog_result setE(RemoteReplicationException e) {
- this.e = e;
- return this;
- }
-
- public void unsetE() {
- this.e = null;
- }
-
- /** Returns true if field e is set (has been assigned a value) and false otherwise */
- public boolean isSetE() {
- return this.e != null;
- }
-
- public void setEIsSet(boolean value) {
- if (!value) {
- this.e = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case E:
- if (value == null) {
- unsetE();
- } else {
- setE((RemoteReplicationException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case E:
- return getE();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case E:
- return isSetE();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateLog_result)
- return this.equals((replicateLog_result)that);
- return false;
- }
-
- public boolean equals(replicateLog_result that) {
- if (that == null)
- return false;
-
- boolean this_present_e = true && this.isSetE();
- boolean that_present_e = true && that.isSetE();
- if (this_present_e || that_present_e) {
- if (!(this_present_e && that_present_e))
- return false;
- if (!this.e.equals(that.e))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateLog_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateLog_result typedOther = (replicateLog_result)other;
-
- lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetE()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateLog_result(");
- boolean first = true;
-
- sb.append("e:");
- if (this.e == null) {
- sb.append("null");
- } else {
- sb.append(this.e);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateLog_resultStandardSchemeFactory implements SchemeFactory {
- public replicateLog_resultStandardScheme getScheme() {
- return new replicateLog_resultStandardScheme();
- }
- }
-
- private static class replicateLog_resultStandardScheme extends StandardScheme<replicateLog_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // E
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.e != null) {
- oprot.writeFieldBegin(E_FIELD_DESC);
- struct.e.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateLog_resultTupleSchemeFactory implements SchemeFactory {
- public replicateLog_resultTupleScheme getScheme() {
- return new replicateLog_resultTupleScheme();
- }
- }
-
- private static class replicateLog_resultTupleScheme extends TupleScheme<replicateLog_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetE()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetE()) {
- struct.e.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateKeyValues_args implements org.apache.thrift.TBase<replicateKeyValues_args, replicateKeyValues_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_args");
-
- private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
- private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateKeyValues_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateKeyValues_argsTupleSchemeFactory());
- }
-
- public int remoteTableId; // required
- public KeyValues data; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- REMOTE_TABLE_ID((short)1, "remoteTableId"),
- DATA((short)2, "data");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // REMOTE_TABLE_ID
- return REMOTE_TABLE_ID;
- case 2: // DATA
- return DATA;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REMOTETABLEID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeyValues.class)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_args.class, metaDataMap);
- }
-
- public replicateKeyValues_args() {
- }
-
- public replicateKeyValues_args(
- int remoteTableId,
- KeyValues data)
- {
- this();
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- this.data = data;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateKeyValues_args(replicateKeyValues_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.remoteTableId = other.remoteTableId;
- if (other.isSetData()) {
- this.data = new KeyValues(other.data);
- }
- }
-
- public replicateKeyValues_args deepCopy() {
- return new replicateKeyValues_args(this);
- }
-
- @Override
- public void clear() {
- setRemoteTableIdIsSet(false);
- this.remoteTableId = 0;
- this.data = null;
- }
-
- public int getRemoteTableId() {
- return this.remoteTableId;
- }
-
- public replicateKeyValues_args setRemoteTableId(int remoteTableId) {
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- return this;
- }
-
- public void unsetRemoteTableId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
- public boolean isSetRemoteTableId() {
- return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- public void setRemoteTableIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
- }
-
- public KeyValues getData() {
- return this.data;
- }
-
- public replicateKeyValues_args setData(KeyValues data) {
- this.data = data;
- return this;
- }
-
- public void unsetData() {
- this.data = null;
- }
-
- /** Returns true if field data is set (has been assigned a value) and false otherwise */
- public boolean isSetData() {
- return this.data != null;
- }
-
- public void setDataIsSet(boolean value) {
- if (!value) {
- this.data = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case REMOTE_TABLE_ID:
- if (value == null) {
- unsetRemoteTableId();
- } else {
- setRemoteTableId((Integer)value);
- }
- break;
-
- case DATA:
- if (value == null) {
- unsetData();
- } else {
- setData((KeyValues)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case REMOTE_TABLE_ID:
- return Integer.valueOf(getRemoteTableId());
-
- case DATA:
- return getData();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case REMOTE_TABLE_ID:
- return isSetRemoteTableId();
- case DATA:
- return isSetData();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateKeyValues_args)
- return this.equals((replicateKeyValues_args)that);
- return false;
- }
-
- public boolean equals(replicateKeyValues_args that) {
- if (that == null)
- return false;
-
- boolean this_present_remoteTableId = true;
- boolean that_present_remoteTableId = true;
- if (this_present_remoteTableId || that_present_remoteTableId) {
- if (!(this_present_remoteTableId && that_present_remoteTableId))
- return false;
- if (this.remoteTableId != that.remoteTableId)
- return false;
- }
-
- boolean this_present_data = true && this.isSetData();
- boolean that_present_data = true && that.isSetData();
- if (this_present_data || that_present_data) {
- if (!(this_present_data && that_present_data))
- return false;
- if (!this.data.equals(that.data))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateKeyValues_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateKeyValues_args typedOther = (replicateKeyValues_args)other;
-
- lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRemoteTableId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetData()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateKeyValues_args(");
- boolean first = true;
-
- sb.append("remoteTableId:");
- sb.append(this.remoteTableId);
- first = false;
- if (!first) sb.append(", ");
- sb.append("data:");
- if (this.data == null) {
- sb.append("null");
- } else {
- sb.append(this.data);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- if (data != null) {
- data.validate();
- }
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateKeyValues_argsStandardSchemeFactory implements SchemeFactory {
- public replicateKeyValues_argsStandardScheme getScheme() {
- return new replicateKeyValues_argsStandardScheme();
- }
- }
-
- private static class replicateKeyValues_argsStandardScheme extends StandardScheme<replicateKeyValues_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // REMOTE_TABLE_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 2: // DATA
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.data = new KeyValues();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
- oprot.writeI32(struct.remoteTableId);
- oprot.writeFieldEnd();
- if (struct.data != null) {
- oprot.writeFieldBegin(DATA_FIELD_DESC);
- struct.data.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateKeyValues_argsTupleSchemeFactory implements SchemeFactory {
- public replicateKeyValues_argsTupleScheme getScheme() {
- return new replicateKeyValues_argsTupleScheme();
- }
- }
-
- private static class replicateKeyValues_argsTupleScheme extends TupleScheme<replicateKeyValues_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetRemoteTableId()) {
- optionals.set(0);
- }
- if (struct.isSetData()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetRemoteTableId()) {
- oprot.writeI32(struct.remoteTableId);
- }
- if (struct.isSetData()) {
- struct.data.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- }
- if (incoming.get(1)) {
- struct.data = new KeyValues();
- struct.data.read(iprot);
- struct.setDataIsSet(true);
- }
- }
- }
-
- }
-
- public static class replicateKeyValues_result implements org.apache.thrift.TBase<replicateKeyValues_result, replicateKeyValues_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_result");
-
- private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new replicateKeyValues_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new replicateKeyValues_resultTupleSchemeFactory());
- }
-
- public RemoteReplicationException e; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- E((short)1, "e");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // E
- return E;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_result.class, metaDataMap);
- }
-
- public replicateKeyValues_result() {
- }
-
- public replicateKeyValues_result(
- RemoteReplicationException e)
- {
- this();
- this.e = e;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public replicateKeyValues_result(replicateKeyValues_result other) {
- if (other.isSetE()) {
- this.e = new RemoteReplicationException(other.e);
- }
- }
-
- public replicateKeyValues_result deepCopy() {
- return new replicateKeyValues_result(this);
- }
-
- @Override
- public void clear() {
- this.e = null;
- }
-
- public RemoteReplicationException getE() {
- return this.e;
- }
-
- public replicateKeyValues_result setE(RemoteReplicationException e) {
- this.e = e;
- return this;
- }
-
- public void unsetE() {
- this.e = null;
- }
-
- /** Returns true if field e is set (has been assigned a value) and false otherwise */
- public boolean isSetE() {
- return this.e != null;
- }
-
- public void setEIsSet(boolean value) {
- if (!value) {
- this.e = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case E:
- if (value == null) {
- unsetE();
- } else {
- setE((RemoteReplicationException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case E:
- return getE();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case E:
- return isSetE();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof replicateKeyValues_result)
- return this.equals((replicateKeyValues_result)that);
- return false;
- }
-
- public boolean equals(replicateKeyValues_result that) {
- if (that == null)
- return false;
-
- boolean this_present_e = true && this.isSetE();
- boolean that_present_e = true && that.isSetE();
- if (this_present_e || that_present_e) {
- if (!(this_present_e && that_present_e))
- return false;
- if (!this.e.equals(that.e))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(replicateKeyValues_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- replicateKeyValues_result typedOther = (replicateKeyValues_result)other;
-
- lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetE()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("replicateKeyValues_result(");
- boolean first = true;
-
- sb.append("e:");
- if (this.e == null) {
- sb.append("null");
- } else {
- sb.append(this.e);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class replicateKeyValues_resultStandardSchemeFactory implements SchemeFactory {
- public replicateKeyValues_resultStandardScheme getScheme() {
- return new replicateKeyValues_resultStandardScheme();
- }
- }
-
- private static class replicateKeyValues_resultStandardScheme extends StandardScheme<replicateKeyValues_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // E
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.e != null) {
- oprot.writeFieldBegin(E_FIELD_DESC);
- struct.e.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class replicateKeyValues_resultTupleSchemeFactory implements SchemeFactory {
- public replicateKeyValues_resultTupleScheme getScheme() {
- return new replicateKeyValues_resultTupleScheme();
- }
- }
-
- private static class replicateKeyValues_resultTupleScheme extends TupleScheme<replicateKeyValues_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetE()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetE()) {
- struct.e.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.e = new RemoteReplicationException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- }
- }
- }
-
- }
-
-}
[17/50] [abbrv] git commit: ACCUMULO-2575 Add comment for critical
ordering in gc with repl
Posted by el...@apache.org.
ACCUMULO-2575 Add comment for critical ordering in gc with repl
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e9a75d13
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e9a75d13
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e9a75d13
Branch: refs/heads/ACCUMULO-378
Commit: e9a75d13943669ee3b8152345d4f0b1126c16645
Parents: f4f7c94
Author: Josh Elser <el...@apache.org>
Authored: Sat May 3 11:35:34 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Sat May 3 11:35:34 2014 -0400
----------------------------------------------------------------------
.../gc/replication/RemoveCompleteReplicationRecords.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e9a75d13/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
----------------------------------------------------------------------
diff --git a/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java b/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
index 1b40cd6..b46416c 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/replication/RemoveCompleteReplicationRecords.java
@@ -117,8 +117,8 @@ public class RemoveCompleteReplicationRecords implements Runnable {
* when that {@link Status} is fully replicated and closed, as defined by {@link StatusUtil#isSafeForRemoval(Status)}.
* @param conn A Connector
* @param bs A BatchScanner to read replication status records from
- * @param bw A BatchWriter to write deletes to the metadata table
- * @param bw A BatchWriter to write deletes to the replication table
+ * @param metaBw A BatchWriter to write deletes to the metadata table
+ * @param replBw A BatchWriter to write deletes to the replication table
* @return Number of records removed
*/
protected long removeCompleteRecords(Connector conn, BatchScanner bs, BatchWriter metaBw, BatchWriter replBw) {
@@ -148,6 +148,8 @@ public class RemoveCompleteReplicationRecords implements Runnable {
log.debug("Issuing delete for {}", k.toStringNoTruncate());
+ // *Must* delete from metadata table first, otherwise we risk re-creating the
+ // record in the replication table and double-replicating the data
Mutation metaMutation = new Mutation(ReplicationSection.getRowPrefix() + row.toString());
metaMutation.putDelete(ReplicationSection.COLF, colq);
try {
[29/50] [abbrv] ACCUMULO-2583 First stab at setting up the actual
wire transfer to the replication peer
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
new file mode 100644
index 0000000..1a40dc4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
@@ -0,0 +1,2145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.accumulo.core.replication.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class ReplicationServicer {
+
+ public interface Iface {
+
+ public ByteBuffer replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException;
+
+ public ByteBuffer replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException;
+
+ }
+
+ public interface AsyncIface {
+
+ public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateLog_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException;
+
+ }
+
+ public static class Client extends org.apache.thrift.TServiceClient implements Iface {
+ public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
+ public Factory() {}
+ public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
+ return new Client(prot);
+ }
+ public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ return new Client(iprot, oprot);
+ }
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol prot)
+ {
+ super(prot, prot);
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ super(iprot, oprot);
+ }
+
+ public ByteBuffer replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ send_replicateLog(remoteTableId, data);
+ return recv_replicateLog();
+ }
+
+ public void send_replicateLog(int remoteTableId, WalEdits data) throws org.apache.thrift.TException
+ {
+ replicateLog_args args = new replicateLog_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ sendBase("replicateLog", args);
+ }
+
+ public ByteBuffer recv_replicateLog() throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ replicateLog_result result = new replicateLog_result();
+ receiveBase(result, "replicateLog");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "replicateLog failed: unknown result");
+ }
+
+ public ByteBuffer replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ send_replicateKeyValues(remoteTableId, data);
+ return recv_replicateKeyValues();
+ }
+
+ public void send_replicateKeyValues(int remoteTableId, KeyValues data) throws org.apache.thrift.TException
+ {
+ replicateKeyValues_args args = new replicateKeyValues_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ sendBase("replicateKeyValues", args);
+ }
+
+ public ByteBuffer recv_replicateKeyValues() throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ replicateKeyValues_result result = new replicateKeyValues_result();
+ receiveBase(result, "replicateKeyValues");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "replicateKeyValues failed: unknown result");
+ }
+
+ }
+ public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
+ public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
+ private org.apache.thrift.async.TAsyncClientManager clientManager;
+ private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
+ public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
+ this.clientManager = clientManager;
+ this.protocolFactory = protocolFactory;
+ }
+ public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
+ return new AsyncClient(protocolFactory, clientManager, transport);
+ }
+ }
+
+ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
+ super(protocolFactory, clientManager, transport);
+ }
+
+ public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ replicateLog_call method_call = new replicateLog_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class replicateLog_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int remoteTableId;
+ private WalEdits data;
+ public replicateLog_call(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.remoteTableId = remoteTableId;
+ this.data = data;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateLog", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ replicateLog_args args = new replicateLog_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public ByteBuffer getResult() throws RemoteReplicationException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_replicateLog();
+ }
+ }
+
+ public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ replicateKeyValues_call method_call = new replicateKeyValues_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class replicateKeyValues_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int remoteTableId;
+ private KeyValues data;
+ public replicateKeyValues_call(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.remoteTableId = remoteTableId;
+ this.data = data;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateKeyValues", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ replicateKeyValues_args args = new replicateKeyValues_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public ByteBuffer getResult() throws RemoteReplicationException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_replicateKeyValues();
+ }
+ }
+
+ }
+
+ public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
+ public Processor(I iface) {
+ super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
+ }
+
+ protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ processMap.put("replicateLog", new replicateLog());
+ processMap.put("replicateKeyValues", new replicateKeyValues());
+ return processMap;
+ }
+
+ public static class replicateLog<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateLog_args> {
+ public replicateLog() {
+ super("replicateLog");
+ }
+
+ public replicateLog_args getEmptyArgsInstance() {
+ return new replicateLog_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public replicateLog_result getResult(I iface, replicateLog_args args) throws org.apache.thrift.TException {
+ replicateLog_result result = new replicateLog_result();
+ try {
+ result.success = iface.replicateLog(args.remoteTableId, args.data);
+ } catch (RemoteReplicationException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ public static class replicateKeyValues<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateKeyValues_args> {
+ public replicateKeyValues() {
+ super("replicateKeyValues");
+ }
+
+ public replicateKeyValues_args getEmptyArgsInstance() {
+ return new replicateKeyValues_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public replicateKeyValues_result getResult(I iface, replicateKeyValues_args args) throws org.apache.thrift.TException {
+ replicateKeyValues_result result = new replicateKeyValues_result();
+ try {
+ result.success = iface.replicateKeyValues(args.remoteTableId, args.data);
+ } catch (RemoteReplicationException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ }
+
+ public static class replicateLog_args implements org.apache.thrift.TBase<replicateLog_args, replicateLog_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_args");
+
+ private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
+ private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateLog_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateLog_argsTupleSchemeFactory());
+ }
+
+ public int remoteTableId; // required
+ public WalEdits data; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ REMOTE_TABLE_ID((short)1, "remoteTableId"),
+ DATA((short)2, "data");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // REMOTE_TABLE_ID
+ return REMOTE_TABLE_ID;
+ case 2: // DATA
+ return DATA;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __REMOTETABLEID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, WalEdits.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_args.class, metaDataMap);
+ }
+
+ public replicateLog_args() {
+ }
+
+ public replicateLog_args(
+ int remoteTableId,
+ WalEdits data)
+ {
+ this();
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ this.data = data;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateLog_args(replicateLog_args other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.remoteTableId = other.remoteTableId;
+ if (other.isSetData()) {
+ this.data = new WalEdits(other.data);
+ }
+ }
+
+ public replicateLog_args deepCopy() {
+ return new replicateLog_args(this);
+ }
+
+ @Override
+ public void clear() {
+ setRemoteTableIdIsSet(false);
+ this.remoteTableId = 0;
+ this.data = null;
+ }
+
+ public int getRemoteTableId() {
+ return this.remoteTableId;
+ }
+
+ public replicateLog_args setRemoteTableId(int remoteTableId) {
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ return this;
+ }
+
+ public void unsetRemoteTableId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
+ public boolean isSetRemoteTableId() {
+ return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ public void setRemoteTableIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
+ }
+
+ public WalEdits getData() {
+ return this.data;
+ }
+
+ public replicateLog_args setData(WalEdits data) {
+ this.data = data;
+ return this;
+ }
+
+ public void unsetData() {
+ this.data = null;
+ }
+
+ /** Returns true if field data is set (has been assigned a value) and false otherwise */
+ public boolean isSetData() {
+ return this.data != null;
+ }
+
+ public void setDataIsSet(boolean value) {
+ if (!value) {
+ this.data = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ if (value == null) {
+ unsetRemoteTableId();
+ } else {
+ setRemoteTableId((Integer)value);
+ }
+ break;
+
+ case DATA:
+ if (value == null) {
+ unsetData();
+ } else {
+ setData((WalEdits)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return Integer.valueOf(getRemoteTableId());
+
+ case DATA:
+ return getData();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return isSetRemoteTableId();
+ case DATA:
+ return isSetData();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateLog_args)
+ return this.equals((replicateLog_args)that);
+ return false;
+ }
+
+ public boolean equals(replicateLog_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_remoteTableId = true;
+ boolean that_present_remoteTableId = true;
+ if (this_present_remoteTableId || that_present_remoteTableId) {
+ if (!(this_present_remoteTableId && that_present_remoteTableId))
+ return false;
+ if (this.remoteTableId != that.remoteTableId)
+ return false;
+ }
+
+ boolean this_present_data = true && this.isSetData();
+ boolean that_present_data = true && that.isSetData();
+ if (this_present_data || that_present_data) {
+ if (!(this_present_data && that_present_data))
+ return false;
+ if (!this.data.equals(that.data))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateLog_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateLog_args typedOther = (replicateLog_args)other;
+
+ lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRemoteTableId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetData()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateLog_args(");
+ boolean first = true;
+
+ sb.append("remoteTableId:");
+ sb.append(this.remoteTableId);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("data:");
+ if (this.data == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.data);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (data != null) {
+ data.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateLog_argsStandardSchemeFactory implements SchemeFactory {
+ public replicateLog_argsStandardScheme getScheme() {
+ return new replicateLog_argsStandardScheme();
+ }
+ }
+
+ private static class replicateLog_argsStandardScheme extends StandardScheme<replicateLog_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // REMOTE_TABLE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // DATA
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.data = new WalEdits();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
+ oprot.writeI32(struct.remoteTableId);
+ oprot.writeFieldEnd();
+ if (struct.data != null) {
+ oprot.writeFieldBegin(DATA_FIELD_DESC);
+ struct.data.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateLog_argsTupleSchemeFactory implements SchemeFactory {
+ public replicateLog_argsTupleScheme getScheme() {
+ return new replicateLog_argsTupleScheme();
+ }
+ }
+
+ private static class replicateLog_argsTupleScheme extends TupleScheme<replicateLog_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRemoteTableId()) {
+ optionals.set(0);
+ }
+ if (struct.isSetData()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetRemoteTableId()) {
+ oprot.writeI32(struct.remoteTableId);
+ }
+ if (struct.isSetData()) {
+ struct.data.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.data = new WalEdits();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class replicateLog_result implements org.apache.thrift.TBase<replicateLog_result, replicateLog_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateLog_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateLog_resultTupleSchemeFactory());
+ }
+
+ public ByteBuffer success; // required
+ public RemoteReplicationException e; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success"),
+ E((short)1, "e");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ case 1: // E
+ return E;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_result.class, metaDataMap);
+ }
+
+ public replicateLog_result() {
+ }
+
+ public replicateLog_result(
+ ByteBuffer success,
+ RemoteReplicationException e)
+ {
+ this();
+ this.success = success;
+ this.e = e;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateLog_result(replicateLog_result other) {
+ if (other.isSetSuccess()) {
+ this.success = org.apache.thrift.TBaseHelper.copyBinary(other.success);
+;
+ }
+ if (other.isSetE()) {
+ this.e = new RemoteReplicationException(other.e);
+ }
+ }
+
+ public replicateLog_result deepCopy() {
+ return new replicateLog_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ this.e = null;
+ }
+
+ public byte[] getSuccess() {
+ setSuccess(org.apache.thrift.TBaseHelper.rightSize(success));
+ return success == null ? null : success.array();
+ }
+
+ public ByteBuffer bufferForSuccess() {
+ return success;
+ }
+
+ public replicateLog_result setSuccess(byte[] success) {
+ setSuccess(success == null ? (ByteBuffer)null : ByteBuffer.wrap(success));
+ return this;
+ }
+
+ public replicateLog_result setSuccess(ByteBuffer success) {
+ this.success = success;
+ return this;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ public RemoteReplicationException getE() {
+ return this.e;
+ }
+
+ public replicateLog_result setE(RemoteReplicationException e) {
+ this.e = e;
+ return this;
+ }
+
+ public void unsetE() {
+ this.e = null;
+ }
+
+ /** Returns true if field e is set (has been assigned a value) and false otherwise */
+ public boolean isSetE() {
+ return this.e != null;
+ }
+
+ public void setEIsSet(boolean value) {
+ if (!value) {
+ this.e = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((ByteBuffer)value);
+ }
+ break;
+
+ case E:
+ if (value == null) {
+ unsetE();
+ } else {
+ setE((RemoteReplicationException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ case E:
+ return getE();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ case E:
+ return isSetE();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateLog_result)
+ return this.equals((replicateLog_result)that);
+ return false;
+ }
+
+ public boolean equals(replicateLog_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ boolean this_present_e = true && this.isSetE();
+ boolean that_present_e = true && that.isSetE();
+ if (this_present_e || that_present_e) {
+ if (!(this_present_e && that_present_e))
+ return false;
+ if (!this.e.equals(that.e))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateLog_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateLog_result typedOther = (replicateLog_result)other;
+
+ lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetE()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateLog_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.success, sb);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("e:");
+ if (this.e == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.e);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateLog_resultStandardSchemeFactory implements SchemeFactory {
+ public replicateLog_resultStandardScheme getScheme() {
+ return new replicateLog_resultStandardScheme();
+ }
+ }
+
+ private static class replicateLog_resultStandardScheme extends StandardScheme<replicateLog_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.success = iprot.readBinary();
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 1: // E
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ oprot.writeBinary(struct.success);
+ oprot.writeFieldEnd();
+ }
+ if (struct.e != null) {
+ oprot.writeFieldBegin(E_FIELD_DESC);
+ struct.e.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateLog_resultTupleSchemeFactory implements SchemeFactory {
+ public replicateLog_resultTupleScheme getScheme() {
+ return new replicateLog_resultTupleScheme();
+ }
+ }
+
+ private static class replicateLog_resultTupleScheme extends TupleScheme<replicateLog_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ if (struct.isSetE()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetSuccess()) {
+ oprot.writeBinary(struct.success);
+ }
+ if (struct.isSetE()) {
+ struct.e.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.success = iprot.readBinary();
+ struct.setSuccessIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class replicateKeyValues_args implements org.apache.thrift.TBase<replicateKeyValues_args, replicateKeyValues_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_args");
+
+ private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
+ private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateKeyValues_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateKeyValues_argsTupleSchemeFactory());
+ }
+
+ public int remoteTableId; // required
+ public KeyValues data; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ REMOTE_TABLE_ID((short)1, "remoteTableId"),
+ DATA((short)2, "data");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // REMOTE_TABLE_ID
+ return REMOTE_TABLE_ID;
+ case 2: // DATA
+ return DATA;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __REMOTETABLEID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeyValues.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_args.class, metaDataMap);
+ }
+
+ public replicateKeyValues_args() {
+ }
+
+ public replicateKeyValues_args(
+ int remoteTableId,
+ KeyValues data)
+ {
+ this();
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ this.data = data;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateKeyValues_args(replicateKeyValues_args other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.remoteTableId = other.remoteTableId;
+ if (other.isSetData()) {
+ this.data = new KeyValues(other.data);
+ }
+ }
+
+ public replicateKeyValues_args deepCopy() {
+ return new replicateKeyValues_args(this);
+ }
+
+ @Override
+ public void clear() {
+ setRemoteTableIdIsSet(false);
+ this.remoteTableId = 0;
+ this.data = null;
+ }
+
+ public int getRemoteTableId() {
+ return this.remoteTableId;
+ }
+
+ public replicateKeyValues_args setRemoteTableId(int remoteTableId) {
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ return this;
+ }
+
+ public void unsetRemoteTableId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
+ public boolean isSetRemoteTableId() {
+ return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ public void setRemoteTableIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
+ }
+
+ public KeyValues getData() {
+ return this.data;
+ }
+
+ public replicateKeyValues_args setData(KeyValues data) {
+ this.data = data;
+ return this;
+ }
+
+ public void unsetData() {
+ this.data = null;
+ }
+
+ /** Returns true if field data is set (has been assigned a value) and false otherwise */
+ public boolean isSetData() {
+ return this.data != null;
+ }
+
+ public void setDataIsSet(boolean value) {
+ if (!value) {
+ this.data = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ if (value == null) {
+ unsetRemoteTableId();
+ } else {
+ setRemoteTableId((Integer)value);
+ }
+ break;
+
+ case DATA:
+ if (value == null) {
+ unsetData();
+ } else {
+ setData((KeyValues)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return Integer.valueOf(getRemoteTableId());
+
+ case DATA:
+ return getData();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return isSetRemoteTableId();
+ case DATA:
+ return isSetData();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateKeyValues_args)
+ return this.equals((replicateKeyValues_args)that);
+ return false;
+ }
+
+ public boolean equals(replicateKeyValues_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_remoteTableId = true;
+ boolean that_present_remoteTableId = true;
+ if (this_present_remoteTableId || that_present_remoteTableId) {
+ if (!(this_present_remoteTableId && that_present_remoteTableId))
+ return false;
+ if (this.remoteTableId != that.remoteTableId)
+ return false;
+ }
+
+ boolean this_present_data = true && this.isSetData();
+ boolean that_present_data = true && that.isSetData();
+ if (this_present_data || that_present_data) {
+ if (!(this_present_data && that_present_data))
+ return false;
+ if (!this.data.equals(that.data))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateKeyValues_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateKeyValues_args typedOther = (replicateKeyValues_args)other;
+
+ lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRemoteTableId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetData()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateKeyValues_args(");
+ boolean first = true;
+
+ sb.append("remoteTableId:");
+ sb.append(this.remoteTableId);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("data:");
+ if (this.data == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.data);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (data != null) {
+ data.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateKeyValues_argsStandardSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_argsStandardScheme getScheme() {
+ return new replicateKeyValues_argsStandardScheme();
+ }
+ }
+
+ private static class replicateKeyValues_argsStandardScheme extends StandardScheme<replicateKeyValues_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // REMOTE_TABLE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // DATA
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.data = new KeyValues();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
+ oprot.writeI32(struct.remoteTableId);
+ oprot.writeFieldEnd();
+ if (struct.data != null) {
+ oprot.writeFieldBegin(DATA_FIELD_DESC);
+ struct.data.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateKeyValues_argsTupleSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_argsTupleScheme getScheme() {
+ return new replicateKeyValues_argsTupleScheme();
+ }
+ }
+
+ private static class replicateKeyValues_argsTupleScheme extends TupleScheme<replicateKeyValues_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRemoteTableId()) {
+ optionals.set(0);
+ }
+ if (struct.isSetData()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetRemoteTableId()) {
+ oprot.writeI32(struct.remoteTableId);
+ }
+ if (struct.isSetData()) {
+ struct.data.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.data = new KeyValues();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class replicateKeyValues_result implements org.apache.thrift.TBase<replicateKeyValues_result, replicateKeyValues_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateKeyValues_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateKeyValues_resultTupleSchemeFactory());
+ }
+
+ public ByteBuffer success; // required
+ public RemoteReplicationException e; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success"),
+ E((short)1, "e");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ case 1: // E
+ return E;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_result.class, metaDataMap);
+ }
+
+ public replicateKeyValues_result() {
+ }
+
+ public replicateKeyValues_result(
+ ByteBuffer success,
+ RemoteReplicationException e)
+ {
+ this();
+ this.success = success;
+ this.e = e;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateKeyValues_result(replicateKeyValues_result other) {
+ if (other.isSetSuccess()) {
+ this.success = org.apache.thrift.TBaseHelper.copyBinary(other.success);
+;
+ }
+ if (other.isSetE()) {
+ this.e = new RemoteReplicationException(other.e);
+ }
+ }
+
+ public replicateKeyValues_result deepCopy() {
+ return new replicateKeyValues_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ this.e = null;
+ }
+
+ public byte[] getSuccess() {
+ setSuccess(org.apache.thrift.TBaseHelper.rightSize(success));
+ return success == null ? null : success.array();
+ }
+
+ public ByteBuffer bufferForSuccess() {
+ return success;
+ }
+
+ public replicateKeyValues_result setSuccess(byte[] success) {
+ setSuccess(success == null ? (ByteBuffer)null : ByteBuffer.wrap(success));
+ return this;
+ }
+
+ public replicateKeyValues_result setSuccess(ByteBuffer success) {
+ this.success = success;
+ return this;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ public RemoteReplicationException getE() {
+ return this.e;
+ }
+
+ public replicateKeyValues_result setE(RemoteReplicationException e) {
+ this.e = e;
+ return this;
+ }
+
+ public void unsetE() {
+ this.e = null;
+ }
+
+ /** Returns true if field e is set (has been assigned a value) and false otherwise */
+ public boolean isSetE() {
+ return this.e != null;
+ }
+
+ public void setEIsSet(boolean value) {
+ if (!value) {
+ this.e = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((ByteBuffer)value);
+ }
+ break;
+
+ case E:
+ if (value == null) {
+ unsetE();
+ } else {
+ setE((RemoteReplicationException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ case E:
+ return getE();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ case E:
+ return isSetE();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateKeyValues_result)
+ return this.equals((replicateKeyValues_result)that);
+ return false;
+ }
+
+ public boolean equals(replicateKeyValues_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ boolean this_present_e = true && this.isSetE();
+ boolean that_present_e = true && that.isSetE();
+ if (this_present_e || that_present_e) {
+ if (!(this_present_e && that_present_e))
+ return false;
+ if (!this.e.equals(that.e))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateKeyValues_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateKeyValues_result typedOther = (replicateKeyValues_result)other;
+
+ lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetE()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateKeyValues_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ org.apache.thrift.TBaseHelper.toString(this.success, sb);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("e:");
+ if (this.e == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.e);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateKeyValues_resultStandardSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_resultStandardScheme getScheme() {
+ return new replicateKeyValues_resultStandardScheme();
+ }
+ }
+
+ private static class replicateKeyValues_resultStandardScheme extends StandardScheme<replicateKeyValues_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.success = iprot.readBinary();
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 1: // E
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ oprot.writeBinary(struct.success);
+ oprot.writeFieldEnd();
+ }
+ if (struct.e != null) {
+ oprot.writeFieldBegin(E_FIELD_DESC);
+ struct.e.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateKeyValues_resultTupleSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_resultTupleScheme getScheme() {
+ return new replicateKeyValues_resultTupleScheme();
+ }
+ }
+
+ private static class replicateKeyValues_resultTupleScheme extends TupleScheme<replicateKeyValues_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ if (struct.isSetE()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetSuccess()) {
+ oprot.writeBinary(struct.success);
+ }
+ if (struct.isSetE()) {
+ struct.e.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.success = iprot.readBinary();
+ struct.setSuccessIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/thrift/replication.thrift
----------------------------------------------------------------------
diff --git a/core/src/main/thrift/replication.thrift b/core/src/main/thrift/replication.thrift
index 481daa9..2994092 100644
--- a/core/src/main/thrift/replication.thrift
+++ b/core/src/main/thrift/replication.thrift
@@ -38,11 +38,11 @@ exception RemoteReplicationException {
2:string reason
}
-service RemoteReplicationCoordinator {
+service ReplicationCoordinator {
string getServicerAddress(1:i32 remoteTableId) throws (1:NoServersAvailableException e),
}
-service RemoteReplicationServicer {
- void replicateLog(1:i32 remoteTableId, 2:WalEdits data) throws (1:RemoteReplicationException e),
- void replicateKeyValues(1:i32 remoteTableId, 2:KeyValues data) throws (1:RemoteReplicationException e)
+service ReplicationServicer {
+ binary replicateLog(1:i32 remoteTableId, 2:WalEdits data) throws (1:RemoteReplicationException e),
+ binary replicateKeyValues(1:i32 remoteTableId, 2:KeyValues data) throws (1:RemoteReplicationException e)
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java b/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java
new file mode 100644
index 0000000..ca66b93
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/replication/AccumuloReplicaSystem.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.replication;
+
+import java.nio.ByteBuffer;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.ReplicationClient;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client;
+import org.apache.accumulo.core.util.ByteBufferUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.hadoop.fs.Path;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ *
+ */
+public class AccumuloReplicaSystem implements ReplicaSystem {
+ private static final Logger log = LoggerFactory.getLogger(AccumuloReplicaSystem.class);
+
+ private String instanceName, zookeepers;
+
+ @Override
+ public void configure(String configuration) {
+ Preconditions.checkNotNull(configuration);
+
+ int index = configuration.indexOf(',');
+ if (-1 == index) {
+ throw new IllegalArgumentException("Expected comma in configuration string");
+ }
+
+ instanceName = configuration.substring(0, index);
+ zookeepers = configuration.substring(index + 1);
+ }
+
+ @Override
+ public Status replicate(Path p, Status status, ReplicationTarget target) {
+ Instance peerInstance = getPeerInstance(target);
+ // Remote identifier is an integer (table id) in this case.
+ final int remoteTableId = Integer.parseInt(target.getRemoteIdentifier());
+
+ for (int i = 0; i < 10; i++) {
+ String peerTserver;
+ try {
+ // Ask the master on the remote what TServer we should talk with to replicate the data
+ peerTserver = ReplicationClient.executeCoordinatorWithReturn(peerInstance, new ClientExecReturn<String,ReplicationCoordinator.Client>() {
+
+ @Override
+ public String execute(ReplicationCoordinator.Client client) throws Exception {
+ return client.getServicerAddress(remoteTableId);
+ }
+
+ });
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ // No progress is made
+ log.error("Could not connect to master at {}, cannot proceed with replication. Will retry", target, e);
+ continue;
+ }
+
+ if (null == peerTserver) {
+ // Something went wrong, and we didn't get a valid tserver from the remote for some reason
+ log.warn("Did not receive tserver from master at {}, cannot proceed with replication. Will retry.", target);
+ continue;
+ }
+
+ // We have a tserver on the remote -- send the data its way.
+ ByteBuffer result;
+ try {
+ result = ReplicationClient.executeServicerWithReturn(peerInstance, peerTserver, new ClientExecReturn<ByteBuffer,ReplicationServicer.Client>() {
+ @Override
+ public ByteBuffer execute(Client client) throws Exception {
+ return client.replicateLog(remoteTableId, null);
+ }
+ });
+
+ // We need to be able to parse the returned Status,
+ // if we can't, we don't know what the server actually parsed.
+ try {
+ return Status.parseFrom(ByteBufferUtil.toBytes(result));
+ } catch (InvalidProtocolBufferException e) {
+ log.error("Could not parse return Status from {}", peerTserver, e);
+ throw new RuntimeException("Could not parse returned Status from " + peerTserver, e);
+ }
+ } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
+ log.warn("Could not connect to remote server {}, will retry", peerTserver, e);
+ UtilWaitThread.sleep(250);
+ }
+ }
+
+ // We made no status, punt on it for now, and let it re-queue itself for work
+ return status;
+ }
+
+ public Instance getPeerInstance(ReplicationTarget target) {
+ return new ZooKeeperInstance(instanceName, zookeepers);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java b/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java
index 4f5bba7..d877a63 100644
--- a/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java
+++ b/core/src/test/java/org/apache/accumulo/core/replication/ReplicationTargetTest.java
@@ -81,7 +81,18 @@ public class ReplicationTargetTest {
Text t = new Text();
t.set(buffer.getData(), 0, buffer.getLength());
- Assert.assertEquals(t, ReplicationTarget.toText(expected));
+ Assert.assertEquals(t, expected.toText());
+ }
+
+ @Test
+ public void staticFromStringHelper() throws Exception {
+ ReplicationTarget expected = new ReplicationTarget("foo", "bar");
+ DataOutputBuffer buffer = new DataOutputBuffer();
+ expected.write(buffer);
+ Text t = new Text();
+ t.set(buffer.getData(), 0, buffer.getLength());
+
+ Assert.assertEquals(expected, ReplicationTarget.from(t.toString()));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
new file mode 100644
index 0000000..dd70134
--- /dev/null
+++ b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.replication;
+
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Maps;
+
+/**
+ *
+ */
+public class ReplicationWorkAssignerHelper {
+ public static final String KEY_SEPARATOR = "|";
+
+ /**
+ * Serialize a filename and a {@link ReplicationTarget} into the expected
+ * key format for use with the {@link DistributedWorkQueue}
+ * @param filename Filename for data to be replicated
+ * @param replTarget Information about replication peer
+ * @return Key for identifying work in queue
+ */
+ public static String getQueueKey(String filename, ReplicationTarget replTarget) {
+ return getQueueKey(filename, replTarget.toText().toString());
+ }
+
+ /**
+ * Serialize a filename and a {@link ReplicationTarget} into the expected
+ * key format for use with the {@link DistributedWorkQueue}
+ * @param filename Filename for data to be replicated
+ * @param replTargetStr Serialized {@link ReplicationTarget}
+ * @return Key for identifying work in queue
+ */
+ public static String getQueueKey(String filename, String replTargetStr) {
+ return filename + KEY_SEPARATOR + replTargetStr;
+ }
+
+ /**
+ * @param queueKey Key from the work queue
+ * @return Components which created the queue key
+ */
+ public static Entry<String,ReplicationTarget> fromQueueKey(String queueKey) {
+ Preconditions.checkNotNull(queueKey);
+
+ int index = queueKey.indexOf(KEY_SEPARATOR);
+ if (-1 == index) {
+ throw new IllegalArgumentException("Could not find expected separator in queue key '" + queueKey + "'");
+ }
+
+ return Maps.immutableEntry(queueKey.substring(0, index), ReplicationTarget.from(queueKey.substring(index + 1)));
+ }
+}
[38/50] [abbrv] ACCUMULO-1327 converted latex manual to asciidoc
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/chapters/troubleshooting.txt
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/chapters/troubleshooting.txt b/docs/src/main/asciidoc/chapters/troubleshooting.txt
new file mode 100644
index 0000000..eec75fa
--- /dev/null
+++ b/docs/src/main/asciidoc/chapters/troubleshooting.txt
@@ -0,0 +1,758 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+## Troubleshooting
+
+### Logs
+
+*Q*: The tablet server does not seem to be running!? What happened?
+
+Accumulo is a distributed system. It is supposed to run on remote
+equipment, across hundreds of computers. Each program that runs on
+these remote computers writes down events as they occur, into a local
+file. By default, this is defined in
++$ACCUMULO_HOME/conf/accumule-env.sh+ as +ACCUMULO_LOG_DIR+.
+
+*A*: Look in the +$ACCUMULO_LOG_DIR/tserver*.log+ file. Specifically, check the end of the file.
+
+*Q*: The tablet server did not start and the debug log does not exists! What happened?
+
+When the individual programs are started, the stdout and stderr output
+of these programs are stored in +.out+ and +.err+ files in
++$ACCUMULO_LOG_DIR+. Often, when there are missing configuration
+options, files or permissions, messages will be left in these files.
+
+*A*: Probably a start-up problem. Look in +$ACCUMULO_LOG_DIR/tserver*.err+
+
+### Monitor
+
+*Q*: Accumulo is not working, what's wrong?
+
+There's a small web server that collects information about all the
+components that make up a running Accumulo instance. It will highlight
+unusual or unexpected conditions.
+
+*A*: Point your browser to the monitor (typically the master host, on port 50095). Is anything red or yellow?
+
+*Q*: My browser is reporting connection refused, and I cannot get to the monitor
+
+The monitor program's output is also written to .err and .out files in
+the +$ACCUMULO_LOG_DIR+. Look for problems in this file if the
++$ACCUMULO_LOG_DIR/monitor*.log+ file does not exist.
+
+*A*: The monitor program is probably not running. Check the log files for errors.
+
+*Q*: My browser hangs trying to talk to the monitor.
+
+Your browser needs to be able to reach the monitor program. Often
+large clusters are firewalled, or use a VPN for internal
+communications. You can use SSH to proxy your browser to the cluster,
+or consult with your system administrator to gain access to the server
+from your browser.
+
+It is sometimes helpful to use a text-only browser to sanity-check the
+monitor while on the machine running the monitor:
+
+ $ links http://localhost:50095
+
+*A*: Verify that you are not firewalled from the monitor if it is running on a remote host.
+
+*Q*: The monitor responds, but there are no numbers for tservers and tables. The summary page says the master is down.
+
+The monitor program gathers all the details about the master and the
+tablet servers through the master. It will be mostly blank if the
+master is down.
+
+*A*: Check for a running master.
+
+### HDFS
+
+Accumulo reads and writes to the Hadoop Distributed File System.
+Accumulo needs this file system available at all times for normal operations.
+
+*Q*: Accumulo is having problems ``getting a block blk_1234567890123.'' How do I fix it?
+
+This troubleshooting guide does not cover HDFS, but in general, you
+want to make sure that all the datanodes are running and an fsck check
+finds the file system clean:
+
+ $ hadoop fsck /accumulo
+
+You can use:
+
+ $ hadoop fsck /accumulo/path/to/corrupt/file -locations -blocks -files
+
+to locate the block references of individual corrupt files and use those
+references to search the name node and individual data node logs to determine which
+servers those blocks have been assigned and then try to fix any underlying file
+system issues on those nodes.
+
+On a larger cluster, you may need to increase the number of Xcievers for HDFS DataNodes:
+
+[source,xml]
+<property>
+ <name>dfs.datanode.max.xcievers</name>
+ <value>4096</value>
+</property>
+
+*A*: Verify HDFS is healthy, check the datanode logs.
+
+### Zookeeper
+
+*Q*: +accumulo init+ is hanging. It says something about talking to zookeeper.
+
+Zookeeper is also a distributed service. You will need to ensure that
+it is up. You can run the zookeeper command line tool to connect to
+any one of the zookeeper servers:
+
+ $ zkCli.sh -server zoohost
+ ...
+ [zk: zoohost:2181(CONNECTED) 0]
+
+It is important to see the word +CONNECTED+! If you only see
++CONNECTING+ you will need to diagnose zookeeper errors.
+
+*A*: Check to make sure that zookeeper is up, and that
++$ACCUMULO_HOME/conf/accumulo-site.xml+ has been pointed to
+your zookeeper server(s).
+
+*Q*: Zookeeper is running, but it does not say +CONNECTED+
+
+Zookeeper processes talk to each other to elect a leader. All updates
+go through the leader and propagate to a majority of all the other
+nodes. If a majority of the nodes cannot be reached, zookeeper will
+not allow updates. Zookeeper also limits the number connections to a
+server from any other single host. By default, this limit can be as small as 10
+and can be reached in some everything-on-one-machine test configurations.
+
+You can check the election status and connection status of clients by
+asking the zookeeper nodes for their status. You connect to zookeeper
+and ask it with the four-letter +stat+ command:
+
+----
+$ nc zoohost 2181
+stat
+Zookeeper version: 3.4.5-1392090, built on 09/30/2012 17:52 GMT
+Clients:
+ /127.0.0.1:58289[0](queued=0,recved=1,sent=0)
+ /127.0.0.1:60231[1](queued=0,recved=53910,sent=53915)
+
+Latency min/avg/max: 0/5/3008
+Received: 1561459
+Sent: 1561592
+Connections: 2
+Outstanding: 0
+Zxid: 0x621a3b
+Mode: standalone
+Node count: 22524
+----
+
+*A*: Check zookeeper status, verify that it has a quorum, and has not exceeded maxClientCnxns.
+
+*Q*: My tablet server crashed! The logs say that it lost its zookeeper lock.
+
+Tablet servers reserve a lock in zookeeper to maintain their ownership
+over the tablets that have been assigned to them. Part of their
+responsibility for keeping the lock is to send zookeeper a keep-alive
+message periodically. If the tablet server fails to send a message in
+a timely fashion, zookeeper will remove the lock and notify the tablet
+server. If the tablet server does not receive a message from
+zookeeper, it will assume its lock has been lost, too. If a tablet
+server loses its lock, it kills itself: everything assumes it is dead
+already.
+
+*A*: Investigate why the tablet server did not send a timely message to
+zookeeper.
+
+#### Keeping the tablet server lock
+
+*Q*: My tablet server lost its lock. Why?
+
+The primary reason a tablet server loses its lock is that it has been pushed into swap.
+
+A large java program (like the tablet server) may have a large portion
+of its memory image unused. The operation system will favor pushing
+this allocated, but unused memory into swap so that the memory can be
+re-used as a disk buffer. When the java virtual machine decides to
+access this memory, the OS will begin flushing disk buffers to return that
+memory to the VM. This can cause the entire process to block long
+enough for the zookeeper lock to be lost.
+
+*A*: Configure your system to reduce the kernel parameter _swappiness_ from the default (60) to zero.
+
+*Q*: My tablet server lost its lock, and I have already set swappiness to
+zero. Why?
+
+Be careful not to over-subscribe memory. This can be easy to do if
+your accumulo processes run on the same nodes as hadoop's map-reduce
+framework. Remember to add up:
+
+* size of the JVM for the tablet server
+* size of the in-memory map, if using the native map implementation
+* size of the JVM for the data node
+* size of the JVM for the task tracker
+* size of the JVM times the maximum number of mappers and reducers
+* size of the kernel and any support processes
+
+If a 16G node can run 2 mappers and 2 reducers, and each can be 2G,
+then there is only 8G for the data node, tserver, task tracker and OS.
+
+*A*: Reduce the memory footprint of each component until it fits comfortably.
+
+*Q*: My tablet server lost its lock, swappiness is zero, and my node has lots of unused memory!
+
+The JVM memory garbage collector may fall behind and cause a
+"stop-the-world" garbage collection. On a large memory virtual
+machine, this collection can take a long time. This happens more
+frequently when the JVM is getting low on free memory. Check the logs
+of the tablet server. You will see lines like this:
+
+ 2013-06-20 13:43:20,607 [tabletserver.TabletServer] DEBUG: gc ParNew=0.00(+0.00) secs
+ ConcurrentMarkSweep=0.00(+0.00) secs freemem=1,868,325,952(+1,868,325,952) totalmem=2,040,135,680
+
+When +freemem+ becomes small relative to the amount of memory
+needed, the JVM will spend more time finding free memory than
+performing work. This can cause long delays in sending keep-alive
+messages to zookeeper.
+
+*A*: Ensure the tablet server JVM is not running low on memory.
+
+### Tools
+
+The accumulo script can be used to run classes from the command line.
+This section shows how a few of the utilities work, but there are many
+more.
+
+There's a class that will examine an accumulo storage file and print
+out basic metadata.
+
+----
+$ ./bin/accumulo org.apache.accumulo.core.file.rfile.PrintInfo /accumulo/tables/1/default_tablet/A000000n.rf
+2013-07-16 08:17:14,778 [util.NativeCodeLoader] INFO : Loaded the native-hadoop library
+Locality group : <DEFAULT>
+ Start block : 0
+ Num blocks : 1
+ Index level 0 : 62 bytes 1 blocks
+ First key : 288be9ab4052fe9e span:34078a86a723e5d3:3da450f02108ced5 [] 1373373521623 false
+ Last key : start:13fc375709e id:615f5ee2dd822d7a [] 1373373821660 false
+ Num entries : 466
+ Column families : [waitForCommits, start, md major compactor 1, md major compactor 2, md major compactor 3,
+ bringOnline, prep, md major compactor 4, md major compactor 5, md root major compactor 3,
+ minorCompaction, wal, compactFiles, md root major compactor 4, md root major compactor 1,
+ md root major compactor 2, compact, id, client:update, span, update, commit, write,
+ majorCompaction]
+
+Meta block : BCFile.index
+ Raw size : 4 bytes
+ Compressed size : 12 bytes
+ Compression type : gz
+
+Meta block : RFile.index
+ Raw size : 780 bytes
+ Compressed size : 344 bytes
+ Compression type : gz
+----
+
+When trying to diagnose problems related to key size, the +PrintInfo+ tool can provide a histogram of the individual key sizes:
+
+ $ ./bin/accumulo org.apache.accumulo.core.file.rfile.PrintInfo --histogram /accumulo/tables/1/default_tablet/A000000n.rf
+ ...
+ Up to size count %-age
+ 10 : 222 28.23%
+ 100 : 244 71.77%
+ 1000 : 0 0.00%
+ 10000 : 0 0.00%
+ 100000 : 0 0.00%
+ 1000000 : 0 0.00%
+ 10000000 : 0 0.00%
+ 100000000 : 0 0.00%
+ 1000000000 : 0 0.00%
+ 10000000000 : 0 0.00%
+
+Likewise, +PrintInfo+ will dump the key-value pairs and show you the contents of the RFile:
+
+ $ ./bin/accumulo org.apache.accumulo.core.file.rfile.PrintInfo --dump /accumulo/tables/1/default_tablet/A000000n.rf
+ row columnFamily:columnQualifier [visibility] timestamp deleteFlag -> Value
+ ...
+
+*Q*: Accumulo is not showing me any data!
+
+*A*: Do you have your auths set so that it matches your visibilities?
+
+*Q*: What are my visibilities?
+
+*A*: Use +PrintInfo+ on a representative file to get some idea of the visibilities in the underlying data.
+
+Note that the use of +PrintInfo+ is an administrative tool and can only
+by used by someone who can access the underlying Accumulo data. It
+does not provide the normal access controls in Accumulo.
+
+If you would like to backup, or otherwise examine the contents of Zookeeper, there are commands to dump and load to/from XML.
+
+ $ ./bin/accumulo org.apache.accumulo.server.util.DumpZookeeper --root /accumulo >dump.xml
+ $ ./bin/accumulo org.apache.accumulo.server.util.RestoreZookeeper --overwrite < dump.xml
+
+*Q*: How can I get the information in the monitor page for my cluster monitoring system?
+
+*A*: Use GetMasterStats:
+
+ $ ./bin/accumulo org.apache.accumulo.test.GetMasterStats | grep Load
+ OS Load Average: 0.27
+
+*Q*: The monitor page is showing an offline tablet. How can I find out which tablet it is?
+
+*A*: Use FindOfflineTablets:
+
+ $ ./bin/accumulo org.apache.accumulo.server.util.FindOfflineTablets
+ 2<<@(null,null,localhost:9997) is UNASSIGNED #walogs:2
+
+Here's what the output means:
+
++2<<+::
+ This is the tablet from (-inf, pass:[+]inf) for the
+ table with id 2. The command +tables -l+ in the shell will show table ids for
+ tables.
+
++@(null, null, localhost:9997)+::
+ Location information. The
+ format is +@(assigned, hosted, last)+. In this case, the
+ tablet has not been assigned, is not hosted anywhere, and was once
+ hosted on localhost.
+
++#walogs:2+::
+ The number of write-ahead logs that this tablet requires for recovery.
+
+An unassigned tablet with write-ahead logs is probably waiting for
+logs to be sorted for efficient recovery.
+
+*Q*: How can I be sure that the metadata tables are up and consistent?
+
+*A*: +CheckForMetadataProblems+ will verify the start/end of
+every tablet matches, and the start and stop for the table is empty:
+
+ $ ./bin/accumulo org.apache.accumulo.server.util.CheckForMetadataProblems -u root --password
+ Enter the connection password:
+ All is well for table !0
+ All is well for table 1
+
+*Q*: My hadoop cluster has lost a file due to a NameNode failure. How can I remove the file?
+
+*A*: There's a utility that will check every file reference and ensure
+that the file exists in HDFS. Optionally, it will remove the
+reference:
+
+ $ ./bin/accumulo org.apache.accumulo.server.util.RemoveEntriesForMissingFiles -u root --password
+ Enter the connection password:
+ 2013-07-16 13:10:57,293 [util.RemoveEntriesForMissingFiles] INFO : File /accumulo/tables/2/default_tablet/F0000005.rf
+ is missing
+ 2013-07-16 13:10:57,296 [util.RemoveEntriesForMissingFiles] INFO : 1 files of 3 missing
+
+*Q*: I have many entries in zookeeper for old instances I no longer need. How can I remove them?
+
+*A*: Use CleanZookeeper:
+
+ $ ./bin/accumulo org.apache.accumulo.server.util.CleanZookeeper
+
+This command will not delete the instance pointed to by the local +conf/accumulo-site.xml+ file.
+
+*Q*: I need to decommission a node. How do I stop the tablet server on it?
+
+*A*: Use the admin command:
+
+ $ ./bin/accumulo admin stop hostname:9997
+ 2013-07-16 13:15:38,403 [util.Admin] INFO : Stopping server 12.34.56.78:9997
+
+*Q*: I cannot login to a tablet server host, and the tablet server will not shut down. How can I kill the server?
+
+*A*: Sometimes you can kill a "stuck" tablet server by deleting its lock in zookeeper:
+
+ $ ./bin/accumulo org.apache.accumulo.server.util.TabletServerLocks --list
+ 127.0.0.1:9997 TSERV_CLIENT=127.0.0.1:9997
+ $ ./bin/accumulo org.apache.accumulo.server.util.TabletServerLocks -delete 127.0.0.1:9997
+ $ ./bin/accumulo org.apache.accumulo.server.util.TabletServerLocks -list
+ 127.0.0.1:9997 null
+
+You can find the master and instance id for any accumulo instances using the same zookeeper instance:
+
+----
+$ ./bin/accumulo org.apache.accumulo.server.util.ListInstances
+INFO : Using ZooKeepers localhost:2181
+
+ Instance Name | Instance ID | Master
+---------------------+--------------------------------------+-------------------------------
+ "test" | 6140b72e-edd8-4126-b2f5-e74a8bbe323b | 127.0.0.1:9999
+----
+
+[[metadata]]
+### System Metadata Tables
+
+Accumulo tracks information about tables in metadata tables. The metadata for
+most tables is contained within the metadata table in the accumulo namespace,
+while metadata for that table is contained in the root table in the accumulo
+namespace. The root table is composed of a single tablet, which does not
+split, so it is also called the root tablet. Information about the root
+table, such as its location and write-ahead logs, are stored in ZooKeeper.
+
+Let's create a table and put some data into it:
+
+----
+shell> createtable test
+
+shell> tables -l
+accumulo.metadata => !0
+accumulo.root => +r
+test => 2
+trace => 1
+
+shell> insert a b c d
+
+shell> flush -w
+----
+
+Now let's take a look at the metadata for this table:
+
+ shell> table accumulo.metadata
+ shell> scan -b 3; -e 3<
+ 3< file:/default_tablet/F000009y.rf [] 186,1
+ 3< last:13fe86cd27101e5 [] 127.0.0.1:9997
+ 3< loc:13fe86cd27101e5 [] 127.0.0.1:9997
+ 3< log:127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995 [] 127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995|6
+ 3< srv:dir [] /default_tablet
+ 3< srv:flush [] 1
+ 3< srv:lock [] tservers/127.0.0.1:9997/zlock-0000000001$13fe86cd27101e5
+ 3< srv:time [] M1373998392323
+ 3< ~tab:~pr [] \x00
+
+Let's decode this little session:
+
++scan -b 3; -e 3<+:: Every tablet gets its own row. Every row starts with the table id followed by
+ +;+ or +<+, and followed by the end row split point for that tablet.
+
++file:/default_tablet/F000009y.rf [] 186,1+::
+ File entry for this tablet. This tablet contains a single file reference. The
+ file is +/accumulo/tables/3/default_tablet/F000009y.rf+. It contains 1
+ key/value pair, and is 186 bytes long.
+
++last:13fe86cd27101e5 [] 127.0.0.1:9997+::
+ Last location for this tablet. It was last held on 127.0.0.1:9997, and the
+ unique tablet server lock data was +13fe86cd27101e5+. The default balancer
+ will tend to put tablets back on their last location.
+
++loc:13fe86cd27101e5 [] 127.0.0.1:9997+::
+ The current location of this tablet.
+
++log:127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995 [] 127.0. ...+::
+ This tablet has a reference to a single write-ahead log. This file can be found in
+ +/accumulo/wal/127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995+. The value
+ of this entry could refer to multiple files. This tablet's data is encoded as
+ +6+ within the log.
+
++srv:dir [] /default_tablet+::
+ Files written for this tablet will be placed into
+ +/accumulo/tables/3/default_tablet+.
+
++srv:flush [] 1+::
+ Flush id. This table has successfully completed the flush with the id of +1+.
+
++srv:lock [] tservers/127.0.0.1:9997/zlock-0000000001\$13fe86cd27101e5+::
+ This is the lock information for the tablet holding the present lock. This
+ information is checked against zookeeper whenever this is updated, which
+ prevents a metadata update from a tablet server that no longer holds its
+ lock.
+
++srv:time [] M1373998392323+::
+ This indicates the time time type (+M+ for milliseconds or +L+ for logical) and the timestamp of the most recently written key in this tablet. It is used to ensure automatically assigned key timestamps are strictly increasing for the tablet, regardless of the tablet server's system time.
+
+`~tab:~pr [] \x00`::
+ The end-row marker for the previous tablet (prev-row). The first byte
+ indicates the presence of a prev-row. This tablet has the range (-inf, +inf),
+ so it has no prev-row (or end row).
+
+Besides these columns, you may see:
+
++rowId future:zooKeeperID location+::
+ Tablet has been assigned to a tablet, but not yet loaded.
+
++~del:filename+::
+ When a tablet server is done use a file, it will create a delete marker in the appropriate metadata table, unassociated with any tablet. The garbage collector will remove the marker, and the file, when no other reference to the file exists.
+
++~blip:txid+::
+ Bulk-Load In Progress marker.
+
++rowId loaded:filename+::
+ A file has been bulk-loaded into this tablet, however the bulk load has not yet completed on other tablets, so this marker prevents the file from being loaded multiple times.
+
++rowId !cloned+::
+ A marker that indicates that this tablet has been successfully cloned.
+
++rowId splitRatio:ratio+::
+ A marker that indicates a split is in progress, and the files are being split at the given ratio.
+
++rowId chopped+::
+ A marker that indicates that the files in the tablet do not contain keys outside the range of the tablet.
+
++rowId scan+::
+ A marker that prevents a file from being removed while there are still active scans using it.
+
+### Simple System Recovery
+
+*Q*: One of my Accumulo processes died. How do I bring it back?
+
+The easiest way to bring all services online for an Accumulo instance is to run the +start-all.sh+ script.
+
+ $ bin/start-all.sh
+
+This process will check the process listing, using +jps+ on each host before attempting to restart a service on the given host.
+Typically, this check is sufficient except in the face of a hung/zombie process. For large clusters, it may be
+undesirable to ssh to every node in the cluster to ensure that all hosts are running the appropriate processes and +start-here.sh+ may be of use.
+
+ $ ssh host_with_dead_process
+ $ bin/start-here.sh
+
++start-here.sh+ should be invoked on the host which is missing a given process. Like start-all.sh, it will start all
+necessary processes that are not currently running, but only on the current host and not cluster-wide. Tools such as +pssh+ or
++pdsh+ can be used to automate this process.
+
++start-server.sh+ can also be used to start a process on a given host; however, it is not generally recommended for
+users to issue this directly as the +start-all.sh+ and +start-here.sh+ scripts provide the same functionality with
+more automation and are less prone to user error.
+
+*A*: Use +start-all.sh+ or +start-here.sh+.
+
+*Q*: My process died again. Should I restart it via +cron+ or tools like +supervisord+?
+
+*A*: A repeatedly dying Accumulo process is a sign of a larger problem. Typically these problems are due to a
+misconfiguration of Accumulo or over-saturation of resources. Blind automation of any service restart inside of Accumulo
+is generally an undesirable situation as it is indicative of a problem that is being masked and ignored. Accumulo
+processes should be stable on the order of months and not require frequent restart.
+
+
+### Advanced System Recovery
+
+#### HDFS Failure
+*Q*: I had disasterous HDFS failure. After bringing everything back up, several tablets refuse to go online.
+
+Data written to tablets is written into memory before being written into indexed files. In case the server
+is lost before the data is saved into a an indexed file, all data stored in memory is first written into a
+write-ahead log (WAL). When a tablet is re-assigned to a new tablet server, the write-ahead logs are read to
+recover any mutations that were in memory when the tablet was last hosted.
+
+If a write-ahead log cannot be read, then the tablet is not re-assigned. All it takes is for one of
+the blocks in the write-ahead log to be missing. This is unlikely unless multiple data nodes in HDFS have been
+lost.
+
+*A*: Get the WAL files online and healthy. Restore any data nodes that may be down.
+
+*Q*: How do find out which tablets are offline?
+
+*A*: Use +accumulo admin checkTablets+
+
+ $ bin/accumulo admin checkTablets
+
+*Q*: I lost three data nodes, and I'm missing blocks in a WAL. I don't care about data loss, how
+can I get those tablets online?
+
+See the discussion in <<metadata>>, which shows a typical metadata table listing.
+The entries with a column family of +log+ are references to the WAL for that tablet.
+If you know what WAL is bad, you can find all the references with a grep in the shell:
+
+ shell> grep 0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995
+ 3< log:127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995 [] 127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995|6
+
+*A*: You can remove the WAL references in the metadata table.
+
+ shell> grant -u root Table.WRITE -t accumulo.metadata
+ shell> delete 3< log 127.0.0.1+9997/0cb7ce52-ac46-4bf7-ae1d-acdcfaa97995
+
+Note: the colon (+:+) is omitted when specifying the _row cf cq_ for the delete command.
+
+The master will automatically discover the tablet no longer has a bad WAL reference and will
+assign the tablet. You will need to remove the reference from all the tablets to get them
+online.
+
+
+*Q*: The metadata (or root) table has references to a corrupt WAL.
+
+This is a much more serious state, since losing updates to the metadata table will result
+in references to old files which may not exist, or lost references to new files, resulting
+in tablets that cannot be read, or large amounts of data loss.
+
+The best hope is to restore the WAL by fixing HDFS data nodes and bringing the data back online.
+If this is not possible, the best approach is to re-create the instance and bulk import all files from
+the old instance into a new tables.
+
+A complete set of instructions for doing this is outside the scope of this guide,
+but the basic approach is:
+
+* Use +tables -l+ in the shell to discover the table name to table id mapping
+* Stop all accumulo processes on all nodes
+* Move the accumulo directory in HDFS out of the way:
+ $ hadoop fs -mv /accumulo /corrupt
+* Re-initalize accumulo
+* Recreate tables, users and permissions
+* Import the directories under +/corrupt/tables/<id>+ into the new instance
+
+*Q*: One or more HDFS Files under /accumulo/tables are corrupt
+
+Accumulo maintains multiple references into the tablet files in the metadata
+tables and within the tablet server hosting the file, this makes it difficult to
+reliably just remove those references.
+
+The directory structure in HDFS for tables will follow the general structure:
+
+ /accumulo
+ /accumulo/tables/
+ /accumulo/tables/!0
+ /accumulo/tables/!0/default_tablet/A000001.rf
+ /accumulo/tables/!0/t-00001/A000002.rf
+ /accumulo/tables/1
+ /accumulo/tables/1/default_tablet/A000003.rf
+ /accumulo/tables/1/t-00001/A000004.rf
+ /accumulo/tables/1/t-00001/A000005.rf
+ /accumulo/tables/2/default_tablet/A000006.rf
+ /accumulo/tables/2/t-00001/A000007.rf
+
+If files under +/accumulo/tables+ are corrupt, the best course of action is to
+recover those files in hdsf see the section on HDFS. Once these recovery efforts
+have been exhausted, the next step depends on where the missing file(s) are
+located. Different actions are required when the bad files are in Accumulo data
+table files or if they are metadata table files.
+
+*Data File Corruption*
+
+When an Accumulo data file is corrupt, the most reliable way to restore Accumulo
+operations is to replace the missing file with an ``empty'' file so that
+references to the file in the METADATA table and within the tablet server
+hosting the file can be resolved by Accumulo. An empty file can be created using
+the CreateEmpty utiity:
+
+ $ accumulo org.apache.accumulo.core.file.rfile.CreateEmpty /path/to/empty/file/empty.rf
+
+The process is to delete the corrupt file and then move the empty file into its
+place (The generated empty file can be copied and used multiple times if necessary and does not need
+to be regenerated each time)
+
+ $ hadoop fs –rm /accumulo/tables/corrupt/file/thename.rf; \
+ hadoop fs -mv /path/to/empty/file/empty.rf /accumulo/tables/corrupt/file/thename.rf
+
+*Metadata File Corruption*
+
+If the corrupt files are metadata files, see <<metadata>> (under the path
++/accumulo/tables/!0+) then you will need to rebuild
+the metadata table by initializing a new instance of Accumulo and then importing
+all of the existing data into the new instance. This is the same procedure as
+recovering from a zookeeper failure (see <<zookeeper_failure>>), except that
+you will have the benefit of having the existing user and table authorizations
+that are maintained in zookeeper.
+
+You can use the DumpZookeeper utility to save this information for reference
+before creating the new instance. You will not be able to use RestoreZookeeper
+because the table names and references are likely to be different between the
+original and the new instances, but it can serve as a reference.
+
+*A*: If the files cannot be recovered, replace corrupt data files with a empty
+rfiles to allow references in the metadata table and in the tablet servers to be
+resolved. Rebuild the metadata table if the corrupt files are metadata files.
+
+[[zookeeper_failure]]
+#### ZooKeeper Failure
+*Q*: I lost my ZooKeeper quorum (hardware failure), but HDFS is still intact. How can I recover my Accumulo instance?
+
+ZooKeeper, in addition to its lock-service capabilities, also serves to bootstrap an Accumulo
+instance from some location in HDFS. It contains the pointers to the root tablet in HDFS which
+is then used to load the Accumulo metadata tablets, which then loads all user tables. ZooKeeper
+also stores all namespace and table configuration, the user database, the mapping of table IDs to
+table names, and more across Accumulo restarts.
+
+Presently, the only way to recover such an instance is to initialize a new instance and import all
+of the old data into the new instance. The easiest way to tackle this problem is to first recreate
+the mapping of table ID to table name and then recreate each of those tables in the new instance.
+Set any necessary configuration on the new tables and add some split points to the tables to close
+the gap between how many splits the old table had and no splits.
+
+The directory structure in HDFS for tables will follow the general structure:
+
+ /accumulo
+ /accumulo/tables/
+ /accumulo/tables/1
+ /accumulo/tables/1/default_tablet/A000001.rf
+ /accumulo/tables/1/t-00001/A000002.rf
+ /accumulo/tables/1/t-00001/A000003.rf
+ /accumulo/tables/2/default_tablet/A000004.rf
+ /accumulo/tables/2/t-00001/A000005.rf
+
+For each table, make a new directory that you can move (or copy if you have the HDFS space to do so)
+all of the rfiles for a given table into. For example, to process the table with an ID of +1+, make a new directory,
+say +/new-table-1+ and then copy all files from +/accumulo/tables/1/*/*.rf+ into that directory. Additionally,
+make a directory, +/new-table-1-failures+, for any failures during the import process. Then, issue the import
+command using the Accumulo shell into the new table, telling Accumulo to not re-set the timestamp:
+
+ user@instance new_table> importdirectory /new-table-1 /new-table-1-failures false
+
+Any RFiles which were failed to be loaded will be placed in +/new-table-1-failures+. Rfiles that were successfully
+imported will no longer exist in +/new-table-1+. For failures, move them back to the import directory and retry
+the +importdirectory+ command.
+
+It is *extremely* important to note that this approach may introduce stale data back into
+the tables. For a few reasons, RFiles may exist in the table directory which are candidates for deletion but have
+not yet been deleted. Additionally, deleted data which was not compacted away, but still exists in write-ahead logs if
+the original instance was somehow recoverable, will be re-introduced in the new instance. Table splits and merges
+(which also include the deleteRows API call on TableOperations, are also vulnerable to this problem. This process should
+*not* be used if these are unacceptable risks. It is possible to try to re-create a view of the +accumulo.metadata+
+table to prune out files that are candidates for deletion, but this is a difficult task that also may not be entirely accurate.
+
+Likewise, it is also possible that data loss may occur from write-ahead log (WAL) files which existed on the old table but
+were not minor-compacted into an RFile. Again, it may be possible to reconstruct the state of these WAL files to
+replay data not yet in an RFile; however, this is a difficult task and is not implemented in any automated fashion.
+
+*A*: The +importdirectory+ shell command can be used to import RFiles from the old instance into a newly created instance,
+but extreme care should go into the decision to do this as it may result in reintroduction of stale data or the
+omission of new data.
+
+### File Naming Conventions
+
+*Q*: Why are files named like they are? Why do some start with +C+ and others with +F+?
+
+*A*: The file names give you a basic idea for the source of the file.
+
+The base of the filename is a base-36 unique number. All filenames in accumulo are coordinated
+with a counter in zookeeper, so they are always unique, which is useful for debugging.
+
+The leading letter gives you an idea of how the file was created:
+
++F+::
+ Flush: entries in memory were written to a file (Minor Compaction)
+
++M+::
+ Merging compaction: entries in memory were combined with the smallest file to create one new file
+
++C+::
+ Several files, but not all files, were combined to produce this file (Major Compaction)
+
++A+::
+ All files were compacted, delete entries were dropped
+
++I+::
+ Bulk import, complete, sorted index files. Always in a directory starting with +b-+
+
+This simple file naming convention allows you to see the basic structure of the files from just
+their filenames, and reason about what should be happening to them next, just
+by scanning their entries in the metadata tables.
+
+For example, if you see multiple files with +M+ prefixes, the tablet is, or was, up against its
+maximum file limit, so it began merging memory updates with files to keep the file count reasonable. This
+slows down ingest performance, so knowing there are many files like this tells you that the system
+is struggling to keep up with ingest vs the compaction strategy which reduces the number of files.
+
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/images/accumulo-logo.png
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/images/accumulo-logo.png b/docs/src/main/asciidoc/images/accumulo-logo.png
new file mode 100644
index 0000000..5b0f6b4
Binary files /dev/null and b/docs/src/main/asciidoc/images/accumulo-logo.png differ
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/images/data_distribution.png
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/images/data_distribution.png b/docs/src/main/asciidoc/images/data_distribution.png
new file mode 100644
index 0000000..7f18d3f
Binary files /dev/null and b/docs/src/main/asciidoc/images/data_distribution.png differ
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/asciidoc/images/failure_handling.png
----------------------------------------------------------------------
diff --git a/docs/src/main/asciidoc/images/failure_handling.png b/docs/src/main/asciidoc/images/failure_handling.png
new file mode 100644
index 0000000..c131de6
Binary files /dev/null and b/docs/src/main/asciidoc/images/failure_handling.png differ
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/accumulo_user_manual.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/accumulo_user_manual.tex b/docs/src/main/latex/accumulo_user_manual/accumulo_user_manual.tex
deleted file mode 100644
index ae2a440..0000000
--- a/docs/src/main/latex/accumulo_user_manual/accumulo_user_manual.tex
+++ /dev/null
@@ -1,55 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\documentclass[11pt]{report}
-\title{Apache Accumulo User Manual\\
-Version 1.6}
-\usepackage{alltt}
-\usepackage{multirow}
-\usepackage{graphicx}
-\usepackage[T1]{fontenc}
-\usepackage{ulem}
-\renewcommand{\ttdefault}{txtt}
-
-\setlength{\textwidth}{6.25in}
-\evensidemargin=0in
-\oddsidemargin=0in
-
-\usepackage{parskip}
-\usepackage{hyperref}
-\newenvironment{shellcommands}[0]
-{\ttfamily \begin{flushleft} \setlength{\parindent}{0cm} \setlength{\parskip}{12mm} \hangindent=1cm}
-{\end{flushleft} \setlength{\parindent}{7mm} \hangindent=0cm \normalfont}
-
-\begin{document}
-\maketitle
-\tableofcontents
-\include{chapters/introduction}
-\include{chapters/design}
-\include{chapters/shell}
-\include{chapters/clients}
-\include{chapters/development_clients}
-\include{chapters/table_configuration}
-\include{chapters/table_design}
-\include{chapters/high_speed_ingest}
-\include{chapters/analytics}
-\include{chapters/security}
-\include{chapters/administration}
-\include{chapters/multivolume}
-\include{chapters/troubleshooting}
-\appendix
-\include{appendices/config}
-\end{document}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex b/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
deleted file mode 100644
index 743b496..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/administration.tex
+++ /dev/null
@@ -1,431 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Administration}
-
-\section{Hardware}
-
-Because we are running essentially two or three systems simultaneously layered
-across the cluster: HDFS, Accumulo and MapReduce, it is typical for hardware to
-consist of 4 to 8 cores, and 8 to 32 GB RAM. This is so each running process can have
-at least one core and 2 - 4 GB each.
-
-One core running HDFS can typically keep 2 to 4 disks busy, so each machine may
-typically have as little as 2 x 300GB disks and as much as 4 x 1TB or 2TB disks.
-
-It is possible to do with less than this, such as with 1u servers with 2 cores and 4GB
-each, but in this case it is recommended to only run up to two processes per
-machine - i.e. DataNode and TabletServer or DataNode and MapReduce worker but
-not all three. The constraint here is having enough available heap space for all the
-processes on a machine.
-
-\section{Network}
-
-Accumulo communicates via remote procedure calls over TCP/IP for both passing
-data and control messages. In addition, Accumulo uses HDFS clients to
-communicate with HDFS. To achieve good ingest and query performance, sufficient
-network bandwidth must be available between any two machines.
-
-In addition to needing access to ports associated with HDFS and ZooKeeper, Accumulo will
-use the following default ports. Please make sure that they are open, or change
-their value in conf/accumulo-site.xml.
-
-\begin{table}[t]
-\centering
-\begin{tabular}{|l|c|c|}
-\hline
-Port & Description & Property Name\\
-\hline
-4445 & Shutdown Port (Accumulo MiniCluster) & n/a\\
-\hline
-4560 & Accumulo monitor (for centralized log display) & monitor.port.log4j\\
-\hline
-9997 & Tablet Server & tserver.port.client\\
-\hline
-9999 & Master Server & master.port.client\\
-\hline
-12234 & Accumulo Tracer & trace.port.client\\
-\hline
-42424 & Accumulo Proxy Server & n/a\\
-\hline
-50091 & Accumulo GC & gc.port.client\\
-\hline
-50095 & Accumulo HTTP monitor & monitor.port.client\\
-\hline
-\end{tabular}
-\caption{Accumulo default ports}
-\label{tab:template}
-\end{table}
-
-In addition, the user can provide '0' and an ephemeral port will be chosen instead. This
-ephemeral port is likely to be unique and not already bound. Thus, configuring ports to
-use '0' instead of an explicit value, should, in most cases, work around any issues of
-running multiple distinct Accumulo instances (or any other process which tries to use the
-same default ports) on the same hardware.
-
-\section{Installation}
-Choose a directory for the Accumulo installation. This directory will be referenced
-by the environment variable \texttt{\$ACCUMULO\_HOME}. Run the following:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ tar xzf accumulo-1.5.0-bin.tar.gz # unpack to subdirectory
-$ mv accumulo-1.5.0 $ACCUMULO_HOME # move to desired location
-\end{verbatim}\endgroup
-
-Repeat this step at each machine within the cluster. Usually all machines have the
-same \texttt{\$ACCUMULO\_HOME}.
-
-\section{Dependencies}
-Accumulo requires HDFS and ZooKeeper to be configured and running
-before starting. Password-less SSH should be configured between at least the
-Accumulo master and TabletServer machines. It is also a good idea to run Network
-Time Protocol (NTP) within the cluster to ensure nodes' clocks don't get too out of
-sync, which can cause problems with automatically timestamped data.
-
-\section{Configuration}
-
-Accumulo is configured by editing several Shell and XML files found in
-\texttt{\$ACCUMULO\_HOME/conf}. The structure closely resembles Hadoop's configuration
-files.
-
-\subsection{Edit conf/accumulo-env.sh}
-
-Accumulo needs to know where to find the software it depends on. Edit accumulo-env.sh
-and specify the following:
-
-\begin{enumerate}
-\item{Enter the location of the installation directory of Accumulo for \texttt{\$ACCUMULO\_HOME}}
-\item{Enter your system's Java home for \texttt{\$JAVA\_HOME}}
-\item{Enter the location of Hadoop for \texttt{\$HADOOP\_PREFIX}}
-\item{Choose a location for Accumulo logs and enter it for \texttt{\$ACCUMULO\_LOG\_DIR}}
-\item{Enter the location of ZooKeeper for \texttt{\$ZOOKEEPER\_HOME}}
-\end{enumerate}
-
-By default Accumulo TabletServers are set to use 1GB of memory. You may change
-this by altering the value of \texttt{\$ACCUMULO\_TSERVER\_OPTS}. Note the syntax is that of
-the Java JVM command line options. This value should be less than the physical
-memory of the machines running TabletServers.
-
-There are similar options for the master's memory usage and the garbage collector
-process. Reduce these if they exceed the physical RAM of your hardware and
-increase them, within the bounds of the physical RAM, if a process fails because of
-insufficient memory.
-
-Note that you will be specifying the Java heap space in accumulo-env.sh. You should
-make sure that the total heap space used for the Accumulo tserver and the Hadoop
-DataNode and TaskTracker is less than the available memory on each slave node in
-the cluster. On large clusters, it is recommended that the Accumulo master, Hadoop
-NameNode, secondary NameNode, and Hadoop JobTracker all be run on separate
-machines to allow them to use more heap space. If you are running these on the
-same machine on a small cluster, likewise make sure their heap space settings fit
-within the available memory.
-
-\subsection{Native Map}
-
-The tablet server uses a data structure called a MemTable to store sorted key/value
-pairs in memory when they are first received from the client. When a minor compaction
-occurs, this data structure is written to HDFS. The MemTable will default to using
-memory in the JVM but a JNI version, called the native map, can be used to significantly
-speed up performance by utilizing the memory space of the native operating system. The
-native map also avoids the performance implications brought on by garbage collection
-in the JVM by causing it to pause much less frequently.
-
-32-bit and 64-bit Linux versions of the native map ship with the Accumulo dist package.
-For other operating systems, the native map can be built from the codebase in two ways-
-from maven or from the Makefile.
-
-\begin{enumerate}
-\item{Build from maven using the following command: \texttt{mvn clean package -Pnative.}}
-\item{Build from the c++ source by running \texttt{make} in the \texttt{\$ACCUMULO\_HOME/server/src/main/c++} directory.}
-\end{enumerate}
-
-After building the native map from the source, you will find the artifact in
-\texttt{\$ACCUMULO\_HOME/lib/native.} Upon starting up, the tablet server will look
-in this directory for the map library. If the file is renamed or moved from its
-target directory, the tablet server may not be able to find it.
-
-\subsection{Cluster Specification}
-
-On the machine that will serve as the Accumulo master:
-
-\begin{enumerate}
-\item{Write the IP address or domain name of the Accumulo Master to the\\\texttt{\$ACCUMULO\_HOME/conf/masters} file.}
-\item{Write the IP addresses or domain name of the machines that will be TabletServers in\\\texttt{\$ACCUMULO\_HOME/conf/slaves}, one per line.}
-\end{enumerate}
-
-Note that if using domain names rather than IP addresses, DNS must be configured
-properly for all machines participating in the cluster. DNS can be a confusing source
-of errors.
-
-\subsection{Accumulo Settings}
-Specify appropriate values for the following settings in\\
-\texttt{\$ACCUMULO\_HOME/conf/accumulo-site.xml} :
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-<property>
- <name>instance.zookeeper.host</name>
- <value>zooserver-one:2181,zooserver-two:2181</value>
- <description>list of zookeeper servers</description>
-</property>
-\end{verbatim}\endgroup
-
-This enables Accumulo to find ZooKeeper. Accumulo uses ZooKeeper to coordinate
-settings between processes and helps finalize TabletServer failure.
-
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-<property>
- <name>instance.secret</name>
- <value>DEFAULT</value>
-</property>
-\end{verbatim}\endgroup
-
-The instance needs a secret to enable secure communication between servers. Configure your
-secret and make sure that the \texttt{accumulo-site.xml} file is not readable to other users.
-
-Some settings can be modified via the Accumulo shell and take effect immediately, but
-some settings require a process restart to take effect. See the configuration documentation
-(available in the docs directory of the tarball and in Appendix~\ref{app:config}) for details.
-
-\subsection{Deploy Configuration}
-
-Copy the masters, slaves, accumulo-env.sh, and if necessary, accumulo-site.xml
-from the\\\texttt{\$ACCUMULO\_HOME/conf/} directory on the master to all the machines
-specified in the slaves file.
-
-\section{Initialization}
-
-Accumulo must be initialized to create the structures it uses internally to locate
-data across the cluster. HDFS is required to be configured and running before
-Accumulo can be initialized.
-
-Once HDFS is started, initialization can be performed by executing\\
-\texttt{\$ACCUMULO\_HOME/bin/accumulo init} . This script will prompt for a name
-for this instance of Accumulo. The instance name is used to identify a set of tables
-and instance-specific settings. The script will then write some information into
-HDFS so Accumulo can start properly.
-
-The initialization script will prompt you to set a root password. Once Accumulo is
-initialized it can be started.
-
-\section{Running}
-
-\subsection{Starting Accumulo}
-
-Make sure Hadoop is configured on all of the machines in the cluster, including
-access to a shared HDFS instance. Make sure HDFS and ZooKeeper are running.
-Make sure ZooKeeper is configured and running on at least one machine in the
-cluster.
-Start Accumulo using the \texttt{bin/start-all.sh} script.
-
-To verify that Accumulo is running, check the Status page as described under
-\emph{Monitoring}. In addition, the Shell can provide some information about the status of
-tables via reading the metadata tables.
-
-\subsection{Stopping Accumulo}
-
-To shutdown cleanly, run \texttt{bin/stop-all.sh} and the master will orchestrate the
-shutdown of all the tablet servers. Shutdown waits for all minor compactions to finish, so it may
-take some time for particular configurations.
-
-\subsection{Adding a Node}
-
-Update your \texttt{\$ACCUMULO\_HOME/conf/slaves} (or \texttt{\$ACCUMULO\_CONF\_DIR/slaves}) file to account for the addition.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/bin/accumulo admin start <host(s)> {<host> ...}
-\end{verbatim}\endgroup
-
-Alternatively, you can ssh to each of the hosts you want to add and run:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO\_HOME/bin/start-here.sh
-\end{verbatim}\endgroup
-
-Make sure the host in question has the new configuration, or else the tablet
-server won't start; at a minimum this needs to be on the host(s) being added,
-but in practice it's good to ensure consistent configuration across all nodes.
-
-\subsection{Decomissioning a Node}
-
-If you need to take a node out of operation, you can trigger a graceful shutdown of a tablet
-server. Accumulo will automatically rebalance the tablets across the available tablet servers.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/bin/accumulo admin stop <host(s)> {<host> ...}
-\end{verbatim}\endgroup
-
-Alternatively, you can ssh to each of the hosts you want to remove and run:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO\_HOME/bin/stop-here.sh
-\end{verbatim}\endgroup
-
-Be sure to update your \texttt{\$ACCUMUL\_HOME/conf/slaves} (or \texttt{\$ACCUMULO\_CONF\_DIR/slaves}) file to
-account for the removal of these hosts. Bear in mind that the monitor will not re-read the
-slaves file automatically, so it will report the decomissioned servers as down; it's
-recommended that you restart the monitor so that the node list is up to date.
-
-\section{Monitoring}
-
-The Accumulo Master provides an interface for monitoring the status and health of
-Accumulo components. This interface can be accessed by pointing a web browser to\\
-\texttt{http://accumulomaster:50095/status}
-
-\section{Tracing}
-It can be difficult to determine why some operations are taking longer
-than expected. For example, you may be looking up items with very low
-latency, but sometimes the lookups take much longer. Determining the
-cause of the delay is difficult because the system is distributed, and
-the typical lookup is fast.
-
-Accumulo has been instrumented to record the time that various
-operations take when tracing is turned on. The fact that tracing is
-enabled follows all the requests made on behalf of the user throughout
-the distributed infrastructure of accumulo, and across all threads of
-execution.
-
-These time spans will be inserted into the \texttt{trace} table in
-Accumulo. You can browse recent traces from the Accumulo monitor
-page. You can also read the \texttt{trace} table directly like any
-other table.
-
-The design of Accumulo's distributed tracing follows that of
-\href{http://research.google.com/pubs/pub36356.html}{Google's Dapper}.
-
-\subsection{Tracers}
-To collect traces, Accumulo needs at least one server listed in
-\\\texttt{\$ACCUMULO\_HOME/conf/tracers}. The server collects traces
-from clients and writes them to the \texttt{trace} table. The Accumulo
-user that the tracer connects to Accumulo with can be configured with
-the following properties
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-trace.user
-trace.token.property.password
-\end{verbatim}\endgroup
-
-\subsection{Instrumenting a Client}
-Tracing can be used to measure a client operation, such as a scan, as
-the operation traverses the distributed system. To enable tracing for
-your application call
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-DistributedTrace.enable(instance, new ZooReader(instance), hostname, "myApplication");
-\end{verbatim}\endgroup
-
-Once tracing has been enabled, a client can wrap an operation in a trace.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Trace.on("Client Scan");
-BatchScanner scanner = conn.createBatchScanner(...);
-// Configure your scanner
-for (Entry entry : scanner) {
-}
-Trace.off();
-\end{verbatim}\endgroup
-
-Additionally, the user can create additional Spans within a Trace.
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Trace.on("Client Update");
-...
-Span readSpan = Trace.start("Read");
-...
-readSpan.stop();
-...
-Span writeSpan = Trace.start("Write");
-...
-writeSpan.stop();
-Trace.off();
-\end{verbatim}\endgroup
-
-Like Dapper, Accumulo tracing supports user defined annotations to associate additional data with a Trace.
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-...
-int numberOfEntriesRead = 0;
-Span readSpan = Trace.start("Read");
-// Do the read, update the counter
-...
-readSpan.data("Number of Entries Read", String.valueOf(numberOfEntriesRead));
-\end{verbatim}\endgroup
-
-Some client operations may have a high volume within your
-application. As such, you may wish to only sample a percentage of
-operations for tracing. As seen below, the CountSampler can be used to
-help enable tracing for 1-in-1000 operations
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Sampler sampler = new CountSampler(1000);
-...
-if (sampler.next()) {
- Trace.on("Read");
-}
-...
-Trace.offNoFlush();
-\end{verbatim}\endgroup
-
-It should be noted that it is safe to turn off tracing even if it
-isn't currently active. The Trace.offNoFlush() should be used if the
-user does not wish to have Trace.off() block while flushing trace
-data.
-
-\subsection{Viewing Collected Traces}
-To view collected traces, use the "Recent Traces" link on the Monitor
-UI. You can also programmatically access and print traces using the
-\texttt{TraceDump} class.
-
-\subsection{Tracing from the Shell}
-You can enable tracing for operations run from the shell by using the
-\texttt{trace on} and \texttt{trace off} commands.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@test test> trace on
-root@test test> scan
-a b:c [] d
-root@test test> trace off
-Waiting for trace information
-Waiting for trace information
-Trace started at 2013/08/26 13:24:08.332
-Time Start Service@Location Name
- 3628+0 shell@localhost shell:root
- 8+1690 shell@localhost scan
- 7+1691 shell@localhost scan:location
- 6+1692 tserver@localhost startScan
- 5+1692 tserver@localhost tablet read ahead 6
-\end{verbatim}\endgroup
-
-\section{Logging}
-Accumulo processes each write to a set of log files. By default these are found under\\
-\texttt{\$ACCUMULO/logs/}.
-
-\section{Recovery}
-
-In the event of TabletServer failure or error on shutting Accumulo down, some
-mutations may not have been minor compacted to HDFS properly. In this case,
-Accumulo will automatically reapply such mutations from the write-ahead log
-either when the tablets from the failed server are reassigned by the Master (in the
-case of a single TabletServer failure) or the next time Accumulo starts (in the event of
-failure during shutdown).
-
-Recovery is performed by asking a tablet server to sort the logs so that tablets can easily find their missing
-updates. The sort status of each file is displayed on
-Accumulo monitor status page. Once the recovery is complete any
-tablets involved should return to an ``online" state. Until then those tablets will be
-unavailable to clients.
-
-The Accumulo client library is configured to retry failed mutations and in many
-cases clients will be able to continue processing after the recovery process without
-throwing an exception.
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/analytics.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/analytics.tex b/docs/src/main/latex/accumulo_user_manual/chapters/analytics.tex
deleted file mode 100644
index 301120b..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/analytics.tex
+++ /dev/null
@@ -1,248 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Analytics}
-
-Accumulo supports more advanced data processing than simply keeping keys
-sorted and performing efficient lookups. Analytics can be developed by using
-MapReduce and Iterators in conjunction with Accumulo tables.
-
-\section{MapReduce}
-
-Accumulo tables can be used as the source and destination of MapReduce jobs. To
-use an Accumulo table with a MapReduce job (specifically with the new Hadoop API
-as of version 0.20), configure the job parameters to use the AccumuloInputFormat
-and AccumuloOutputFormat. Accumulo specific parameters can be set via these
-two format classes to do the following:
-\begin{itemize}
-\item{Authenticate and provide user credentials for the input}
-\item{Restrict the scan to a range of rows}
-\item{Restrict the input to a subset of available columns}
-\end{itemize}
-
-\subsection{Mapper and Reducer classes}
-
-To read from an Accumulo table create a Mapper with the following class
-parameterization and be sure to configure the AccumuloInputFormat.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-class MyMapper extends Mapper<Key,Value,WritableComparable,Writable> {
- public void map(Key k, Value v, Context c) {
- // transform key and value data here
- }
-}
-\end{verbatim}\endgroup
-
-To write to an Accumulo table, create a Reducer with the following class
-parameterization and be sure to configure the AccumuloOutputFormat. The key
-emitted from the Reducer identifies the table to which the mutation is sent. This
-allows a single Reducer to write to more than one table if desired. A default table
-can be configured using the AccumuloOutputFormat, in which case the output table
-name does not have to be passed to the Context object within the Reducer.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-class MyReducer extends Reducer<WritableComparable, Writable, Text, Mutation> {
- public void reduce(WritableComparable key, Iterable<Text> values, Context c) {
- Mutation m;
- // create the mutation based on input key and value
- c.write(new Text("output-table"), m);
- }
-}
-\end{verbatim}\endgroup
-
-The Text object passed as the output should contain the name of the table to which
-this mutation should be applied. The Text can be null in which case the mutation
-will be applied to the default table name specified in the AccumuloOutputFormat
-options.
-
-\subsection{AccumuloInputFormat options}
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Job job = new Job(getConf());
-AccumuloInputFormat.setInputInfo(job,
- "user",
- "passwd".getBytes(),
- "table",
- new Authorizations());
-
-AccumuloInputFormat.setZooKeeperInstance(job, "myinstance",
- "zooserver-one,zooserver-two");
-\end{verbatim}\endgroup
-
-\Large
-\textbf{Optional Settings:}
-\normalsize
-
-To restrict Accumulo to a set of row ranges:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-ArrayList<Range> ranges = new ArrayList<Range>();
-// populate array list of row ranges ...
-AccumuloInputFormat.setRanges(job, ranges);
-\end{verbatim}\endgroup
-
-To restrict Accumulo to a list of columns:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-ArrayList<Pair<Text,Text>> columns = new ArrayList<Pair<Text,Text>>();
-// populate list of columns
-AccumuloInputFormat.fetchColumns(job, columns);
-\end{verbatim}\endgroup
-
-To use a regular expression to match row IDs:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-IteratorSetting is = new IteratorSetting(30, RexExFilter.class);
-RegExFilter.setRegexs(is, ".*suffix", null, null, null, true);
-AccumuloInputFormat.addIterator(job, is);
-\end{verbatim}\endgroup
-
-\subsection{AccumuloMultiTableInputFormat options}
-
-The AccumuloMultiTableInputFormat allows the scanning over multiple tables
-in a single MapReduce job. Separate ranges, columns, and iterators can be
-used for each table.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-InputTableConfig tableOneConfig = new InputTableConfig();
-InputTableConfig tableTwoConfig = new InputTableConfig();
-\end{verbatim}\endgroup
-
-To set the configuration objects on the job:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Map<String, InputTableConfig> configs = new HashMap<String,InputTableConfig>();
-configs.put("table1", tableOneConfig);
-configs.put("table2", tableTwoConfig);
-AccumuloMultiTableInputFormat.setInputTableConfigs(job, configs);
-\end{verbatim}\endgroup
-
-\Large
-\textbf{Optional settings:}
-\normalsize
-
-To restrict to a set of ranges:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-ArrayList<Range> tableOneRanges = new ArrayList<Range>();
-ArrayList<Range> tableTwoRanges = new ArrayList<Range>();
-// populate array lists of row ranges for tables...
-tableOneConfig.setRanges(tableOneRanges);
-tableTwoConfig.setRanges(tableTwoRanges);
-\end{verbatim}\endgroup
-
-To restrict Accumulo to a list of columns:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-ArrayList<Pair<Text,Text>> tableOneColumns = new ArrayList<Pair<Text,Text>>();
-ArrayList<Pair<Text,Text>> tableTwoColumns = new ArrayList<Pair<Text,Text>>();
-// populate lists of columns for each of the tables ...
-tableOneConfig.fetchColumns(tableOneColumns);
-tableTwoConfig.fetchColumns(tableTwoColumns);
-\end{verbatim}\endgroup
-
-To set scan iterators:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-List<IteratorSetting> tableOneIterators = new ArrayList<IteratorSetting>();
-List<IteratorSetting> tableTwoIterators = new ArrayList<IteratorSetting>();
-// populate the lists of iterator settings for each of the tables ...
-tableOneConfig.setIterators(tableOneIterators);
-tableTwoConfig.setIterators(tableTwoIterators);
-\end{verbatim}\endgroup
-
-
-The name of the table can be retrieved from the input split:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-class MyMapper extends Mapper<Key,Value,WritableComparable,Writable> {
- public void map(Key k, Value v, Context c) {
- RangeInputSplit split = (RangeInputSplit)c.getInputSplit();
- String tableName = split.getTableName();
- // do something with table name
- }
-}
-\end{verbatim}\endgroup
-
-
-\subsection{AccumuloOutputFormat options}
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-boolean createTables = true;
-String defaultTable = "mytable";
-
-AccumuloOutputFormat.setOutputInfo(job,
- "user",
- "passwd".getBytes(),
- createTables,
- defaultTable);
-
-AccumuloOutputFormat.setZooKeeperInstance(job, "myinstance",
- "zooserver-one,zooserver-two");
-\end{verbatim}\endgroup
-
-\Large
-\textbf{Optional Settings:}
-\normalsize
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-AccumuloOutputFormat.setMaxLatency(job, 300000); // milliseconds
-AccumuloOutputFormat.setMaxMutationBufferSize(job, 50000000); // bytes
-\end{verbatim}\endgroup
-
-An example of using MapReduce with Accumulo can be found at\\
-accumulo/docs/examples/README.mapred
-
-\section{Combiners}
-
-Many applications can benefit from the ability to aggregate values across common
-keys. This can be done via Combiner iterators and is similar to the Reduce step in
-MapReduce. This provides the ability to define online, incrementally updated
-analytics without the overhead or latency associated with batch-oriented
-MapReduce jobs.
-
-All that is needed to aggregate values of a table is to identify the fields over which
-values will be grouped, insert mutations with those fields as the key, and configure
-the table with a combining iterator that supports the summarizing operation
-desired.
-
-The only restriction on an combining iterator is that the combiner developer
-should not assume that all values for a given key have been seen, since new
-mutations can be inserted at anytime. This precludes using the total number of
-values in the aggregation such as when calculating an average, for example.
-
-\subsection{Feature Vectors}
-
-An interesting use of combining iterators within an Accumulo table is to store
-feature vectors for use in machine learning algorithms. For example, many
-algorithms such as k-means clustering, support vector machines, anomaly detection,
-etc. use the concept of a feature vector and the calculation of distance metrics to
-learn a particular model. The columns in an Accumulo table can be used to efficiently
-store sparse features and their weights to be incrementally updated via the use of an
-combining iterator.
-
-\section{Statistical Modeling}
-
-Statistical models that need to be updated by many machines in parallel could be
-similarly stored within an Accumulo table. For example, a MapReduce job that is
-iteratively updating a global statistical model could have each map or reduce worker
-reference the parts of the model to be read and updated through an embedded
-Accumulo client.
-
-Using Accumulo this way enables efficient and fast lookups and updates of small
-pieces of information in a random access pattern, which is complementary to
-MapReduce's sequential access model.
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/clients.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/clients.tex b/docs/src/main/latex/accumulo_user_manual/chapters/clients.tex
deleted file mode 100644
index 925c352..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/clients.tex
+++ /dev/null
@@ -1,331 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Writing Accumulo Clients}
-
-\section{Running Client Code}
-
-There are multiple ways to run Java code that uses Accumulo. Below is a list
-of the different ways to execute client code.
-
-\begin{itemize}
- \item using java executable
- \item using the accumulo script
- \item using the tool script
-\end{itemize}
-
-In order to run client code written to run against Accumulo, you will need to
-include the jars that Accumulo depends on in your classpath. Accumulo client
-code depends on Hadoop and Zookeeper. For Hadoop add the hadoop client jar, all
-of the jars in the Hadoop lib directory, and the conf directory to the
-classpath. For Zookeeper 3.3 you only need to add the Zookeeper jar, and not
-what is in the Zookeeper lib directory. You can run the following command on a
-configured Accumulo system to see what its using for its classpath.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/bin/accumulo classpath
-\end{verbatim}\endgroup
-
-Another option for running your code is to put a jar file in
-\texttt{\$ACCUMULO\_HOME/lib/ext}. After doing this you can use the accumulo
-script to execute your code. For example if you create a jar containing the
-class com.foo.Client and placed that in lib/ext, then you could use the command
-\texttt{\$ACCUMULO\_HOME/bin/accumulo com.foo.Client} to execute your code.
-
-If you are writing map reduce job that access Accumulo, then you can use the
-bin/tool.sh script to run those jobs. See the map reduce example.
-
-\section{Connecting}
-
-All clients must first identify the Accumulo instance to which they will be
-communicating. Code to do this is as follows:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-String instanceName = "myinstance";
-String zooServers = "zooserver-one,zooserver-two"
-Instance inst = new ZooKeeperInstance(instanceName, zooServers);
-
-Connector conn = inst.getConnector("user", new PasswordToken("passwd"));
-\end{verbatim}\endgroup
-
-\section{Writing Data}
-
-Data are written to Accumulo by creating Mutation objects that represent all the
-changes to the columns of a single row. The changes are made atomically in the
-TabletServer. Clients then add Mutations to a BatchWriter which submits them to
-the appropriate TabletServers.
-
-Mutations can be created thus:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Text rowID = new Text("row1");
-Text colFam = new Text("myColFam");
-Text colQual = new Text("myColQual");
-ColumnVisibility colVis = new ColumnVisibility("public");
-long timestamp = System.currentTimeMillis();
-
-Value value = new Value("myValue".getBytes());
-
-Mutation mutation = new Mutation(rowID);
-mutation.put(colFam, colQual, colVis, timestamp, value);
-\end{verbatim}\endgroup
-
-\subsection{BatchWriter}
-The BatchWriter is highly optimized to send Mutations to multiple TabletServers
-and automatically batches Mutations destined for the same TabletServer to
-amortize network overhead. Care must be taken to avoid changing the contents of
-any Object passed to the BatchWriter since it keeps objects in memory while
-batching.
-
-Mutations are added to a BatchWriter thus:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-//BatchWriterConfig has reasonable defaults
-BatchWriterConfig config = new BatchWriterConfig();
-config.setMaxMemory(10000000L); // bytes available to batchwriter for buffering mutations
-
-BatchWriter writer = conn.createBatchWriter("table", config)
-
-writer.add(mutation);
-
-writer.close();
-\end{verbatim}\endgroup
-
-An example of using the batch writer can be found at\\
-accumulo/docs/examples/README.batch
-
-\subsection{ConditionalWriter}
-The ConditionalWriter enables efficient, atomic read-modify-write operations on
-rows. The ConditionalWriter writes special Mutations which have a list of per
-column conditions that must all be met before the mutation is applied. The
-conditions are checked in the tablet server while a row lock is
-held\footnote{Mutations written by the BatchWriter will not obtain a row
-lock.}. The conditions that can be checked for a column are equality and
-absence. For example a conditional mutation can require that column A is
-absent inorder to be applied. Iterators can be applied when checking
-conditions. Using iterators, many other operations besides equality and
-absence can be checked. For example, using an iterator that converts values
-less than 5 to 0 and everything else to 1, its possible to only apply a
-mutation when a column is less than 5.
-
-In the case when a tablet server dies after a client sent a conditional
-mutation, its not known if the mutation was applied or not. When this happens
-the ConditionalWriter reports a status of UNKNOWN for the ConditionalMutation.
-In many cases this situation can be dealt with by simply reading the row again
-and possibly sending another conditional mutation. If this is not sufficient,
-then a higher level of abstraction can be built by storing transactional
-information within a row.
-
-An example of using the batch writer can be found at\\
-accumulo/docs/examples/README.reservations
-
-\section{Reading Data}
-
-Accumulo is optimized to quickly retrieve the value associated with a given key, and
-to efficiently return ranges of consecutive keys and their associated values.
-
-\subsection{Scanner}
-
-To retrieve data, Clients use a Scanner, which acts like an Iterator over
-keys and values. Scanners can be configured to start and stop at particular keys, and
-to return a subset of the columns available.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// specify which visibilities we are allowed to see
-Authorizations auths = new Authorizations("public");
-
-Scanner scan =
- conn.createScanner("table", auths);
-
-scan.setRange(new Range("harry","john"));
-scan.fetchFamily("attributes");
-
-for(Entry<Key,Value> entry : scan) {
- String row = entry.getKey().getRow();
- Value value = entry.getValue();
-}
-\end{verbatim}\endgroup
-
-\subsection{Isolated Scanner}
-
-Accumulo supports the ability to present an isolated view of rows when
-scanning. There are three possible ways that a row could change in Accumulo :
-
-\begin{itemize}
- \item a mutation applied to a table
- \item iterators executed as part of a minor or major compaction
- \item bulk import of new files
-\end{itemize}
-
-Isolation guarantees that either all or none of the changes made by these
-operations on a row are seen. Use the IsolatedScanner to obtain an isolated
-view of an Accumulo table. When using the regular scanner it is possible to see
-a non isolated view of a row. For example if a mutation modifies three
-columns, it is possible that you will only see two of those modifications.
-With the isolated scanner either all three of the changes are seen or none.
-
-The IsolatedScanner buffers rows on the client side so a large row will not
-crash a tablet server. By default rows are buffered in memory, but the user
-can easily supply their own buffer if they wish to buffer to disk when rows are
-large.
-
-For an example, look at the following
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-examples/simple/src/main/java/org/apache/accumulo/examples/simple/isolation/InterferenceTest.java
-\end{verbatim}\endgroup
-
-\subsection{BatchScanner}
-
-For some types of access, it is more efficient to retrieve several ranges
-simultaneously. This arises when accessing a set of rows that are not consecutive
-whose IDs have been retrieved from a secondary index, for example.
-
-The BatchScanner is configured similarly to the Scanner; it can be configured to
-retrieve a subset of the columns available, but rather than passing a single Range,
-BatchScanners accept a set of Ranges. It is important to note that the keys returned
-by a BatchScanner are not in sorted order since the keys streamed are from multiple
-TabletServers in parallel.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-ArrayList<Range> ranges = new ArrayList<Range>();
-// populate list of ranges ...
-
-BatchScanner bscan =
- conn.createBatchScanner("table", auths, 10);
-
-bscan.setRanges(ranges);
-bscan.fetchFamily("attributes");
-for(Entry<Key,Value> entry : scan)
- System.out.println(entry.getValue());
-\end{verbatim}\endgroup
-
-An example of the BatchScanner can be found at\\
-accumulo/docs/examples/README.batch
-
-\section{Proxy}
-
-The proxy API allows the interaction with Accumulo with languages other than Java.
-A proxy server is provided in the codebase and a client can further be generated.
-
-\subsection{Prequisites}
-
-The proxy server can live on any node in which the basic client API would work. That
-means it must be able to communicate with the Master, ZooKeepers, NameNode, and the
-Data nodes. A proxy client only needs the ability to communicate with the proxy server.
-
-
-\subsection{Configuration}
-
-The configuration options for the proxy server live inside of a properties file. At
-the very least, you need to supply the following properties:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-protocolFactory=org.apache.thrift.protocol.TCompactProtocol$Factory
-tokenClass=org.apache.accumulo.core.client.security.tokens.PasswordToken
-port=42424
-instance=test
-zookeepers=localhost:2181
-\end{verbatim}\endgroup
-
-You can find a sample configuration file in your distribution:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/proxy/proxy.properties.
-\end{verbatim}\endgroup
-
-This sample configuration file further demonstrates an abilty to back the proxy server
-by MockAccumulo or the MiniAccumuloCluster.
-
-\subsection{Running the Proxy Server}
-
-After the properties file holding the configuration is created, the proxy server
-can be started using the following command in the Accumulo distribution (assuming
-you your properties file is named config.properties):
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/bin/accumulo proxy -p config.properties
-\end{verbatim}\endgroup
-
-\subsection{Creating a Proxy Client}
-
-Aside from installing the Thrift compiler, you will also need the language-specific library
-for Thrift installed to generate client code in that language. Typically, your operating
-system's package manager will be able to automatically install these for you in an expected
-location such as /usr/lib/python/site-packages/thrift.
-
-You can find the thrift file for generating the client:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/proxy/proxy.thrift.
-\end{verbatim}\endgroup
-
-After a client is generated, the port specified in the configuration properties above will be
-used to connect to the server.
-
-\subsection{Using a Proxy Client}
-
-The following examples have been written in Java and the method signatures may be
-slightly different depending on the language specified when generating client with
-the Thrift compiler. After initiating a connection to the Proxy (see Apache Thrift's
-documentation for examples of connecting to a Thrift service), the methods on the
-proxy client will be available. The first thing to do is log in:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Map password = new HashMap<String,String>();
-password.put("password", "secret");
-ByteBuffer token = client.login("root", password);
-\end{verbatim}\endgroup
-
-Once logged in, the token returned will be used for most subsequent calls to the client.
-Let's create a table, add some data, scan the table, and delete it.
-
-
-First, create a table.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-client.createTable(token, "myTable", true, TimeType.MILLIS);
-\end{verbatim}\endgroup
-
-
-Next, add some data:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// first, create a writer on the server
-String writer = client.createWriter(token, "myTable", new WriterOptions());
-
-// build column updates
-Map<ByteBuffer, List<ColumnUpdate> cells> cellsToUpdate = //...
-
-// send updates to the server
-client.updateAndFlush(writer, "myTable", cellsToUpdate);
-
-client.closeWriter(writer);
-\end{verbatim}\endgroup
-
-
-Scan for the data and batch the return of the results on the server:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-String scanner = client.createScanner(token, "myTable", new ScanOptions());
-ScanResult results = client.nextK(scanner, 100);
-
-for(KeyValue keyValue : results.getResultsIterator()) {
- // do something with results
-}
-
-client.closeScanner(scanner);
-\end{verbatim}\endgroup
[18/50] [abbrv] ACCUMULO-2583 Change up the thrift and stub out a
service to choose a random tserver to service replication work.
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java
new file mode 100644
index 0000000..d828d7d
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationServicer.java
@@ -0,0 +1,1917 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.accumulo.core.replication.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class RemoteReplicationServicer {
+
+ public interface Iface {
+
+ public void replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException;
+
+ public void replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException;
+
+ }
+
+ public interface AsyncIface {
+
+ public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateLog_call> resultHandler) throws org.apache.thrift.TException;
+
+ public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException;
+
+ }
+
+ public static class Client extends org.apache.thrift.TServiceClient implements Iface {
+ public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
+ public Factory() {}
+ public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
+ return new Client(prot);
+ }
+ public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ return new Client(iprot, oprot);
+ }
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol prot)
+ {
+ super(prot, prot);
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ super(iprot, oprot);
+ }
+
+ public void replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ send_replicateLog(remoteTableId, data);
+ recv_replicateLog();
+ }
+
+ public void send_replicateLog(int remoteTableId, WalEdits data) throws org.apache.thrift.TException
+ {
+ replicateLog_args args = new replicateLog_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ sendBase("replicateLog", args);
+ }
+
+ public void recv_replicateLog() throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ replicateLog_result result = new replicateLog_result();
+ receiveBase(result, "replicateLog");
+ if (result.e != null) {
+ throw result.e;
+ }
+ return;
+ }
+
+ public void replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ send_replicateKeyValues(remoteTableId, data);
+ recv_replicateKeyValues();
+ }
+
+ public void send_replicateKeyValues(int remoteTableId, KeyValues data) throws org.apache.thrift.TException
+ {
+ replicateKeyValues_args args = new replicateKeyValues_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ sendBase("replicateKeyValues", args);
+ }
+
+ public void recv_replicateKeyValues() throws RemoteReplicationException, org.apache.thrift.TException
+ {
+ replicateKeyValues_result result = new replicateKeyValues_result();
+ receiveBase(result, "replicateKeyValues");
+ if (result.e != null) {
+ throw result.e;
+ }
+ return;
+ }
+
+ }
+ public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
+ public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
+ private org.apache.thrift.async.TAsyncClientManager clientManager;
+ private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
+ public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
+ this.clientManager = clientManager;
+ this.protocolFactory = protocolFactory;
+ }
+ public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
+ return new AsyncClient(protocolFactory, clientManager, transport);
+ }
+ }
+
+ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
+ super(protocolFactory, clientManager, transport);
+ }
+
+ public void replicateLog(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ replicateLog_call method_call = new replicateLog_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class replicateLog_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int remoteTableId;
+ private WalEdits data;
+ public replicateLog_call(int remoteTableId, WalEdits data, org.apache.thrift.async.AsyncMethodCallback<replicateLog_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.remoteTableId = remoteTableId;
+ this.data = data;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateLog", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ replicateLog_args args = new replicateLog_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public void getResult() throws RemoteReplicationException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ (new Client(prot)).recv_replicateLog();
+ }
+ }
+
+ public void replicateKeyValues(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ replicateKeyValues_call method_call = new replicateKeyValues_call(remoteTableId, data, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class replicateKeyValues_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int remoteTableId;
+ private KeyValues data;
+ public replicateKeyValues_call(int remoteTableId, KeyValues data, org.apache.thrift.async.AsyncMethodCallback<replicateKeyValues_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.remoteTableId = remoteTableId;
+ this.data = data;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("replicateKeyValues", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ replicateKeyValues_args args = new replicateKeyValues_args();
+ args.setRemoteTableId(remoteTableId);
+ args.setData(data);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public void getResult() throws RemoteReplicationException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ (new Client(prot)).recv_replicateKeyValues();
+ }
+ }
+
+ }
+
+ public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
+ public Processor(I iface) {
+ super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
+ }
+
+ protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ processMap.put("replicateLog", new replicateLog());
+ processMap.put("replicateKeyValues", new replicateKeyValues());
+ return processMap;
+ }
+
+ public static class replicateLog<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateLog_args> {
+ public replicateLog() {
+ super("replicateLog");
+ }
+
+ public replicateLog_args getEmptyArgsInstance() {
+ return new replicateLog_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public replicateLog_result getResult(I iface, replicateLog_args args) throws org.apache.thrift.TException {
+ replicateLog_result result = new replicateLog_result();
+ try {
+ iface.replicateLog(args.remoteTableId, args.data);
+ } catch (RemoteReplicationException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ public static class replicateKeyValues<I extends Iface> extends org.apache.thrift.ProcessFunction<I, replicateKeyValues_args> {
+ public replicateKeyValues() {
+ super("replicateKeyValues");
+ }
+
+ public replicateKeyValues_args getEmptyArgsInstance() {
+ return new replicateKeyValues_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public replicateKeyValues_result getResult(I iface, replicateKeyValues_args args) throws org.apache.thrift.TException {
+ replicateKeyValues_result result = new replicateKeyValues_result();
+ try {
+ iface.replicateKeyValues(args.remoteTableId, args.data);
+ } catch (RemoteReplicationException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ }
+
+ public static class replicateLog_args implements org.apache.thrift.TBase<replicateLog_args, replicateLog_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_args");
+
+ private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
+ private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateLog_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateLog_argsTupleSchemeFactory());
+ }
+
+ public int remoteTableId; // required
+ public WalEdits data; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ REMOTE_TABLE_ID((short)1, "remoteTableId"),
+ DATA((short)2, "data");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // REMOTE_TABLE_ID
+ return REMOTE_TABLE_ID;
+ case 2: // DATA
+ return DATA;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __REMOTETABLEID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, WalEdits.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_args.class, metaDataMap);
+ }
+
+ public replicateLog_args() {
+ }
+
+ public replicateLog_args(
+ int remoteTableId,
+ WalEdits data)
+ {
+ this();
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ this.data = data;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateLog_args(replicateLog_args other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.remoteTableId = other.remoteTableId;
+ if (other.isSetData()) {
+ this.data = new WalEdits(other.data);
+ }
+ }
+
+ public replicateLog_args deepCopy() {
+ return new replicateLog_args(this);
+ }
+
+ @Override
+ public void clear() {
+ setRemoteTableIdIsSet(false);
+ this.remoteTableId = 0;
+ this.data = null;
+ }
+
+ public int getRemoteTableId() {
+ return this.remoteTableId;
+ }
+
+ public replicateLog_args setRemoteTableId(int remoteTableId) {
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ return this;
+ }
+
+ public void unsetRemoteTableId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
+ public boolean isSetRemoteTableId() {
+ return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ public void setRemoteTableIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
+ }
+
+ public WalEdits getData() {
+ return this.data;
+ }
+
+ public replicateLog_args setData(WalEdits data) {
+ this.data = data;
+ return this;
+ }
+
+ public void unsetData() {
+ this.data = null;
+ }
+
+ /** Returns true if field data is set (has been assigned a value) and false otherwise */
+ public boolean isSetData() {
+ return this.data != null;
+ }
+
+ public void setDataIsSet(boolean value) {
+ if (!value) {
+ this.data = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ if (value == null) {
+ unsetRemoteTableId();
+ } else {
+ setRemoteTableId((Integer)value);
+ }
+ break;
+
+ case DATA:
+ if (value == null) {
+ unsetData();
+ } else {
+ setData((WalEdits)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return Integer.valueOf(getRemoteTableId());
+
+ case DATA:
+ return getData();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return isSetRemoteTableId();
+ case DATA:
+ return isSetData();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateLog_args)
+ return this.equals((replicateLog_args)that);
+ return false;
+ }
+
+ public boolean equals(replicateLog_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_remoteTableId = true;
+ boolean that_present_remoteTableId = true;
+ if (this_present_remoteTableId || that_present_remoteTableId) {
+ if (!(this_present_remoteTableId && that_present_remoteTableId))
+ return false;
+ if (this.remoteTableId != that.remoteTableId)
+ return false;
+ }
+
+ boolean this_present_data = true && this.isSetData();
+ boolean that_present_data = true && that.isSetData();
+ if (this_present_data || that_present_data) {
+ if (!(this_present_data && that_present_data))
+ return false;
+ if (!this.data.equals(that.data))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateLog_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateLog_args typedOther = (replicateLog_args)other;
+
+ lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRemoteTableId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetData()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateLog_args(");
+ boolean first = true;
+
+ sb.append("remoteTableId:");
+ sb.append(this.remoteTableId);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("data:");
+ if (this.data == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.data);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (data != null) {
+ data.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateLog_argsStandardSchemeFactory implements SchemeFactory {
+ public replicateLog_argsStandardScheme getScheme() {
+ return new replicateLog_argsStandardScheme();
+ }
+ }
+
+ private static class replicateLog_argsStandardScheme extends StandardScheme<replicateLog_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // REMOTE_TABLE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // DATA
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.data = new WalEdits();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
+ oprot.writeI32(struct.remoteTableId);
+ oprot.writeFieldEnd();
+ if (struct.data != null) {
+ oprot.writeFieldBegin(DATA_FIELD_DESC);
+ struct.data.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateLog_argsTupleSchemeFactory implements SchemeFactory {
+ public replicateLog_argsTupleScheme getScheme() {
+ return new replicateLog_argsTupleScheme();
+ }
+ }
+
+ private static class replicateLog_argsTupleScheme extends TupleScheme<replicateLog_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRemoteTableId()) {
+ optionals.set(0);
+ }
+ if (struct.isSetData()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetRemoteTableId()) {
+ oprot.writeI32(struct.remoteTableId);
+ }
+ if (struct.isSetData()) {
+ struct.data.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.data = new WalEdits();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class replicateLog_result implements org.apache.thrift.TBase<replicateLog_result, replicateLog_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_result");
+
+ private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateLog_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateLog_resultTupleSchemeFactory());
+ }
+
+ public RemoteReplicationException e; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ E((short)1, "e");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // E
+ return E;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateLog_result.class, metaDataMap);
+ }
+
+ public replicateLog_result() {
+ }
+
+ public replicateLog_result(
+ RemoteReplicationException e)
+ {
+ this();
+ this.e = e;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateLog_result(replicateLog_result other) {
+ if (other.isSetE()) {
+ this.e = new RemoteReplicationException(other.e);
+ }
+ }
+
+ public replicateLog_result deepCopy() {
+ return new replicateLog_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.e = null;
+ }
+
+ public RemoteReplicationException getE() {
+ return this.e;
+ }
+
+ public replicateLog_result setE(RemoteReplicationException e) {
+ this.e = e;
+ return this;
+ }
+
+ public void unsetE() {
+ this.e = null;
+ }
+
+ /** Returns true if field e is set (has been assigned a value) and false otherwise */
+ public boolean isSetE() {
+ return this.e != null;
+ }
+
+ public void setEIsSet(boolean value) {
+ if (!value) {
+ this.e = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case E:
+ if (value == null) {
+ unsetE();
+ } else {
+ setE((RemoteReplicationException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case E:
+ return getE();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case E:
+ return isSetE();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateLog_result)
+ return this.equals((replicateLog_result)that);
+ return false;
+ }
+
+ public boolean equals(replicateLog_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_e = true && this.isSetE();
+ boolean that_present_e = true && that.isSetE();
+ if (this_present_e || that_present_e) {
+ if (!(this_present_e && that_present_e))
+ return false;
+ if (!this.e.equals(that.e))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateLog_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateLog_result typedOther = (replicateLog_result)other;
+
+ lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetE()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateLog_result(");
+ boolean first = true;
+
+ sb.append("e:");
+ if (this.e == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.e);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateLog_resultStandardSchemeFactory implements SchemeFactory {
+ public replicateLog_resultStandardScheme getScheme() {
+ return new replicateLog_resultStandardScheme();
+ }
+ }
+
+ private static class replicateLog_resultStandardScheme extends StandardScheme<replicateLog_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateLog_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // E
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateLog_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.e != null) {
+ oprot.writeFieldBegin(E_FIELD_DESC);
+ struct.e.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateLog_resultTupleSchemeFactory implements SchemeFactory {
+ public replicateLog_resultTupleScheme getScheme() {
+ return new replicateLog_resultTupleScheme();
+ }
+ }
+
+ private static class replicateLog_resultTupleScheme extends TupleScheme<replicateLog_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetE()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetE()) {
+ struct.e.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateLog_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class replicateKeyValues_args implements org.apache.thrift.TBase<replicateKeyValues_args, replicateKeyValues_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_args");
+
+ private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
+ private static final org.apache.thrift.protocol.TField DATA_FIELD_DESC = new org.apache.thrift.protocol.TField("data", org.apache.thrift.protocol.TType.STRUCT, (short)2);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateKeyValues_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateKeyValues_argsTupleSchemeFactory());
+ }
+
+ public int remoteTableId; // required
+ public KeyValues data; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ REMOTE_TABLE_ID((short)1, "remoteTableId"),
+ DATA((short)2, "data");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // REMOTE_TABLE_ID
+ return REMOTE_TABLE_ID;
+ case 2: // DATA
+ return DATA;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __REMOTETABLEID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ tmpMap.put(_Fields.DATA, new org.apache.thrift.meta_data.FieldMetaData("data", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.StructMetaData(org.apache.thrift.protocol.TType.STRUCT, KeyValues.class)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_args.class, metaDataMap);
+ }
+
+ public replicateKeyValues_args() {
+ }
+
+ public replicateKeyValues_args(
+ int remoteTableId,
+ KeyValues data)
+ {
+ this();
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ this.data = data;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateKeyValues_args(replicateKeyValues_args other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.remoteTableId = other.remoteTableId;
+ if (other.isSetData()) {
+ this.data = new KeyValues(other.data);
+ }
+ }
+
+ public replicateKeyValues_args deepCopy() {
+ return new replicateKeyValues_args(this);
+ }
+
+ @Override
+ public void clear() {
+ setRemoteTableIdIsSet(false);
+ this.remoteTableId = 0;
+ this.data = null;
+ }
+
+ public int getRemoteTableId() {
+ return this.remoteTableId;
+ }
+
+ public replicateKeyValues_args setRemoteTableId(int remoteTableId) {
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ return this;
+ }
+
+ public void unsetRemoteTableId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
+ public boolean isSetRemoteTableId() {
+ return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ public void setRemoteTableIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
+ }
+
+ public KeyValues getData() {
+ return this.data;
+ }
+
+ public replicateKeyValues_args setData(KeyValues data) {
+ this.data = data;
+ return this;
+ }
+
+ public void unsetData() {
+ this.data = null;
+ }
+
+ /** Returns true if field data is set (has been assigned a value) and false otherwise */
+ public boolean isSetData() {
+ return this.data != null;
+ }
+
+ public void setDataIsSet(boolean value) {
+ if (!value) {
+ this.data = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ if (value == null) {
+ unsetRemoteTableId();
+ } else {
+ setRemoteTableId((Integer)value);
+ }
+ break;
+
+ case DATA:
+ if (value == null) {
+ unsetData();
+ } else {
+ setData((KeyValues)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return Integer.valueOf(getRemoteTableId());
+
+ case DATA:
+ return getData();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return isSetRemoteTableId();
+ case DATA:
+ return isSetData();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateKeyValues_args)
+ return this.equals((replicateKeyValues_args)that);
+ return false;
+ }
+
+ public boolean equals(replicateKeyValues_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_remoteTableId = true;
+ boolean that_present_remoteTableId = true;
+ if (this_present_remoteTableId || that_present_remoteTableId) {
+ if (!(this_present_remoteTableId && that_present_remoteTableId))
+ return false;
+ if (this.remoteTableId != that.remoteTableId)
+ return false;
+ }
+
+ boolean this_present_data = true && this.isSetData();
+ boolean that_present_data = true && that.isSetData();
+ if (this_present_data || that_present_data) {
+ if (!(this_present_data && that_present_data))
+ return false;
+ if (!this.data.equals(that.data))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateKeyValues_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateKeyValues_args typedOther = (replicateKeyValues_args)other;
+
+ lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRemoteTableId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetData()).compareTo(typedOther.isSetData());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetData()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.data, typedOther.data);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateKeyValues_args(");
+ boolean first = true;
+
+ sb.append("remoteTableId:");
+ sb.append(this.remoteTableId);
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("data:");
+ if (this.data == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.data);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ if (data != null) {
+ data.validate();
+ }
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateKeyValues_argsStandardSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_argsStandardScheme getScheme() {
+ return new replicateKeyValues_argsStandardScheme();
+ }
+ }
+
+ private static class replicateKeyValues_argsStandardScheme extends StandardScheme<replicateKeyValues_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // REMOTE_TABLE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 2: // DATA
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.data = new KeyValues();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
+ oprot.writeI32(struct.remoteTableId);
+ oprot.writeFieldEnd();
+ if (struct.data != null) {
+ oprot.writeFieldBegin(DATA_FIELD_DESC);
+ struct.data.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateKeyValues_argsTupleSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_argsTupleScheme getScheme() {
+ return new replicateKeyValues_argsTupleScheme();
+ }
+ }
+
+ private static class replicateKeyValues_argsTupleScheme extends TupleScheme<replicateKeyValues_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRemoteTableId()) {
+ optionals.set(0);
+ }
+ if (struct.isSetData()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetRemoteTableId()) {
+ oprot.writeI32(struct.remoteTableId);
+ }
+ if (struct.isSetData()) {
+ struct.data.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.data = new KeyValues();
+ struct.data.read(iprot);
+ struct.setDataIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class replicateKeyValues_result implements org.apache.thrift.TBase<replicateKeyValues_result, replicateKeyValues_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_result");
+
+ private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new replicateKeyValues_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new replicateKeyValues_resultTupleSchemeFactory());
+ }
+
+ public RemoteReplicationException e; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ E((short)1, "e");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // E
+ return E;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(replicateKeyValues_result.class, metaDataMap);
+ }
+
+ public replicateKeyValues_result() {
+ }
+
+ public replicateKeyValues_result(
+ RemoteReplicationException e)
+ {
+ this();
+ this.e = e;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public replicateKeyValues_result(replicateKeyValues_result other) {
+ if (other.isSetE()) {
+ this.e = new RemoteReplicationException(other.e);
+ }
+ }
+
+ public replicateKeyValues_result deepCopy() {
+ return new replicateKeyValues_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.e = null;
+ }
+
+ public RemoteReplicationException getE() {
+ return this.e;
+ }
+
+ public replicateKeyValues_result setE(RemoteReplicationException e) {
+ this.e = e;
+ return this;
+ }
+
+ public void unsetE() {
+ this.e = null;
+ }
+
+ /** Returns true if field e is set (has been assigned a value) and false otherwise */
+ public boolean isSetE() {
+ return this.e != null;
+ }
+
+ public void setEIsSet(boolean value) {
+ if (!value) {
+ this.e = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case E:
+ if (value == null) {
+ unsetE();
+ } else {
+ setE((RemoteReplicationException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case E:
+ return getE();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case E:
+ return isSetE();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof replicateKeyValues_result)
+ return this.equals((replicateKeyValues_result)that);
+ return false;
+ }
+
+ public boolean equals(replicateKeyValues_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_e = true && this.isSetE();
+ boolean that_present_e = true && that.isSetE();
+ if (this_present_e || that_present_e) {
+ if (!(this_present_e && that_present_e))
+ return false;
+ if (!this.e.equals(that.e))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(replicateKeyValues_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ replicateKeyValues_result typedOther = (replicateKeyValues_result)other;
+
+ lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetE()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("replicateKeyValues_result(");
+ boolean first = true;
+
+ sb.append("e:");
+ if (this.e == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.e);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class replicateKeyValues_resultStandardSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_resultStandardScheme getScheme() {
+ return new replicateKeyValues_resultStandardScheme();
+ }
+ }
+
+ private static class replicateKeyValues_resultStandardScheme extends StandardScheme<replicateKeyValues_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // E
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.e != null) {
+ oprot.writeFieldBegin(E_FIELD_DESC);
+ struct.e.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class replicateKeyValues_resultTupleSchemeFactory implements SchemeFactory {
+ public replicateKeyValues_resultTupleScheme getScheme() {
+ return new replicateKeyValues_resultTupleScheme();
+ }
+ }
+
+ private static class replicateKeyValues_resultTupleScheme extends TupleScheme<replicateKeyValues_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetE()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetE()) {
+ struct.e.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, replicateKeyValues_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.e = new RemoteReplicationException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ }
+ }
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/core/src/main/thrift/replication.thrift
----------------------------------------------------------------------
diff --git a/core/src/main/thrift/replication.thrift b/core/src/main/thrift/replication.thrift
index a71d200..481daa9 100644
--- a/core/src/main/thrift/replication.thrift
+++ b/core/src/main/thrift/replication.thrift
@@ -28,12 +28,21 @@ struct KeyValues {
1:list<data.TKeyValue> keyValues
}
+exception NoServersAvailableException {
+ 1:i32 code,
+ 2:string reason
+}
+
exception RemoteReplicationException {
1:i32 code,
2:string reason
}
-service RemoteReplication {
+service RemoteReplicationCoordinator {
+ string getServicerAddress(1:i32 remoteTableId) throws (1:NoServersAvailableException e),
+}
+
+service RemoteReplicationServicer {
void replicateLog(1:i32 remoteTableId, 2:WalEdits data) throws (1:RemoteReplicationException e),
void replicateKeyValues(1:i32 remoteTableId, 2:KeyValues data) throws (1:RemoteReplicationException e)
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index c4408e2..db7507e 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -967,6 +967,8 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
throw new IOException(e);
}
+
+
Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(new MasterClientServiceHandler(this)));
ServerAddress sa = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_CLIENTPORT, processor, "Master",
"Master Client Service Handler", null, Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java b/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
new file mode 100644
index 0000000..0497892
--- /dev/null
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/MasterReplicationCoordinator.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
+import java.util.Iterator;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.replication.thrift.NoServersAvailableException;
+import org.apache.accumulo.core.replication.thrift.RemoteReplicationCoordinator;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.thrift.TException;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Choose a tserver to service a replication task
+ */
+public class MasterReplicationCoordinator implements RemoteReplicationCoordinator.Iface {
+
+ public static enum NoServersAvailable {
+ NO_ONLINE_SERVERS
+ }
+
+ private final Master master;
+ private final AccumuloConfiguration conf;
+ private final Random rand;
+ private final int port;
+
+ public MasterReplicationCoordinator(Master master, AccumuloConfiguration conf) {
+ this.master = master;
+ this.conf = conf;
+ this.port = conf.getPort(Property.REPLICATION_RECEIPT_SERVICE_PORT);
+ this.rand = new Random(358923462l);
+ }
+
+ @Override
+ public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, TException {
+ Set<TServerInstance> tservers = master.onlineTabletServers();
+ if (tservers.isEmpty()) {
+ throw new NoServersAvailableException(NoServersAvailable.NO_ONLINE_SERVERS.ordinal(), "No tservers are online");
+ }
+
+ TServerInstance tserver = getRandomTServer(tservers, rand.nextInt(tservers.size()));
+ return tserver.host() + ":" + port;
+ }
+
+ protected TServerInstance getRandomTServer(Set<TServerInstance> tservers, int offset) {
+ Preconditions.checkArgument(tservers.size() > offset, "Must provide an offset less than the size of the set");
+ Iterator<TServerInstance> iter = tservers.iterator();
+ while (offset > 0) {
+ iter.next();
+ offset--;
+ }
+
+ return iter.next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3e21a41d/server/master/src/test/java/org/apache/accumulo/master/replication/MasterReplicationCoordinatorTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/MasterReplicationCoordinatorTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/MasterReplicationCoordinatorTest.java
new file mode 100644
index 0000000..9cb196c
--- /dev/null
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/MasterReplicationCoordinatorTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
+import java.util.Collections;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.net.HostAndPort;
+
+/**
+ *
+ */
+public class MasterReplicationCoordinatorTest {
+
+ @Test
+ public void randomServer() {
+ Master master = EasyMock.createMock(Master.class);
+ AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
+ MasterReplicationCoordinator coordinator = new MasterReplicationCoordinator(master, conf);
+ TServerInstance inst1 = new TServerInstance(HostAndPort.fromParts("host1", 1234), "session");
+
+ Assert.assertEquals(inst1, coordinator.getRandomTServer(Collections.singleton(inst1), 0));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void invalidOffset() {
+ Master master = EasyMock.createMock(Master.class);
+ AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
+ MasterReplicationCoordinator coordinator = new MasterReplicationCoordinator(master, conf);
+ TServerInstance inst1 = new TServerInstance(HostAndPort.fromParts("host1", 1234), "session");
+
+ Assert.assertEquals(inst1, coordinator.getRandomTServer(Collections.singleton(inst1), 1));
+ }
+
+ @Test
+ public void randomServerFromMany() {
+ Master master = EasyMock.createMock(Master.class);
+ AccumuloConfiguration conf = EasyMock.createMock(AccumuloConfiguration.class);
+ MasterReplicationCoordinator coordinator = new MasterReplicationCoordinator(master, conf);
+ TreeSet<TServerInstance> instances = new TreeSet<>();
+ TServerInstance inst1 = new TServerInstance(HostAndPort.fromParts("host1", 1234), "session");
+ instances.add(inst1);
+ TServerInstance inst2 = new TServerInstance(HostAndPort.fromParts("host2", 1234), "session");
+ instances.add(inst2);
+
+ Assert.assertEquals(inst1, coordinator.getRandomTServer(instances, 0));
+ Assert.assertEquals(inst2, coordinator.getRandomTServer(instances, 1));
+ }
+}
[48/50] [abbrv] git commit: ACCUMULO-2583 Advertise peer master
coordinator service port in ZK.
Posted by el...@apache.org.
ACCUMULO-2583 Advertise peer master coordinator service port in ZK.
Need to use the proper contact info for the peer master. Some more configuration
Properties for tweaking things. Better logging.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/e84879c8
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/e84879c8
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/e84879c8
Branch: refs/heads/ACCUMULO-378
Commit: e84879c8dd814e1f00c9b109f095a45015224c7f
Parents: 53fc90f
Author: Josh Elser <el...@apache.org>
Authored: Thu May 8 21:56:45 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 8 21:56:45 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 1 +
.../core/client/impl/ReplicationClient.java | 39 +++++++++++++++++---
.../org/apache/accumulo/core/conf/Property.java | 10 ++++-
.../java/org/apache/accumulo/master/Master.java | 16 +++++++-
.../replication/ReplicationWorkAssigner.java | 7 +++-
.../apache/accumulo/tserver/TabletServer.java | 2 +-
.../replication/AccumuloReplicaSystem.java | 20 +++++++---
.../test/replication/ReplicationIT.java | 34 +++++++++++------
8 files changed, 100 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java
index c87119c..f230690 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -43,6 +43,7 @@ public class Constants {
public static final String ZMASTERS = "/masters";
public static final String ZMASTER_LOCK = ZMASTERS + "/lock";
public static final String ZMASTER_GOAL_STATE = ZMASTERS + "/goal_state";
+ public static final String ZMASTER_REPLICATION_COORDINATOR_PORT = ZMASTERS + "/repl_coord_port";
public static final String ZGC = "/gc";
public static final String ZGC_LOCK = ZGC + "/lock";
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
index df12ae8..65565f8 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
@@ -20,22 +20,29 @@ import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
import java.util.List;
+import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
-import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooReader;
import org.apache.thrift.TServiceClient;
import org.apache.thrift.transport.TTransportException;
+import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.net.HostAndPort;
+
public class ReplicationClient {
private static final Logger log = LoggerFactory.getLogger(ReplicationClient.class);
@@ -65,21 +72,41 @@ public class ReplicationClient {
return null;
}
- String master = locations.get(0);
- if (master.endsWith(":0"))
+ // This is the master thrift service, we just want the hostname, not the port
+ String masterThriftService = locations.get(0);
+ if (masterThriftService.endsWith(":0"))
+ return null;
+
+
+ AccumuloConfiguration conf = ServerConfigurationUtil.getConfiguration(instance);
+
+ HostAndPort masterAddr = HostAndPort.fromString(masterThriftService);
+ String zkPath = ZooUtil.getRoot(instance) + Constants.ZMASTER_REPLICATION_COORDINATOR_PORT;
+ String replCoordinatorPort;
+
+ // Get the coordinator port for the master we're trying to connect to
+ try {
+ ZooReader reader = new ZooReader(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
+ replCoordinatorPort = new String(reader.getData(zkPath, null), StandardCharsets.UTF_8);
+ } catch (KeeperException | InterruptedException e) {
+ log.debug("Could not fetch remote coordinator port");
return null;
+ }
+
+ // Throw the hostname and port through HostAndPort to get some normalization
+ HostAndPort coordinatorAddr = HostAndPort.fromParts(masterAddr.getHostText(), Integer.parseInt(replCoordinatorPort));
try {
// Master requests can take a long time: don't ever time out
- ReplicationCoordinator.Client client = ThriftUtil.getClientNoTimeout(new ReplicationCoordinator.Client.Factory(), master,
- ServerConfigurationUtil.getConfiguration(instance));
+ ReplicationCoordinator.Client client = ThriftUtil.getClientNoTimeout(new ReplicationCoordinator.Client.Factory(), coordinatorAddr.toString(),
+ conf);
return client;
} catch (TTransportException tte) {
if (tte.getCause().getClass().equals(UnknownHostException.class)) {
// do not expect to recover from this
throw new RuntimeException(tte);
}
- log.debug("Failed to connect to master={}, will retry... ", master, tte);
+ log.debug("Failed to connect to master coordinator service ({}), will retry... ", coordinatorAddr.toString(), tte);
return null;
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index d611bd5..a87b1b4 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -182,6 +182,12 @@ public enum Property {
@Experimental
MASTER_REPLICATION_SCAN_INTERVAL("master.replication.status.scan.interval", "30s", PropertyType.TIMEDURATION,
"Amount of time to sleep before scanning the status section of the replication table for new data"),
+ @Experimental
+ MASTER_REPLICATION_COORDINATOR_PORT("master.replication.coordinator.port", "10001", PropertyType.PORT, "Port for the replication coordinator service"),
+ @Experimental
+ MASTER_REPLICATION_COORDINATOR_MINTHREADS("master.replication.coordinator.minthreads", "4", PropertyType.COUNT, "Minimum number of threads dedicated to answering coordinator requests"),
+ @Experimental
+ MASTER_REPLICATION_COORDINATOR_THREADCHECK("master.replication.coordinator.threadcheck.time", "5s", PropertyType.TIMEDURATION, "The time between adjustments of the coordinator thread pool"),
// properties that are specific to tablet server behavior
TSERV_PREFIX("tserver.", null, PropertyType.PREFIX, "Properties in this category affect the behavior of the tablet servers"),
@@ -457,13 +463,15 @@ public enum Property {
@Experimental
REPLICATION_WORKER_THREADS("replication.worker.threads", "4", PropertyType.COUNT, "Size of the threadpool that each tabletserver devotes to replicating data"),
@Experimental
- REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "10001", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
+ REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "10002", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
@Experimental
REPLICATION_WORK_ATTEMPTS("replication.work.attempts", "10", PropertyType.COUNT, "Number of attempts to try to replicate some data before giving up and letting it naturally be retried later"),
@Experimental
REPLICATION_MIN_THREADS("replication.receiver.min.threads", "1", PropertyType.COUNT, "Minimum number of threads for replciation"),
@Experimental
REPLICATION_THREADCHECK("replication.receiver.threadcheck.time", "5s", PropertyType.TIMEDURATION, "The time between adjustments of the replication thread pool."),
+ @Experimental
+ REPLICATION_MAX_UNIT_SIZE("replication.max.unit.size", "64M", PropertyType.MEMORY, "Maximum size of data to send in a replication message"),
;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index 0eac9ab..b0745eb 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -60,6 +60,7 @@ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.security.NamespacePermission;
@@ -76,6 +77,7 @@ import org.apache.accumulo.fate.zookeeper.ZooLock.LockLossReason;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.master.recovery.RecoveryManager;
+import org.apache.accumulo.master.replication.MasterReplicationCoordinator;
import org.apache.accumulo.master.replication.ReplicationDriver;
import org.apache.accumulo.master.replication.ReplicationWorkAssigner;
import org.apache.accumulo.master.state.TableCounts;
@@ -977,8 +979,6 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
throw new IOException(e);
}
-
-
Processor<Iface> processor = new Processor<Iface>(TraceWrap.service(new MasterClientServiceHandler(this)));
ServerAddress sa = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_CLIENTPORT, processor, "Master",
"Master Client Service Handler", null, Property.MASTER_MINTHREADS, Property.MASTER_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
@@ -990,6 +990,17 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
while (!clientService.isServing()) {
UtilWaitThread.sleep(100);
}
+
+ // Start the replication coordinator which assigns tservers to service replication requests
+ ReplicationCoordinator.Processor<ReplicationCoordinator.Iface> replicationCoordinatorProcessor = new ReplicationCoordinator.Processor<ReplicationCoordinator.Iface>(
+ TraceWrap.service(new MasterReplicationCoordinator(this, getSystemConfiguration())));
+ ServerAddress replAddress = TServerUtils.startServer(getSystemConfiguration(), hostname, Property.MASTER_REPLICATION_COORDINATOR_PORT, replicationCoordinatorProcessor, "Master Replication Coordinator",
+ "Replication Coordinator", null, Property.MASTER_REPLICATION_COORDINATOR_MINTHREADS, Property.MASTER_REPLICATION_COORDINATOR_THREADCHECK, Property.GENERAL_MAX_MESSAGE_SIZE);
+
+ // Advertise that port we used so peers don't have to be told what it is
+ ZooReaderWriter.getInstance().putPersistentData(ZooUtil.getRoot(instance) + Constants.ZMASTER_REPLICATION_COORDINATOR_PORT,
+ Integer.toString(replAddress.address.getPort()).getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
+
while (clientService.isServing()) {
UtilWaitThread.sleep(500);
}
@@ -1000,6 +1011,7 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
statusThread.join(remaining(deadline));
replicationWorkAssigner.join(remaining(deadline));
replicationWorkDriver.join(remaining(deadline));
+ replAddress.server.stop();
// quit, even if the tablet servers somehow jam up and the watchers
// don't stop
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
index 24842a9..1dd20da 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -152,6 +152,8 @@ public class ReplicationWorkAssigner extends Daemon {
@Override
public void run() {
+ log.info("Starting replication work assignment thread");
+
while (master.stillMaster()) {
if (null == conf) {
conf = master.getConfiguration().getConfiguration();
@@ -178,7 +180,9 @@ public class ReplicationWorkAssigner extends Daemon {
// Keep the state of the work we queued correct
cleanupFinishedWork();
- UtilWaitThread.sleep(conf.getTimeInMillis(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP));
+ long sleepTime = conf.getTimeInMillis(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP);
+ log.debug("Sleeping {} ms", sleepTime);
+ UtilWaitThread.sleep(sleepTime);
}
}
@@ -205,6 +209,7 @@ public class ReplicationWorkAssigner extends Daemon {
// to add more work entries
if (queuedWork.size() > maxQueueSize) {
log.warn("Queued replication work exceeds configured maximum ({}), sleeping to allow work to occur", maxQueueSize);
+ UtilWaitThread.sleep(5000);
return;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 803419c..0b5c6bf 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -3122,7 +3122,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
private HostAndPort startReplicationService() throws UnknownHostException {
- ReplicationServicer.Iface repl = TraceWrap.service(new ReplicationServicerHandler());
+ ReplicationServicer.Iface repl = TraceWrap.service(new ReplicationServicerHandler(this));
ReplicationServicer.Processor<ReplicationServicer.Iface> processor = new ReplicationServicer.Processor<ReplicationServicer.Iface>(repl);
AccumuloConfiguration conf = getSystemConfiguration();
Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null ? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
index 40676f7..f275cdb 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
@@ -131,21 +131,29 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
Long entriesReplicated;
//TODO should chunk up the given file into some configurable sizes instead of just sending the entire file all at once
// configuration should probably just be size based.
- final long sizeLimit = Long.MAX_VALUE;
+ final long sizeLimit = conf.getMemoryInBytes(Property.REPLICATION_MAX_UNIT_SIZE);
try {
entriesReplicated = ReplicationClient.executeServicerWithReturn(peerInstance, peerTserver, new ClientExecReturn<Long,ReplicationServicer.Client>() {
@Override
public Long execute(Client client) throws Exception {
// RFiles have an extension, call everything else a WAL
if (p.getName().endsWith(RFILE_SUFFIX)) {
- return client.replicateKeyValues(remoteTableId, getKeyValues(p, status, sizeLimit));
+ KeyValues kvs = getKeyValues(p, status, sizeLimit);
+ if (0 < kvs.getKeyValuesSize()) {
+ return client.replicateKeyValues(remoteTableId, kvs);
+ }
} else {
- return client.replicateLog(remoteTableId, getWalEdits(p, status, sizeLimit));
+ WalEdits edits = getWalEdits(p, status, sizeLimit);
+ if (0 < edits.getEditsSize()) {
+ return client.replicateLog(remoteTableId, edits);
+ }
}
+
+ return 0l;
}
});
- log.debug("Replicated {} entries from {} to {} which is a part of {}", entriesReplicated, p, peerTserver, peerInstance.getInstanceName());
+ log.debug("Replicated {} entries from {} to {} which is a member of the peer '{}'", entriesReplicated, p, peerTserver, peerInstance.getInstanceName());
// Update the begin to account for what we replicated
Status updatedStatus = Status.newBuilder(status).setBegin(status.getBegin() + entriesReplicated).build();
@@ -195,7 +203,7 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
key.readFields(wal);
value.readFields(wal);
} catch (EOFException e) {
- log.trace("Caught EOFException, no more data to replicate");
+ log.debug("Caught EOFException, no more data to replicate");
break;
}
@@ -217,7 +225,7 @@ public class AccumuloReplicaSystem implements ReplicaSystem {
}
}
- log.debug("Returning {} bytes of WAL entries for replication for {}", size, p);
+ log.debug("Binned {} bytes of WAL entries for replication to peer '{}'", size, p);
return edits;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/e84879c8/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
index 0d4099c..0ad8066 100644
--- a/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/replication/ReplicationIT.java
@@ -33,9 +33,12 @@ import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.test.functional.ConfigurableMacIT;
import org.apache.accumulo.tserver.replication.AccumuloReplicaSystem;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.protobuf.TextFormat;
@@ -43,13 +46,16 @@ import com.google.protobuf.TextFormat;
*
*/
public class ReplicationIT extends ConfigurableMacIT {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationIT.class);
@Override
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setNumTservers(1);
- cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "1M");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "32M");
cfg.setProperty(Property.GC_CYCLE_START, "1s");
- cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
+ cfg.setProperty(Property.GC_CYCLE_DELAY, "5s");
+ cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "5s");
+ cfg.setProperty(Property.REPLICATION_MAX_UNIT_SIZE, "8M");
hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName());
}
@@ -59,7 +65,8 @@ public class ReplicationIT extends ConfigurableMacIT {
ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
- peerCfg.setProperty(Property.REPLICATION_RECEIPT_SERVICE_PORT, "10002");
+ peerCfg.setProperty(Property.MASTER_REPLICATION_COORDINATOR_PORT, "10003");
+ peerCfg.setProperty(Property.REPLICATION_RECEIPT_SERVICE_PORT, "10004");
MiniAccumuloClusterImpl peerCluster = peerCfg.build();
peerCluster.start();
@@ -89,13 +96,15 @@ public class ReplicationIT extends ConfigurableMacIT {
// Write some data to table1
BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
- for (int rows = 0; rows < 250; rows++) {
- Mutation m = new Mutation(Integer.toString(rows));
- for (int cols = 0; cols < 500; cols++) {
- String value = Integer.toString(cols);
- m.put(value, "", value);
+ for (int i = 0; i < 100; i++) {
+ for (int rows = 0; rows < 1000; rows++) {
+ Mutation m = new Mutation(i + "_" + Integer.toString(rows));
+ for (int cols = 0; cols < 400; cols++) {
+ String value = Integer.toString(cols);
+ m.put(value, "", value);
+ }
+ bw.addMutation(m);
}
- bw.addMutation(m);
}
bw.close();
@@ -104,17 +113,18 @@ public class ReplicationIT extends ConfigurableMacIT {
Thread.sleep(500);
}
+ connMaster.tableOperations().compact(masterTable, null, null, true, false);
for (int i = 0; i < 10; i++) {
-
Scanner s = ReplicationTable.getScanner(connMaster);
for (Entry<Key,Value> e : s) {
- log.info(e.getKey().toStringNoTruncate() + " " + TextFormat.shortDebugString(Status.parseFrom(e.getValue().get())));
+ Path p = new Path(e.getKey().getRow().toString());
+ log.info(p.getName() + " " + e.getKey().getColumnFamily() + " " + e.getKey().getColumnQualifier() + " " + TextFormat.shortDebugString(Status.parseFrom(e.getValue().get())));
}
log.info("");
log.info("");
- Thread.sleep(1000);
+ Thread.sleep(3000);
}
peerCluster.stop();
[30/50] [abbrv] ACCUMULO-2583 First stab at setting up the actual
wire transfer to the replication peer
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationCoordinator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationCoordinator.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationCoordinator.java
new file mode 100644
index 0000000..14729eb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationCoordinator.java
@@ -0,0 +1,1010 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Autogenerated by Thrift Compiler (0.9.0)
+ *
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ * @generated
+ */
+package org.apache.accumulo.core.replication.thrift;
+
+import org.apache.thrift.scheme.IScheme;
+import org.apache.thrift.scheme.SchemeFactory;
+import org.apache.thrift.scheme.StandardScheme;
+
+import org.apache.thrift.scheme.TupleScheme;
+import org.apache.thrift.protocol.TTupleProtocol;
+import org.apache.thrift.protocol.TProtocolException;
+import org.apache.thrift.EncodingUtils;
+import org.apache.thrift.TException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.EnumMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.EnumSet;
+import java.util.Collections;
+import java.util.BitSet;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@SuppressWarnings("all") public class ReplicationCoordinator {
+
+ public interface Iface {
+
+ public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, org.apache.thrift.TException;
+
+ }
+
+ public interface AsyncIface {
+
+ public void getServicerAddress(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.getServicerAddress_call> resultHandler) throws org.apache.thrift.TException;
+
+ }
+
+ public static class Client extends org.apache.thrift.TServiceClient implements Iface {
+ public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
+ public Factory() {}
+ public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
+ return new Client(prot);
+ }
+ public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ return new Client(iprot, oprot);
+ }
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol prot)
+ {
+ super(prot, prot);
+ }
+
+ public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
+ super(iprot, oprot);
+ }
+
+ public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, org.apache.thrift.TException
+ {
+ send_getServicerAddress(remoteTableId);
+ return recv_getServicerAddress();
+ }
+
+ public void send_getServicerAddress(int remoteTableId) throws org.apache.thrift.TException
+ {
+ getServicerAddress_args args = new getServicerAddress_args();
+ args.setRemoteTableId(remoteTableId);
+ sendBase("getServicerAddress", args);
+ }
+
+ public String recv_getServicerAddress() throws NoServersAvailableException, org.apache.thrift.TException
+ {
+ getServicerAddress_result result = new getServicerAddress_result();
+ receiveBase(result, "getServicerAddress");
+ if (result.isSetSuccess()) {
+ return result.success;
+ }
+ if (result.e != null) {
+ throw result.e;
+ }
+ throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getServicerAddress failed: unknown result");
+ }
+
+ }
+ public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
+ public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
+ private org.apache.thrift.async.TAsyncClientManager clientManager;
+ private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
+ public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
+ this.clientManager = clientManager;
+ this.protocolFactory = protocolFactory;
+ }
+ public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
+ return new AsyncClient(protocolFactory, clientManager, transport);
+ }
+ }
+
+ public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
+ super(protocolFactory, clientManager, transport);
+ }
+
+ public void getServicerAddress(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<getServicerAddress_call> resultHandler) throws org.apache.thrift.TException {
+ checkReady();
+ getServicerAddress_call method_call = new getServicerAddress_call(remoteTableId, resultHandler, this, ___protocolFactory, ___transport);
+ this.___currentMethod = method_call;
+ ___manager.call(method_call);
+ }
+
+ public static class getServicerAddress_call extends org.apache.thrift.async.TAsyncMethodCall {
+ private int remoteTableId;
+ public getServicerAddress_call(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<getServicerAddress_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
+ super(client, protocolFactory, transport, resultHandler, false);
+ this.remoteTableId = remoteTableId;
+ }
+
+ public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
+ prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getServicerAddress", org.apache.thrift.protocol.TMessageType.CALL, 0));
+ getServicerAddress_args args = new getServicerAddress_args();
+ args.setRemoteTableId(remoteTableId);
+ args.write(prot);
+ prot.writeMessageEnd();
+ }
+
+ public String getResult() throws NoServersAvailableException, org.apache.thrift.TException {
+ if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
+ throw new IllegalStateException("Method call not finished!");
+ }
+ org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
+ org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
+ return (new Client(prot)).recv_getServicerAddress();
+ }
+ }
+
+ }
+
+ public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
+ private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
+ public Processor(I iface) {
+ super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
+ }
+
+ protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ super(iface, getProcessMap(processMap));
+ }
+
+ private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
+ processMap.put("getServicerAddress", new getServicerAddress());
+ return processMap;
+ }
+
+ public static class getServicerAddress<I extends Iface> extends org.apache.thrift.ProcessFunction<I, getServicerAddress_args> {
+ public getServicerAddress() {
+ super("getServicerAddress");
+ }
+
+ public getServicerAddress_args getEmptyArgsInstance() {
+ return new getServicerAddress_args();
+ }
+
+ protected boolean isOneway() {
+ return false;
+ }
+
+ public getServicerAddress_result getResult(I iface, getServicerAddress_args args) throws org.apache.thrift.TException {
+ getServicerAddress_result result = new getServicerAddress_result();
+ try {
+ result.success = iface.getServicerAddress(args.remoteTableId);
+ } catch (NoServersAvailableException e) {
+ result.e = e;
+ }
+ return result;
+ }
+ }
+
+ }
+
+ public static class getServicerAddress_args implements org.apache.thrift.TBase<getServicerAddress_args, getServicerAddress_args._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getServicerAddress_args");
+
+ private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new getServicerAddress_argsStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new getServicerAddress_argsTupleSchemeFactory());
+ }
+
+ public int remoteTableId; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ REMOTE_TABLE_ID((short)1, "remoteTableId");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 1: // REMOTE_TABLE_ID
+ return REMOTE_TABLE_ID;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ private static final int __REMOTETABLEID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getServicerAddress_args.class, metaDataMap);
+ }
+
+ public getServicerAddress_args() {
+ }
+
+ public getServicerAddress_args(
+ int remoteTableId)
+ {
+ this();
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public getServicerAddress_args(getServicerAddress_args other) {
+ __isset_bitfield = other.__isset_bitfield;
+ this.remoteTableId = other.remoteTableId;
+ }
+
+ public getServicerAddress_args deepCopy() {
+ return new getServicerAddress_args(this);
+ }
+
+ @Override
+ public void clear() {
+ setRemoteTableIdIsSet(false);
+ this.remoteTableId = 0;
+ }
+
+ public int getRemoteTableId() {
+ return this.remoteTableId;
+ }
+
+ public getServicerAddress_args setRemoteTableId(int remoteTableId) {
+ this.remoteTableId = remoteTableId;
+ setRemoteTableIdIsSet(true);
+ return this;
+ }
+
+ public void unsetRemoteTableId() {
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
+ public boolean isSetRemoteTableId() {
+ return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
+ }
+
+ public void setRemoteTableIdIsSet(boolean value) {
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ if (value == null) {
+ unsetRemoteTableId();
+ } else {
+ setRemoteTableId((Integer)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return Integer.valueOf(getRemoteTableId());
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case REMOTE_TABLE_ID:
+ return isSetRemoteTableId();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof getServicerAddress_args)
+ return this.equals((getServicerAddress_args)that);
+ return false;
+ }
+
+ public boolean equals(getServicerAddress_args that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_remoteTableId = true;
+ boolean that_present_remoteTableId = true;
+ if (this_present_remoteTableId || that_present_remoteTableId) {
+ if (!(this_present_remoteTableId && that_present_remoteTableId))
+ return false;
+ if (this.remoteTableId != that.remoteTableId)
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(getServicerAddress_args other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ getServicerAddress_args typedOther = (getServicerAddress_args)other;
+
+ lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetRemoteTableId()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("getServicerAddress_args(");
+ boolean first = true;
+
+ sb.append("remoteTableId:");
+ sb.append(this.remoteTableId);
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class getServicerAddress_argsStandardSchemeFactory implements SchemeFactory {
+ public getServicerAddress_argsStandardScheme getScheme() {
+ return new getServicerAddress_argsStandardScheme();
+ }
+ }
+
+ private static class getServicerAddress_argsStandardScheme extends StandardScheme<getServicerAddress_args> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 1: // REMOTE_TABLE_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
+ oprot.writeI32(struct.remoteTableId);
+ oprot.writeFieldEnd();
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class getServicerAddress_argsTupleSchemeFactory implements SchemeFactory {
+ public getServicerAddress_argsTupleScheme getScheme() {
+ return new getServicerAddress_argsTupleScheme();
+ }
+ }
+
+ private static class getServicerAddress_argsTupleScheme extends TupleScheme<getServicerAddress_args> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetRemoteTableId()) {
+ optionals.set(0);
+ }
+ oprot.writeBitSet(optionals, 1);
+ if (struct.isSetRemoteTableId()) {
+ oprot.writeI32(struct.remoteTableId);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_args struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(1);
+ if (incoming.get(0)) {
+ struct.remoteTableId = iprot.readI32();
+ struct.setRemoteTableIdIsSet(true);
+ }
+ }
+ }
+
+ }
+
+ public static class getServicerAddress_result implements org.apache.thrift.TBase<getServicerAddress_result, getServicerAddress_result._Fields>, java.io.Serializable, Cloneable {
+ private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getServicerAddress_result");
+
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
+
+ private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
+ static {
+ schemes.put(StandardScheme.class, new getServicerAddress_resultStandardSchemeFactory());
+ schemes.put(TupleScheme.class, new getServicerAddress_resultTupleSchemeFactory());
+ }
+
+ public String success; // required
+ public NoServersAvailableException e; // required
+
+ /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
+ @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
+ SUCCESS((short)0, "success"),
+ E((short)1, "e");
+
+ private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
+
+ static {
+ for (_Fields field : EnumSet.allOf(_Fields.class)) {
+ byName.put(field.getFieldName(), field);
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, or null if its not found.
+ */
+ public static _Fields findByThriftId(int fieldId) {
+ switch(fieldId) {
+ case 0: // SUCCESS
+ return SUCCESS;
+ case 1: // E
+ return E;
+ default:
+ return null;
+ }
+ }
+
+ /**
+ * Find the _Fields constant that matches fieldId, throwing an exception
+ * if it is not found.
+ */
+ public static _Fields findByThriftIdOrThrow(int fieldId) {
+ _Fields fields = findByThriftId(fieldId);
+ if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
+ return fields;
+ }
+
+ /**
+ * Find the _Fields constant that matches name, or null if its not found.
+ */
+ public static _Fields findByName(String name) {
+ return byName.get(name);
+ }
+
+ private final short _thriftId;
+ private final String _fieldName;
+
+ _Fields(short thriftId, String fieldName) {
+ _thriftId = thriftId;
+ _fieldName = fieldName;
+ }
+
+ public short getThriftFieldId() {
+ return _thriftId;
+ }
+
+ public String getFieldName() {
+ return _fieldName;
+ }
+ }
+
+ // isset id assignments
+ public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
+ static {
+ Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
+ tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
+ tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
+ metaDataMap = Collections.unmodifiableMap(tmpMap);
+ org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getServicerAddress_result.class, metaDataMap);
+ }
+
+ public getServicerAddress_result() {
+ }
+
+ public getServicerAddress_result(
+ String success,
+ NoServersAvailableException e)
+ {
+ this();
+ this.success = success;
+ this.e = e;
+ }
+
+ /**
+ * Performs a deep copy on <i>other</i>.
+ */
+ public getServicerAddress_result(getServicerAddress_result other) {
+ if (other.isSetSuccess()) {
+ this.success = other.success;
+ }
+ if (other.isSetE()) {
+ this.e = new NoServersAvailableException(other.e);
+ }
+ }
+
+ public getServicerAddress_result deepCopy() {
+ return new getServicerAddress_result(this);
+ }
+
+ @Override
+ public void clear() {
+ this.success = null;
+ this.e = null;
+ }
+
+ public String getSuccess() {
+ return this.success;
+ }
+
+ public getServicerAddress_result setSuccess(String success) {
+ this.success = success;
+ return this;
+ }
+
+ public void unsetSuccess() {
+ this.success = null;
+ }
+
+ /** Returns true if field success is set (has been assigned a value) and false otherwise */
+ public boolean isSetSuccess() {
+ return this.success != null;
+ }
+
+ public void setSuccessIsSet(boolean value) {
+ if (!value) {
+ this.success = null;
+ }
+ }
+
+ public NoServersAvailableException getE() {
+ return this.e;
+ }
+
+ public getServicerAddress_result setE(NoServersAvailableException e) {
+ this.e = e;
+ return this;
+ }
+
+ public void unsetE() {
+ this.e = null;
+ }
+
+ /** Returns true if field e is set (has been assigned a value) and false otherwise */
+ public boolean isSetE() {
+ return this.e != null;
+ }
+
+ public void setEIsSet(boolean value) {
+ if (!value) {
+ this.e = null;
+ }
+ }
+
+ public void setFieldValue(_Fields field, Object value) {
+ switch (field) {
+ case SUCCESS:
+ if (value == null) {
+ unsetSuccess();
+ } else {
+ setSuccess((String)value);
+ }
+ break;
+
+ case E:
+ if (value == null) {
+ unsetE();
+ } else {
+ setE((NoServersAvailableException)value);
+ }
+ break;
+
+ }
+ }
+
+ public Object getFieldValue(_Fields field) {
+ switch (field) {
+ case SUCCESS:
+ return getSuccess();
+
+ case E:
+ return getE();
+
+ }
+ throw new IllegalStateException();
+ }
+
+ /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
+ public boolean isSet(_Fields field) {
+ if (field == null) {
+ throw new IllegalArgumentException();
+ }
+
+ switch (field) {
+ case SUCCESS:
+ return isSetSuccess();
+ case E:
+ return isSetE();
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == null)
+ return false;
+ if (that instanceof getServicerAddress_result)
+ return this.equals((getServicerAddress_result)that);
+ return false;
+ }
+
+ public boolean equals(getServicerAddress_result that) {
+ if (that == null)
+ return false;
+
+ boolean this_present_success = true && this.isSetSuccess();
+ boolean that_present_success = true && that.isSetSuccess();
+ if (this_present_success || that_present_success) {
+ if (!(this_present_success && that_present_success))
+ return false;
+ if (!this.success.equals(that.success))
+ return false;
+ }
+
+ boolean this_present_e = true && this.isSetE();
+ boolean that_present_e = true && that.isSetE();
+ if (this_present_e || that_present_e) {
+ if (!(this_present_e && that_present_e))
+ return false;
+ if (!this.e.equals(that.e))
+ return false;
+ }
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return 0;
+ }
+
+ public int compareTo(getServicerAddress_result other) {
+ if (!getClass().equals(other.getClass())) {
+ return getClass().getName().compareTo(other.getClass().getName());
+ }
+
+ int lastComparison = 0;
+ getServicerAddress_result typedOther = (getServicerAddress_result)other;
+
+ lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetSuccess()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetE()) {
+ lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
+ return 0;
+ }
+
+ public _Fields fieldForId(int fieldId) {
+ return _Fields.findByThriftId(fieldId);
+ }
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
+ schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
+ schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder("getServicerAddress_result(");
+ boolean first = true;
+
+ sb.append("success:");
+ if (this.success == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.success);
+ }
+ first = false;
+ if (!first) sb.append(", ");
+ sb.append("e:");
+ if (this.e == null) {
+ sb.append("null");
+ } else {
+ sb.append(this.e);
+ }
+ first = false;
+ sb.append(")");
+ return sb.toString();
+ }
+
+ public void validate() throws org.apache.thrift.TException {
+ // check for required fields
+ // check for sub-struct validity
+ }
+
+ private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
+ try {
+ write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
+ try {
+ read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
+ } catch (org.apache.thrift.TException te) {
+ throw new java.io.IOException(te);
+ }
+ }
+
+ private static class getServicerAddress_resultStandardSchemeFactory implements SchemeFactory {
+ public getServicerAddress_resultStandardScheme getScheme() {
+ return new getServicerAddress_resultStandardScheme();
+ }
+ }
+
+ private static class getServicerAddress_resultStandardScheme extends StandardScheme<getServicerAddress_result> {
+
+ public void read(org.apache.thrift.protocol.TProtocol iprot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ org.apache.thrift.protocol.TField schemeField;
+ iprot.readStructBegin();
+ while (true)
+ {
+ schemeField = iprot.readFieldBegin();
+ if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
+ break;
+ }
+ switch (schemeField.id) {
+ case 0: // SUCCESS
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
+ struct.success = iprot.readString();
+ struct.setSuccessIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ case 1: // E
+ if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
+ struct.e = new NoServersAvailableException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ break;
+ default:
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
+ }
+ iprot.readFieldEnd();
+ }
+ iprot.readStructEnd();
+
+ // check for required fields of primitive type, which can't be checked in the validate method
+ struct.validate();
+ }
+
+ public void write(org.apache.thrift.protocol.TProtocol oprot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ struct.validate();
+
+ oprot.writeStructBegin(STRUCT_DESC);
+ if (struct.success != null) {
+ oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
+ oprot.writeString(struct.success);
+ oprot.writeFieldEnd();
+ }
+ if (struct.e != null) {
+ oprot.writeFieldBegin(E_FIELD_DESC);
+ struct.e.write(oprot);
+ oprot.writeFieldEnd();
+ }
+ oprot.writeFieldStop();
+ oprot.writeStructEnd();
+ }
+
+ }
+
+ private static class getServicerAddress_resultTupleSchemeFactory implements SchemeFactory {
+ public getServicerAddress_resultTupleScheme getScheme() {
+ return new getServicerAddress_resultTupleScheme();
+ }
+ }
+
+ private static class getServicerAddress_resultTupleScheme extends TupleScheme<getServicerAddress_result> {
+
+ @Override
+ public void write(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol oprot = (TTupleProtocol) prot;
+ BitSet optionals = new BitSet();
+ if (struct.isSetSuccess()) {
+ optionals.set(0);
+ }
+ if (struct.isSetE()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
+ if (struct.isSetSuccess()) {
+ oprot.writeString(struct.success);
+ }
+ if (struct.isSetE()) {
+ struct.e.write(oprot);
+ }
+ }
+
+ @Override
+ public void read(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_result struct) throws org.apache.thrift.TException {
+ TTupleProtocol iprot = (TTupleProtocol) prot;
+ BitSet incoming = iprot.readBitSet(2);
+ if (incoming.get(0)) {
+ struct.success = iprot.readString();
+ struct.setSuccessIsSet(true);
+ }
+ if (incoming.get(1)) {
+ struct.e = new NoServersAvailableException();
+ struct.e.read(iprot);
+ struct.setEIsSet(true);
+ }
+ }
+ }
+
+ }
+
+}
[04/50] [abbrv] git commit: ACCUMULO-2765 Convert FrameworkTest to
junit4
Posted by el...@apache.org.
ACCUMULO-2765 Convert FrameworkTest to junit4
Use JUnit 4
To enable new features
Don't catch exceptions
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1c38957c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1c38957c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1c38957c
Branch: refs/heads/ACCUMULO-378
Commit: 1c38957c453df84462a2cd837b155d590d012d05
Parents: b3c8936
Author: Mike Drob <md...@cloudera.com>
Authored: Wed Apr 30 15:29:18 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Wed Apr 30 15:29:18 2014 -0400
----------------------------------------------------------------------
.../accumulo/test/randomwalk/FrameworkTest.java | 48 ++++++++++----------
1 file changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/1c38957c/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java b/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
index d43347e..ca003ab 100644
--- a/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
+++ b/test/src/test/java/org/apache/accumulo/test/randomwalk/FrameworkTest.java
@@ -16,52 +16,52 @@
*/
package org.apache.accumulo.test.randomwalk;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
import javax.xml.XMLConstants;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
import javax.xml.validation.Schema;
import javax.xml.validation.SchemaFactory;
-import junit.framework.TestCase;
-
import org.apache.accumulo.test.randomwalk.unit.CreateTable;
-import org.junit.Assert;
+import org.w3c.dom.Document;
+import org.xml.sax.SAXException;
-public class FrameworkTest extends TestCase {
+public class FrameworkTest {
- public void testXML() {
+ // Need to use fully qualified name here because of conflict with org.apache.accumulo.test.randomwalk.Test
+ @org.junit.Test
+ public void testXML() throws SAXException, URISyntaxException, ParserConfigurationException, IOException {
+ SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
+ Schema moduleSchema = sf.newSchema(getFile("/randomwalk/module.xsd"));
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
- DocumentBuilder docbuilder;
+ dbf.setSchema(moduleSchema);
- SchemaFactory sf = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
- Schema moduleSchema = null;
- try {
- moduleSchema = sf.newSchema(new File(this.getClass().getResource("/randomwalk/module.xsd").toURI()));
- } catch (Exception e) {
- Assert.fail("Caught exception: " + e);
- }
+ DocumentBuilder docbuilder = dbf.newDocumentBuilder();
+ Document document = docbuilder.parse(getFile("/randomwalk/Basic.xml"));
- dbf.setSchema(moduleSchema);
+ assertNotEquals("Parsing randomwalk xml should result in nodes.", 0, document.getChildNodes().getLength());
+ }
- try {
- File f = new File(this.getClass().getResource("/randomwalk/Basic.xml").toURI());
- docbuilder = dbf.newDocumentBuilder();
- docbuilder.parse(f);
- } catch (Exception e) {
- Assert.fail("Caught exception: " + e);
- }
+ private File getFile(String resource) throws URISyntaxException {
+ return new File(this.getClass().getResource(resource).toURI());
}
+ @org.junit.Test
public void testRWTest() {
-
Test t1 = new CreateTable();
- assertTrue(t1.toString().equals("org.apache.accumulo.test.randomwalk.unit.CreateTable"));
+ assertEquals("org.apache.accumulo.test.randomwalk.unit.CreateTable", t1.toString());
Test t2 = new CreateTable();
- assertTrue(t1.equals(t2));
+ assertEquals("CreateTable test nodes were not equal.", t1, t2);
}
}
[22/50] [abbrv] git commit: ACCUMULO-2343 Add AsyncSocketAppender
Posted by el...@apache.org.
ACCUMULO-2343 Add AsyncSocketAppender
AsyncSocketAppender is a Log4J AsyncAppender with its own internal SocketAppender.
Configuration for either appender can be set on the AsyncSocketAppender itself. An
AsyncSocketAppender can be configured using a Log4J properties file, while an ordinary
AsyncAppender cannot.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/5a72c37e
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/5a72c37e
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/5a72c37e
Branch: refs/heads/ACCUMULO-378
Commit: 5a72c37eadeca58e670517611f39824f4307321e
Parents: 4d5ba9d
Author: Bill Havanki <bh...@cloudera.com>
Authored: Thu Apr 17 17:01:20 2014 -0400
Committer: Bill Havanki <bh...@cloudera.com>
Committed: Mon May 5 09:16:36 2014 -0400
----------------------------------------------------------------------
.../accumulo/core/util/AsyncSocketAppender.java | 112 +++++++++++++++++++
.../core/util/AsyncSocketAppenderTest.java | 79 +++++++++++++
.../test/functional/ConfigurableMacIT.java | 3 +
.../test/functional/MonitorLoggingIT.java | 101 +++++++++++++++++
test/src/test/resources/conf/generic_logger.xml | 83 ++++++++++++++
test/src/test/resources/conf/monitor_logger.xml | 64 +++++++++++
6 files changed, 442 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5a72c37e/core/src/main/java/org/apache/accumulo/core/util/AsyncSocketAppender.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/util/AsyncSocketAppender.java b/core/src/main/java/org/apache/accumulo/core/util/AsyncSocketAppender.java
new file mode 100644
index 0000000..baae9ba
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/util/AsyncSocketAppender.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.log4j.AsyncAppender;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.net.SocketAppender;
+import org.apache.log4j.spi.LoggingEvent;
+
+/**
+ * An asynchronous appender that maintains its own internal socket appender. Unlike <code>AsyncAppender</code>, this appender can be configured with a Log4J
+ * properties file, although in that case no additional appenders can be added.
+ */
+public class AsyncSocketAppender extends AsyncAppender {
+
+ private final SocketAppender socketAppender;
+ private final AtomicBoolean activated = new AtomicBoolean(false);
+
+ /**
+ * Creates a new appender.
+ */
+ public AsyncSocketAppender() {
+ socketAppender = new SocketAppender();
+ }
+
+ /**
+ * Creates a new appender using the given socket appender internally. Use this constructor for testing only.
+ */
+ AsyncSocketAppender(SocketAppender socketAppender) {
+ this.socketAppender = socketAppender;
+ }
+
+ @Override
+ public void append(final LoggingEvent event) {
+ // Lazy attachment, to avoid calling non-final method in constructor
+ if (!isAttached(socketAppender)) {
+ addAppender(socketAppender);
+ }
+
+ // Lazy activation / connection too, to allow setting host and port
+ if (activated.compareAndSet(false, true)) {
+ socketAppender.activateOptions();
+ }
+
+ super.append(event);
+ }
+
+ // SocketAppender delegate methods
+
+ public String getApplication() {
+ return socketAppender.getApplication();
+ }
+
+ // super.getLocationInfo() will always agree with socketAppender
+ public int getPort() {
+ return socketAppender.getPort();
+ }
+
+ public int getReconnectionDelay() {
+ return socketAppender.getReconnectionDelay();
+ }
+
+ public String getRemoteHost() {
+ return socketAppender.getRemoteHost();
+ }
+
+ public boolean isAdvertiseViaMulticastDNS() {
+ return socketAppender.isAdvertiseViaMulticastDNS();
+ }
+
+ public void setAdvertiseViaMulticastDNS(boolean advertiseViaMulticastDNS) {
+ socketAppender.setAdvertiseViaMulticastDNS(advertiseViaMulticastDNS);
+ }
+
+ public void setApplication(String lapp) {
+ socketAppender.setApplication(lapp);
+ }
+
+ @Override
+ public void setLocationInfo(boolean locationInfo) {
+ super.setLocationInfo(locationInfo);
+ socketAppender.setLocationInfo(locationInfo);
+ }
+
+ public void setPort(int port) {
+ socketAppender.setPort(port);
+ }
+
+ public void setReconnectionDelay(int delay) {
+ socketAppender.setReconnectionDelay(delay);
+ }
+
+ public void setRemoteHost(String host) {
+ socketAppender.setRemoteHost(host);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5a72c37e/core/src/test/java/org/apache/accumulo/core/util/AsyncSocketAppenderTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/accumulo/core/util/AsyncSocketAppenderTest.java b/core/src/test/java/org/apache/accumulo/core/util/AsyncSocketAppenderTest.java
new file mode 100644
index 0000000..414125a
--- /dev/null
+++ b/core/src/test/java/org/apache/accumulo/core/util/AsyncSocketAppenderTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import org.apache.log4j.Logger;
+import org.apache.log4j.Priority;
+import org.apache.log4j.net.SocketAppender;
+import org.apache.log4j.spi.LoggingEvent;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.easymock.Capture;
+import static org.easymock.EasyMock.*;
+
+public class AsyncSocketAppenderTest {
+ private SocketAppender sa;
+ private AsyncSocketAppender asa;
+
+ @Before
+ public void setUp() throws Exception {
+ sa = createMock(SocketAppender.class);
+ }
+
+ @Test
+ public void testDelegates() {
+ asa = new AsyncSocketAppender();
+ asa.setApplication("myapp");
+ asa.setLocationInfo(true);
+ asa.setPort(1234);
+ asa.setReconnectionDelay(56);
+ asa.setRemoteHost("remotehost");
+ assertEquals("myapp", asa.getApplication());
+ assertEquals(true, asa.getLocationInfo()); // not really delegating
+ assertEquals(1234, asa.getPort());
+ assertEquals(56, asa.getReconnectionDelay());
+ assertEquals("remotehost", asa.getRemoteHost());
+ }
+
+ @Test
+ public void testSetLocationInfo() {
+ sa.setLocationInfo(true);
+ replay(sa);
+ asa = new AsyncSocketAppender(sa);
+ asa.setLocationInfo(true);
+ verify(sa);
+ }
+
+ @Test
+ public void testAppend() {
+ asa = new AsyncSocketAppender(sa);
+ assertFalse(asa.isAttached(sa));
+ LoggingEvent event1 = new LoggingEvent("java.lang.String", Logger.getRootLogger(), Priority.INFO, "event1", null);
+ LoggingEvent event2 = new LoggingEvent("java.lang.Integer", Logger.getRootLogger(), Priority.WARN, "event2", null);
+ sa.activateOptions();
+ sa.doAppend(event1);
+ sa.doAppend(event2);
+ sa.close();
+ replay(sa);
+ asa.doAppend(event1);
+ asa.doAppend(event2);
+ asa.close(); // forces events to be appended to socket appender
+ assertTrue(asa.isAttached(sa));
+ verify(sa);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5a72c37e/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
index d9bed7f..59b0977 100644
--- a/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
+++ b/test/src/test/java/org/apache/accumulo/test/functional/ConfigurableMacIT.java
@@ -44,6 +44,8 @@ public class ConfigurableMacIT extends AbstractMacIT {
public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {}
+ public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {}
+
@Before
public void setUp() throws Exception {
MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(
@@ -63,6 +65,7 @@ public class ConfigurableMacIT extends AbstractMacIT {
coreSite.writeXml(out);
out.close();
}
+ beforeClusterStart(cfg);
cluster.start();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5a72c37e/test/src/test/java/org/apache/accumulo/test/functional/MonitorLoggingIT.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/accumulo/test/functional/MonitorLoggingIT.java b/test/src/test/java/org/apache/accumulo/test/functional/MonitorLoggingIT.java
new file mode 100644
index 0000000..2dadafe
--- /dev/null
+++ b/test/src/test/java/org/apache/accumulo/test/functional/MonitorLoggingIT.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.net.URL;
+
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl;
+import org.apache.accumulo.monitor.Monitor;
+import org.apache.accumulo.server.util.Admin;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.KeeperException;
+import org.junit.Test;
+
+public class MonitorLoggingIT extends ConfigurableMacIT {
+ public static final Logger log = Logger.getLogger(MonitorLoggingIT.class);
+
+ @Override
+ public void beforeClusterStart(MiniAccumuloConfigImpl cfg) throws Exception {
+ File confDir = cfg.getConfDir();
+ try {
+ FileUtils.copyFileToDirectory(new File(MonitorLoggingIT.class.getResource("/conf/generic_logger.xml").toURI()), confDir);
+ FileUtils.copyFileToDirectory(new File(MonitorLoggingIT.class.getResource("/conf/monitor_logger.xml").toURI()), confDir);
+ } catch (Exception e) {
+ log.error("Failed to copy Log4J XML files to conf dir", e);
+ }
+ }
+
+ private static final int NUM_LOCATION_PASSES = 5;
+ private static final long LOCATION_DELAY = 5000L;
+
+ @Test
+ public void logToMonitor() throws Exception {
+ // Start the monitor.
+ log.debug("Starting Monitor");
+ Process monitor = cluster.exec(Monitor.class);
+
+ // Get monitor location to ensure it is running.
+ String monitorLocation = null;
+ for (int i = 0; i < NUM_LOCATION_PASSES; i++) {
+ Thread.sleep(LOCATION_DELAY);
+ try {
+ monitorLocation = getMonitor();
+ break;
+ } catch (KeeperException e) {
+ log.debug("Monitor not up yet, trying again in " + LOCATION_DELAY + " ms");
+ }
+ }
+ assertNotNull("Monitor failed to start within " + (LOCATION_DELAY * NUM_LOCATION_PASSES) + " ms", monitorLocation);
+ log.debug("Monitor running at " + monitorLocation);
+
+ // Attempt a scan with an invalid iterator to force a log message in the monitor.
+ try {
+ Connector c = getConnector();
+ Scanner s = c.createScanner("accumulo.root", new Authorizations());
+ IteratorSetting cfg = new IteratorSetting(100, "incorrect", "java.lang.String");
+ s.addScanIterator(cfg);
+ s.iterator().next();
+ } catch (Exception e) {
+ // expected, the iterator was bad
+ }
+ Thread.sleep(5000L); // extra precaution to ensure monitor has opportunity to log
+
+ // Verify messages were received at the monitor.
+ URL url = new URL("http://" + monitorLocation + "/log");
+ log.debug("Fetching web page " + url);
+ String result = FunctionalTestUtils.readAll(url.openStream());
+ assertTrue("No log messages found", result.contains("<pre class='logevent'>"));
+
+ // Shutdown cleanly.
+ log.debug("Stopping mini accumulo cluster");
+ Process shutdown = cluster.exec(Admin.class, "stopAll");
+ shutdown.waitFor();
+ assertTrue(shutdown.exitValue() == 0);
+ log.debug("success!");
+ monitor.destroy();
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5a72c37e/test/src/test/resources/conf/generic_logger.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/conf/generic_logger.xml b/test/src/test/resources/conf/generic_logger.xml
new file mode 100644
index 0000000..db79efe
--- /dev/null
+++ b/test/src/test/resources/conf/generic_logger.xml
@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+ <param name="MaxFileSize" value="1000MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Send all logging data to a centralized logger -->
+ <appender name="N1" class="org.apache.log4j.net.SocketAppender">
+ <param name="remoteHost" value="${org.apache.accumulo.core.host.log}"/>
+ <param name="port" value="${org.apache.accumulo.core.host.log.port}"/>
+ <param name="application" value="${org.apache.accumulo.core.application}:${org.apache.accumulo.core.ip.localhost.hostname}"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- If the centralized logger is down, buffer the log events, but drop them if it stays down -->
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="N1" />
+ </appender>
+
+ <!-- Log accumulo events to the debug, normal and remote logs. -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="ASYNC" />
+ </logger>
+
+ <logger name="org.apache.accumulo.core.file.rfile.bcfile">
+ <level value="INFO"/>
+ </logger>
+
+ <logger name="org.mortbay.log">
+ <level value="WARN"/>
+ </logger>
+
+ <logger name="org.apache.zookeeper">
+ <level value="ERROR"/>
+ </logger>
+
+ <!-- Log non-accumulo events to the debug and normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/5a72c37e/test/src/test/resources/conf/monitor_logger.xml
----------------------------------------------------------------------
diff --git a/test/src/test/resources/conf/monitor_logger.xml b/test/src/test/resources/conf/monitor_logger.xml
new file mode 100644
index 0000000..91a7671
--- /dev/null
+++ b/test/src/test/resources/conf/monitor_logger.xml
@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <!-- Write out everything at the DEBUG level to the debug log -->
+ <appender name="A2" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.debug.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="DEBUG"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Write out INFO and higher to the regular log -->
+ <appender name="A3" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${org.apache.accumulo.core.dir.log}/${org.apache.accumulo.core.application}_${org.apache.accumulo.core.ip.localhost.hostname}.log"/>
+ <param name="MaxFileSize" value="100MB"/>
+ <param name="MaxBackupIndex" value="10"/>
+ <param name="Threshold" value="INFO"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- Keep the last few log messages for display to the user -->
+ <appender name="GUI" class="org.apache.accumulo.server.monitor.LogService">
+ <param name="keep" value="40"/>
+ <param name="Threshold" value="WARN"/>
+ </appender>
+
+ <!-- Log accumulo messages to debug, normal and GUI -->
+ <logger name="org.apache.accumulo" additivity="false">
+ <level value="DEBUG"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ <appender-ref ref="GUI" />
+ </logger>
+
+ <!-- Log non-accumulo messages to debug, normal logs. -->
+ <root>
+ <level value="INFO"/>
+ <appender-ref ref="A2" />
+ <appender-ref ref="A3" />
+ </root>
+
+</log4j:configuration>
[05/50] [abbrv] git commit: Merge branch '1.6.0-SNAPSHOT'
Posted by el...@apache.org.
Merge branch '1.6.0-SNAPSHOT'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/1dae0298
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/1dae0298
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/1dae0298
Branch: refs/heads/ACCUMULO-378
Commit: 1dae0298fd2c1bee80855e5c4067ef569c6299b5
Parents: 7a82625 1c38957
Author: Mike Drob <md...@cloudera.com>
Authored: Wed Apr 30 15:33:41 2014 -0400
Committer: Mike Drob <md...@cloudera.com>
Committed: Wed Apr 30 15:33:41 2014 -0400
----------------------------------------------------------------------
.../accumulo/test/randomwalk/FrameworkTest.java | 48 ++++++++++----------
1 file changed, 24 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
[42/50] [abbrv] git commit: ACCUMULO-2573 ACCUMULO-2583 Initial stub
test to see WAL get transmitted to the peer.
Posted by el...@apache.org.
ACCUMULO-2573 ACCUMULO-2583 Initial stub test to see WAL get transmitted to the peer.
Lots of random cleanup included that I stumbled on. Fix some issues with
the DWQ key name. A minor change to how the thrift IDL looks.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/3579b674
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/3579b674
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/3579b674
Branch: refs/heads/ACCUMULO-378
Commit: 3579b674a32b61564a29d6dd66ea4057d568c329
Parents: c7b6062
Author: Josh Elser <el...@apache.org>
Authored: Wed May 7 23:28:32 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed May 7 23:28:32 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/conf/Property.java | 10 +-
.../replication/thrift/ReplicationServicer.java | 168 ++++++--------
core/src/main/thrift/replication.thrift | 4 +-
.../replication/AccumuloReplicaSystem.java | 137 ------------
.../ReplicationWorkAssignerHelper.java | 22 +-
.../accumulo/server/util/SystemPropUtil.java | 9 +-
.../ReplicationWorkAssignerHelperTest.java | 56 +++++
.../java/org/apache/accumulo/master/Master.java | 11 +
.../master/replication/ReplicationDriver.java | 4 +-
.../replication/ReplicationWorkAssigner.java | 16 +-
.../master/replication/StatusMaker.java | 16 +-
.../ReplicationWorkAssignerTest.java | 33 +--
.../apache/accumulo/tserver/TabletServer.java | 28 ++-
.../replication/AccumuloReplicaSystem.java | 224 +++++++++++++++++++
.../replication/ReplicationProcessor.java | 43 ++--
.../replication/ReplicationServicerHandler.java | 44 ++++
.../tserver/replication/ReplicationWorker.java | 3 +-
.../replication/ReplicationProcessorTest.java | 68 ++++++
.../tserver/src/test/resources/log4j.properties | 36 +++
.../test/replication/ReplicationIT.java | 123 ++++++++++
20 files changed, 764 insertions(+), 291 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6d3b00c..d611bd5 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -451,10 +451,6 @@ public enum Property {
@Experimental
REPLICATION_PEERS("replication.peer.", null, PropertyType.PREFIX, "Properties in this category control what systems data can be replicated to"),
@Experimental
- REPLICATION_BATCH_SIZE("replication.batch.size", "1000", PropertyType.COUNT, "Maximum number of updates (WAL) or key-value pairs (RFile) to send in one replication task"),
- @Experimental
- REPLICATION_SEND_THREAD_POOL_SIZE("replication.send.threads", "1", PropertyType.COUNT, "Size of threadpool used to start replication to slaves"),
- @Experimental
REPLICATION_MAX_WORK_QUEUE("replication.max.work.queue", "20000000", PropertyType.COUNT, "Upper bound of the number of files queued for replication"),
@Experimental
REPLICATION_WORK_ASSIGNMENT_SLEEP("replication.work.assignment.sleep", "30s", PropertyType.TIMEDURATION, "Amount of time to sleep between replication work assignment"),
@@ -464,6 +460,10 @@ public enum Property {
REPLICATION_RECEIPT_SERVICE_PORT("replication.receipt.service.port", "10001", PropertyType.PORT, "Listen port used by thrift service in tserver listening for replication"),
@Experimental
REPLICATION_WORK_ATTEMPTS("replication.work.attempts", "10", PropertyType.COUNT, "Number of attempts to try to replicate some data before giving up and letting it naturally be retried later"),
+ @Experimental
+ REPLICATION_MIN_THREADS("replication.receiver.min.threads", "1", PropertyType.COUNT, "Minimum number of threads for replciation"),
+ @Experimental
+ REPLICATION_THREADCHECK("replication.receiver.threadcheck.time", "5s", PropertyType.TIMEDURATION, "The time between adjustments of the replication thread pool."),
;
@@ -713,7 +713,7 @@ public enum Property {
return key.startsWith(Property.TABLE_PREFIX.getKey()) || key.startsWith(Property.TSERV_PREFIX.getKey()) || key.startsWith(Property.LOGGER_PREFIX.getKey())
|| key.startsWith(Property.MASTER_PREFIX.getKey()) || key.startsWith(Property.GC_PREFIX.getKey())
|| key.startsWith(Property.MONITOR_PREFIX.getKey() + "banner.") || key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey())
- || key.startsWith(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey());
+ || key.startsWith(Property.TABLE_COMPACTION_STRATEGY_PREFIX.getKey()) || key.startsWith(REPLICATION_PREFIX.getKey());
}
/**
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
index 1a40dc4..e5e26ca 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/thrift/ReplicationServicer.java
@@ -50,9 +50,9 @@ import org.slf4j.LoggerFactory;
public interface Iface {
- public ByteBuffer replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException;
+ public long replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException;
- public ByteBuffer replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException;
+ public long replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException;
}
@@ -84,7 +84,7 @@ import org.slf4j.LoggerFactory;
super(iprot, oprot);
}
- public ByteBuffer replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException
+ public long replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, org.apache.thrift.TException
{
send_replicateLog(remoteTableId, data);
return recv_replicateLog();
@@ -98,7 +98,7 @@ import org.slf4j.LoggerFactory;
sendBase("replicateLog", args);
}
- public ByteBuffer recv_replicateLog() throws RemoteReplicationException, org.apache.thrift.TException
+ public long recv_replicateLog() throws RemoteReplicationException, org.apache.thrift.TException
{
replicateLog_result result = new replicateLog_result();
receiveBase(result, "replicateLog");
@@ -111,7 +111,7 @@ import org.slf4j.LoggerFactory;
throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "replicateLog failed: unknown result");
}
- public ByteBuffer replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException
+ public long replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, org.apache.thrift.TException
{
send_replicateKeyValues(remoteTableId, data);
return recv_replicateKeyValues();
@@ -125,7 +125,7 @@ import org.slf4j.LoggerFactory;
sendBase("replicateKeyValues", args);
}
- public ByteBuffer recv_replicateKeyValues() throws RemoteReplicationException, org.apache.thrift.TException
+ public long recv_replicateKeyValues() throws RemoteReplicationException, org.apache.thrift.TException
{
replicateKeyValues_result result = new replicateKeyValues_result();
receiveBase(result, "replicateKeyValues");
@@ -181,7 +181,7 @@ import org.slf4j.LoggerFactory;
prot.writeMessageEnd();
}
- public ByteBuffer getResult() throws RemoteReplicationException, org.apache.thrift.TException {
+ public long getResult() throws RemoteReplicationException, org.apache.thrift.TException {
if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
throw new IllegalStateException("Method call not finished!");
}
@@ -216,7 +216,7 @@ import org.slf4j.LoggerFactory;
prot.writeMessageEnd();
}
- public ByteBuffer getResult() throws RemoteReplicationException, org.apache.thrift.TException {
+ public long getResult() throws RemoteReplicationException, org.apache.thrift.TException {
if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
throw new IllegalStateException("Method call not finished!");
}
@@ -261,6 +261,7 @@ import org.slf4j.LoggerFactory;
replicateLog_result result = new replicateLog_result();
try {
result.success = iface.replicateLog(args.remoteTableId, args.data);
+ result.setSuccessIsSet(true);
} catch (RemoteReplicationException e) {
result.e = e;
}
@@ -285,6 +286,7 @@ import org.slf4j.LoggerFactory;
replicateKeyValues_result result = new replicateKeyValues_result();
try {
result.success = iface.replicateKeyValues(args.remoteTableId, args.data);
+ result.setSuccessIsSet(true);
} catch (RemoteReplicationException e) {
result.e = e;
}
@@ -754,7 +756,7 @@ import org.slf4j.LoggerFactory;
public static class replicateLog_result implements org.apache.thrift.TBase<replicateLog_result, replicateLog_result._Fields>, java.io.Serializable, Cloneable {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateLog_result");
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I64, (short)0);
private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
@@ -763,7 +765,7 @@ import org.slf4j.LoggerFactory;
schemes.put(TupleScheme.class, new replicateLog_resultTupleSchemeFactory());
}
- public ByteBuffer success; // required
+ public long success; // required
public RemoteReplicationException e; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
@@ -828,11 +830,13 @@ import org.slf4j.LoggerFactory;
}
// isset id assignments
+ private static final int __SUCCESS_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
@@ -843,11 +847,12 @@ import org.slf4j.LoggerFactory;
}
public replicateLog_result(
- ByteBuffer success,
+ long success,
RemoteReplicationException e)
{
this();
this.success = success;
+ setSuccessIsSet(true);
this.e = e;
}
@@ -855,10 +860,8 @@ import org.slf4j.LoggerFactory;
* Performs a deep copy on <i>other</i>.
*/
public replicateLog_result(replicateLog_result other) {
- if (other.isSetSuccess()) {
- this.success = org.apache.thrift.TBaseHelper.copyBinary(other.success);
-;
- }
+ __isset_bitfield = other.__isset_bitfield;
+ this.success = other.success;
if (other.isSetE()) {
this.e = new RemoteReplicationException(other.e);
}
@@ -870,42 +873,32 @@ import org.slf4j.LoggerFactory;
@Override
public void clear() {
- this.success = null;
+ setSuccessIsSet(false);
+ this.success = 0;
this.e = null;
}
- public byte[] getSuccess() {
- setSuccess(org.apache.thrift.TBaseHelper.rightSize(success));
- return success == null ? null : success.array();
- }
-
- public ByteBuffer bufferForSuccess() {
- return success;
- }
-
- public replicateLog_result setSuccess(byte[] success) {
- setSuccess(success == null ? (ByteBuffer)null : ByteBuffer.wrap(success));
- return this;
+ public long getSuccess() {
+ return this.success;
}
- public replicateLog_result setSuccess(ByteBuffer success) {
+ public replicateLog_result setSuccess(long success) {
this.success = success;
+ setSuccessIsSet(true);
return this;
}
public void unsetSuccess() {
- this.success = null;
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID);
}
/** Returns true if field success is set (has been assigned a value) and false otherwise */
public boolean isSetSuccess() {
- return this.success != null;
+ return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID);
}
public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value);
}
public RemoteReplicationException getE() {
@@ -938,7 +931,7 @@ import org.slf4j.LoggerFactory;
if (value == null) {
unsetSuccess();
} else {
- setSuccess((ByteBuffer)value);
+ setSuccess((Long)value);
}
break;
@@ -956,7 +949,7 @@ import org.slf4j.LoggerFactory;
public Object getFieldValue(_Fields field) {
switch (field) {
case SUCCESS:
- return getSuccess();
+ return Long.valueOf(getSuccess());
case E:
return getE();
@@ -993,12 +986,12 @@ import org.slf4j.LoggerFactory;
if (that == null)
return false;
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
+ boolean this_present_success = true;
+ boolean that_present_success = true;
if (this_present_success || that_present_success) {
if (!(this_present_success && that_present_success))
return false;
- if (!this.success.equals(that.success))
+ if (this.success != that.success)
return false;
}
@@ -1068,11 +1061,7 @@ import org.slf4j.LoggerFactory;
boolean first = true;
sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.success, sb);
- }
+ sb.append(this.success);
first = false;
if (!first) sb.append(", ");
sb.append("e:");
@@ -1101,6 +1090,8 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -1126,8 +1117,8 @@ import org.slf4j.LoggerFactory;
}
switch (schemeField.id) {
case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readBinary();
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.success = iprot.readI64();
struct.setSuccessIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
@@ -1157,9 +1148,9 @@ import org.slf4j.LoggerFactory;
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
+ if (struct.isSetSuccess()) {
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeBinary(struct.success);
+ oprot.writeI64(struct.success);
oprot.writeFieldEnd();
}
if (struct.e != null) {
@@ -1193,7 +1184,7 @@ import org.slf4j.LoggerFactory;
}
oprot.writeBitSet(optionals, 2);
if (struct.isSetSuccess()) {
- oprot.writeBinary(struct.success);
+ oprot.writeI64(struct.success);
}
if (struct.isSetE()) {
struct.e.write(oprot);
@@ -1205,7 +1196,7 @@ import org.slf4j.LoggerFactory;
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
- struct.success = iprot.readBinary();
+ struct.success = iprot.readI64();
struct.setSuccessIsSet(true);
}
if (incoming.get(1)) {
@@ -1678,7 +1669,7 @@ import org.slf4j.LoggerFactory;
public static class replicateKeyValues_result implements org.apache.thrift.TBase<replicateKeyValues_result, replicateKeyValues_result._Fields>, java.io.Serializable, Cloneable {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("replicateKeyValues_result");
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
+ private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.I64, (short)0);
private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
@@ -1687,7 +1678,7 @@ import org.slf4j.LoggerFactory;
schemes.put(TupleScheme.class, new replicateKeyValues_resultTupleSchemeFactory());
}
- public ByteBuffer success; // required
+ public long success; // required
public RemoteReplicationException e; // required
/** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
@@ -1752,11 +1743,13 @@ import org.slf4j.LoggerFactory;
}
// isset id assignments
+ private static final int __SUCCESS_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING , true)));
+ new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
metaDataMap = Collections.unmodifiableMap(tmpMap);
@@ -1767,11 +1760,12 @@ import org.slf4j.LoggerFactory;
}
public replicateKeyValues_result(
- ByteBuffer success,
+ long success,
RemoteReplicationException e)
{
this();
this.success = success;
+ setSuccessIsSet(true);
this.e = e;
}
@@ -1779,10 +1773,8 @@ import org.slf4j.LoggerFactory;
* Performs a deep copy on <i>other</i>.
*/
public replicateKeyValues_result(replicateKeyValues_result other) {
- if (other.isSetSuccess()) {
- this.success = org.apache.thrift.TBaseHelper.copyBinary(other.success);
-;
- }
+ __isset_bitfield = other.__isset_bitfield;
+ this.success = other.success;
if (other.isSetE()) {
this.e = new RemoteReplicationException(other.e);
}
@@ -1794,42 +1786,32 @@ import org.slf4j.LoggerFactory;
@Override
public void clear() {
- this.success = null;
+ setSuccessIsSet(false);
+ this.success = 0;
this.e = null;
}
- public byte[] getSuccess() {
- setSuccess(org.apache.thrift.TBaseHelper.rightSize(success));
- return success == null ? null : success.array();
- }
-
- public ByteBuffer bufferForSuccess() {
- return success;
- }
-
- public replicateKeyValues_result setSuccess(byte[] success) {
- setSuccess(success == null ? (ByteBuffer)null : ByteBuffer.wrap(success));
- return this;
+ public long getSuccess() {
+ return this.success;
}
- public replicateKeyValues_result setSuccess(ByteBuffer success) {
+ public replicateKeyValues_result setSuccess(long success) {
this.success = success;
+ setSuccessIsSet(true);
return this;
}
public void unsetSuccess() {
- this.success = null;
+ __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __SUCCESS_ISSET_ID);
}
/** Returns true if field success is set (has been assigned a value) and false otherwise */
public boolean isSetSuccess() {
- return this.success != null;
+ return EncodingUtils.testBit(__isset_bitfield, __SUCCESS_ISSET_ID);
}
public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
+ __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __SUCCESS_ISSET_ID, value);
}
public RemoteReplicationException getE() {
@@ -1862,7 +1844,7 @@ import org.slf4j.LoggerFactory;
if (value == null) {
unsetSuccess();
} else {
- setSuccess((ByteBuffer)value);
+ setSuccess((Long)value);
}
break;
@@ -1880,7 +1862,7 @@ import org.slf4j.LoggerFactory;
public Object getFieldValue(_Fields field) {
switch (field) {
case SUCCESS:
- return getSuccess();
+ return Long.valueOf(getSuccess());
case E:
return getE();
@@ -1917,12 +1899,12 @@ import org.slf4j.LoggerFactory;
if (that == null)
return false;
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
+ boolean this_present_success = true;
+ boolean that_present_success = true;
if (this_present_success || that_present_success) {
if (!(this_present_success && that_present_success))
return false;
- if (!this.success.equals(that.success))
+ if (this.success != that.success)
return false;
}
@@ -1992,11 +1974,7 @@ import org.slf4j.LoggerFactory;
boolean first = true;
sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- org.apache.thrift.TBaseHelper.toString(this.success, sb);
- }
+ sb.append(this.success);
first = false;
if (!first) sb.append(", ");
sb.append("e:");
@@ -2025,6 +2003,8 @@ import org.slf4j.LoggerFactory;
private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
try {
+ // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -2050,8 +2030,8 @@ import org.slf4j.LoggerFactory;
}
switch (schemeField.id) {
case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readBinary();
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.success = iprot.readI64();
struct.setSuccessIsSet(true);
} else {
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
@@ -2081,9 +2061,9 @@ import org.slf4j.LoggerFactory;
struct.validate();
oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
+ if (struct.isSetSuccess()) {
oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeBinary(struct.success);
+ oprot.writeI64(struct.success);
oprot.writeFieldEnd();
}
if (struct.e != null) {
@@ -2117,7 +2097,7 @@ import org.slf4j.LoggerFactory;
}
oprot.writeBitSet(optionals, 2);
if (struct.isSetSuccess()) {
- oprot.writeBinary(struct.success);
+ oprot.writeI64(struct.success);
}
if (struct.isSetE()) {
struct.e.write(oprot);
@@ -2129,7 +2109,7 @@ import org.slf4j.LoggerFactory;
TTupleProtocol iprot = (TTupleProtocol) prot;
BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
- struct.success = iprot.readBinary();
+ struct.success = iprot.readI64();
struct.setSuccessIsSet(true);
}
if (incoming.get(1)) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/core/src/main/thrift/replication.thrift
----------------------------------------------------------------------
diff --git a/core/src/main/thrift/replication.thrift b/core/src/main/thrift/replication.thrift
index 2994092..6245973 100644
--- a/core/src/main/thrift/replication.thrift
+++ b/core/src/main/thrift/replication.thrift
@@ -43,6 +43,6 @@ service ReplicationCoordinator {
}
service ReplicationServicer {
- binary replicateLog(1:i32 remoteTableId, 2:WalEdits data) throws (1:RemoteReplicationException e),
- binary replicateKeyValues(1:i32 remoteTableId, 2:KeyValues data) throws (1:RemoteReplicationException e)
+ i64 replicateLog(1:i32 remoteTableId, 2:WalEdits data) throws (1:RemoteReplicationException e),
+ i64 replicateKeyValues(1:i32 remoteTableId, 2:KeyValues data) throws (1:RemoteReplicationException e)
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java b/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java
deleted file mode 100644
index 138eb71..0000000
--- a/server/base/src/main/java/org/apache/accumulo/server/replication/AccumuloReplicaSystem.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.server.replication;
-
-import java.nio.ByteBuffer;
-
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.ZooKeeperInstance;
-import org.apache.accumulo.core.client.impl.ClientExecReturn;
-import org.apache.accumulo.core.client.impl.ReplicationClient;
-import org.apache.accumulo.core.client.impl.ServerConfigurationUtil;
-import org.apache.accumulo.core.client.replication.ReplicaSystem;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.conf.Property;
-import org.apache.accumulo.core.replication.ReplicationTarget;
-import org.apache.accumulo.core.replication.proto.Replication.Status;
-import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
-import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
-import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client;
-import org.apache.accumulo.core.util.ByteBufferUtil;
-import org.apache.accumulo.core.util.UtilWaitThread;
-import org.apache.accumulo.server.client.HdfsZooInstance;
-import org.apache.hadoop.fs.Path;
-import org.apache.thrift.transport.TTransportException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-/**
- *
- */
-public class AccumuloReplicaSystem implements ReplicaSystem {
- private static final Logger log = LoggerFactory.getLogger(AccumuloReplicaSystem.class);
-
- private String instanceName, zookeepers;
-
- @Override
- public void configure(String configuration) {
- Preconditions.checkNotNull(configuration);
-
- int index = configuration.indexOf(',');
- if (-1 == index) {
- throw new IllegalArgumentException("Expected comma in configuration string");
- }
-
- instanceName = configuration.substring(0, index);
- zookeepers = configuration.substring(index + 1);
- }
-
- @Override
- public Status replicate(Path p, Status status, ReplicationTarget target) {
- Instance localInstance = HdfsZooInstance.getInstance();
- AccumuloConfiguration localConf = ServerConfigurationUtil.getConfiguration(localInstance);
-
- Instance peerInstance = getPeerInstance(target);
- // Remote identifier is an integer (table id) in this case.
- final int remoteTableId = Integer.parseInt(target.getRemoteIdentifier());
-
- // Attempt the replication of this status a number of times before giving up and
- // trying to replicate it again later some other time.
- for (int i = 0; i < localConf.getCount(Property.REPLICATION_WORK_ATTEMPTS); i++) {
- String peerTserver;
- try {
- // Ask the master on the remote what TServer we should talk with to replicate the data
- peerTserver = ReplicationClient.executeCoordinatorWithReturn(peerInstance, new ClientExecReturn<String,ReplicationCoordinator.Client>() {
-
- @Override
- public String execute(ReplicationCoordinator.Client client) throws Exception {
- return client.getServicerAddress(remoteTableId);
- }
-
- });
- } catch (AccumuloException | AccumuloSecurityException e) {
- // No progress is made
- log.error("Could not connect to master at {}, cannot proceed with replication. Will retry", target, e);
- continue;
- }
-
- if (null == peerTserver) {
- // Something went wrong, and we didn't get a valid tserver from the remote for some reason
- log.warn("Did not receive tserver from master at {}, cannot proceed with replication. Will retry.", target);
- continue;
- }
-
- // We have a tserver on the remote -- send the data its way.
- ByteBuffer result;
- //TODO should chunk up the given file into some configurable sizes instead of just sending the entire file all at once
- // configuration should probably just be size based.
- try {
- result = ReplicationClient.executeServicerWithReturn(peerInstance, peerTserver, new ClientExecReturn<ByteBuffer,ReplicationServicer.Client>() {
- @Override
- public ByteBuffer execute(Client client) throws Exception {
- //TODO This needs to actually send the appropriate data, and choose replicateLog or replicateKeyValues
- return client.replicateLog(remoteTableId, null);
- }
- });
-
- // We need to be able to parse the returned Status,
- // if we can't, we don't know what the server actually parsed.
- try {
- return Status.parseFrom(ByteBufferUtil.toBytes(result));
- } catch (InvalidProtocolBufferException e) {
- log.error("Could not parse return Status from {}", peerTserver, e);
- throw new RuntimeException("Could not parse returned Status from " + peerTserver, e);
- }
- } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
- log.warn("Could not connect to remote server {}, will retry", peerTserver, e);
- UtilWaitThread.sleep(250);
- }
- }
-
- // We made no status, punt on it for now, and let it re-queue itself for work
- return status;
- }
-
- public Instance getPeerInstance(ReplicationTarget target) {
- return new ZooKeeperInstance(instanceName, zookeepers);
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
index dd70134..6b408b7 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/replication/ReplicationWorkAssignerHelper.java
@@ -38,18 +38,7 @@ public class ReplicationWorkAssignerHelper {
* @return Key for identifying work in queue
*/
public static String getQueueKey(String filename, ReplicationTarget replTarget) {
- return getQueueKey(filename, replTarget.toText().toString());
- }
-
- /**
- * Serialize a filename and a {@link ReplicationTarget} into the expected
- * key format for use with the {@link DistributedWorkQueue}
- * @param filename Filename for data to be replicated
- * @param replTargetStr Serialized {@link ReplicationTarget}
- * @return Key for identifying work in queue
- */
- public static String getQueueKey(String filename, String replTargetStr) {
- return filename + KEY_SEPARATOR + replTargetStr;
+ return filename + KEY_SEPARATOR + replTarget.getPeerName() + KEY_SEPARATOR + replTarget.getRemoteIdentifier();
}
/**
@@ -64,6 +53,13 @@ public class ReplicationWorkAssignerHelper {
throw new IllegalArgumentException("Could not find expected separator in queue key '" + queueKey + "'");
}
- return Maps.immutableEntry(queueKey.substring(0, index), ReplicationTarget.from(queueKey.substring(index + 1)));
+ String filename = queueKey.substring(0, index);
+
+ int secondIndex = queueKey.indexOf(KEY_SEPARATOR, index + 1);
+ if (-1 == secondIndex) {
+ throw new IllegalArgumentException("Could not find expected separator in queue key '" + queueKey + "'");
+ }
+
+ return Maps.immutableEntry(filename, new ReplicationTarget(queueKey.substring(index + 1, secondIndex), queueKey.substring(secondIndex + 1)));
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
index 1d93f90..90047b1 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/util/SystemPropUtil.java
@@ -26,15 +26,22 @@ import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.server.client.HdfsZooInstance;
import org.apache.accumulo.server.zookeeper.ZooReaderWriter;
import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SystemPropUtil {
+ private static final Logger log = LoggerFactory.getLogger(SystemPropUtil.class);
+
public static boolean setSystemProperty(String property, String value) throws KeeperException, InterruptedException {
Property p = Property.getPropertyByKey(property);
- if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property))
+ if ((p != null && !p.getType().isValidFormat(value)) || !Property.isValidZooPropertyKey(property)) {
+ log.warn("Ignoring property {}", property);
return false;
+ }
// create the zk node for this property and set it's data to the specified value
String zPath = ZooUtil.getRoot(HdfsZooInstance.getInstance()) + Constants.ZCONFIG + "/" + property;
+ log.warn("Setting {}={}", zPath, value);
ZooReaderWriter.getInstance().putPersistentData(zPath, value.getBytes(StandardCharsets.UTF_8), NodeExistsPolicy.OVERWRITE);
return true;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/base/src/test/java/ReplicationWorkAssignerHelper/ReplicationWorkAssignerHelperTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/ReplicationWorkAssignerHelper/ReplicationWorkAssignerHelperTest.java b/server/base/src/test/java/ReplicationWorkAssignerHelper/ReplicationWorkAssignerHelperTest.java
new file mode 100644
index 0000000..75fc494
--- /dev/null
+++ b/server/base/src/test/java/ReplicationWorkAssignerHelper/ReplicationWorkAssignerHelperTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ReplicationWorkAssignerHelper;
+
+import java.util.Map.Entry;
+import java.util.UUID;
+
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.server.replication.ReplicationWorkAssignerHelper;
+import org.apache.hadoop.fs.Path;
+import org.apache.zookeeper.common.PathUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class ReplicationWorkAssignerHelperTest {
+
+ @Test
+ public void createsValidZKNodeName() {
+ Path p = new Path ("/accumulo/wals/tserver+port/" + UUID.randomUUID().toString());
+ ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
+
+ String key = ReplicationWorkAssignerHelper.getQueueKey(p.toString(), target);
+
+ PathUtils.validatePath(key);
+ }
+
+ @Test
+ public void queueKeySerialization() {
+ Path p = new Path("/accumulo/wals/tserver+port/" + UUID.randomUUID().toString());
+ ReplicationTarget target = new ReplicationTarget("cluster1", "1");
+
+ String key = ReplicationWorkAssignerHelper.getQueueKey(p.toString(), target);
+
+ Entry<String,ReplicationTarget> result = ReplicationWorkAssignerHelper.fromQueueKey(key);
+ Assert.assertEquals(p.toString(), result.getKey());
+ Assert.assertEquals(target, result.getValue());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/Master.java b/server/master/src/main/java/org/apache/accumulo/master/Master.java
index db7507e..0eac9ab 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/Master.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/Master.java
@@ -77,6 +77,7 @@ import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeExistsPolicy;
import org.apache.accumulo.fate.zookeeper.ZooUtil.NodeMissingPolicy;
import org.apache.accumulo.master.recovery.RecoveryManager;
import org.apache.accumulo.master.replication.ReplicationDriver;
+import org.apache.accumulo.master.replication.ReplicationWorkAssigner;
import org.apache.accumulo.master.state.TableCounts;
import org.apache.accumulo.server.Accumulo;
import org.apache.accumulo.server.ServerConstants;
@@ -166,6 +167,7 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
final EventCoordinator nextEvent = new EventCoordinator();
final private Object mergeLock = new Object();
private ReplicationDriver replicationWorkDriver;
+ private ReplicationWorkAssigner replicationWorkAssigner;
RecoveryManager recoveryManager = null;
ZooLock masterLock = null;
@@ -942,6 +944,14 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
replicationWorkDriver = new ReplicationDriver(this);
replicationWorkDriver.start();
+ // Start the daemon to assign work to tservers to replicate to our peers
+ try {
+ replicationWorkAssigner = new ReplicationWorkAssigner(this, getConnector());
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ throw new RuntimeException(e);
+ }
+ replicationWorkAssigner.start();
+
// Once we are sure the upgrade is complete, we can safely allow fate use.
waitForMetadataUpgrade.await();
@@ -988,6 +998,7 @@ public class Master implements LiveTServerSet.Listener, TableObserver, CurrentSt
final long deadline = System.currentTimeMillis() + MAX_CLEANUP_WAIT_TIME;
statusThread.join(remaining(deadline));
+ replicationWorkAssigner.join(remaining(deadline));
replicationWorkDriver.join(remaining(deadline));
// quit, even if the tablet servers somehow jam up and the watchers
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
index 53a70a7..1bc002e 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationDriver.java
@@ -24,6 +24,7 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.Daemon;
import org.apache.accumulo.fate.util.UtilWaitThread;
import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.log4j.Logger;
/**
@@ -37,6 +38,7 @@ public class ReplicationDriver extends Daemon {
private WorkMaker workMaker;
private StatusMaker statusMaker;
+ private Connector conn;
public ReplicationDriver(Master master) {
super("Replication Driver");
@@ -49,7 +51,6 @@ public class ReplicationDriver extends Daemon {
public void run() {
while (master.stillMaster()) {
if (null == workMaker) {
- Connector conn;
try {
conn = master.getConnector();
} catch (AccumuloException | AccumuloSecurityException e) {
@@ -64,6 +65,7 @@ public class ReplicationDriver extends Daemon {
}
// Make status markers from replication records in metadata
+ // This will end up creating the replication table too
statusMaker.run();
// Tell the work maker to make work
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
index 699381f..dddbf58 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -32,8 +32,11 @@ import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTarget;
import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.util.Daemon;
+import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.replication.ReplicationTable;
@@ -48,6 +51,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.TextFormat;
/**
* Read work records from the replication table, create work entries for other nodes to complete.
@@ -55,7 +59,7 @@ import com.google.protobuf.InvalidProtocolBufferException;
* Uses the DistributedWorkQueue to make the work available for any tserver. This approach does not consider the locality of the tabletserver performing the
* work in relation to the data being replicated (local HDFS blocks).
*/
-public class ReplicationWorkAssigner implements Runnable {
+public class ReplicationWorkAssigner extends Daemon {
private static final Logger log = LoggerFactory.getLogger(ReplicationWorkAssigner.class);
private Master master;
@@ -68,6 +72,7 @@ public class ReplicationWorkAssigner implements Runnable {
private ZooCache zooCache;
public ReplicationWorkAssigner(Master master, Connector conn) {
+ super("Replication Work Assigner");
this.master = master;
this.conn = conn;
}
@@ -185,7 +190,7 @@ public class ReplicationWorkAssigner implements Runnable {
try {
bs = ReplicationTable.getBatchScanner(conn, 4);
} catch (TableNotFoundException e) {
- log.warn("Could not find replication table", e);
+ UtilWaitThread.sleep(1000);
return;
}
@@ -216,12 +221,16 @@ public class ReplicationWorkAssigner implements Runnable {
Path p = new Path(file);
String filename = p.getName();
WorkSection.getTarget(entry.getKey(), buffer);
- String key = ReplicationWorkAssignerHelper.getQueueKey(filename, buffer.toString());
+ String key = ReplicationWorkAssignerHelper.getQueueKey(filename, ReplicationTarget.from(buffer));
// And, we haven't already queued this file up for work already
if (!queuedWork.contains(key)) {
queueWork(key, file);
+ } else {
+ log.debug("Not re-queueing work for {}", key);
}
+ } else {
+ log.debug("Not queueing work for {} because [{}] doesn't need replication", file, TextFormat.shortDebugString(status));
}
}
} finally {
@@ -241,6 +250,7 @@ public class ReplicationWorkAssigner implements Runnable {
*/
protected void queueWork(String key, String path) {
try {
+ log.debug("Queued work for {} and {}", key, path);
workQueue.addWork(key, path);
queuedWork.add(key);
} catch (KeeperException | InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/master/src/main/java/org/apache/accumulo/master/replication/StatusMaker.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/StatusMaker.java b/server/master/src/main/java/org/apache/accumulo/master/replication/StatusMaker.java
index 927d539..140e92d 100644
--- a/server/master/src/main/java/org/apache/accumulo/master/replication/StatusMaker.java
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/StatusMaker.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.protobuf.InvalidProtocolBufferException;
+import com.google.protobuf.TextFormat;
/**
* Reads replication records from the metadata table and creates status records in the replication table
@@ -71,6 +72,7 @@ public class StatusMaker {
public void run() {
Span span = Trace.start("replicationStatusMaker");
try {
+ // Read from a source table (typically accumulo.metadata)
final Scanner s;
try {
s = conn.createScanner(sourceTableName, new Authorizations());
@@ -78,12 +80,13 @@ public class StatusMaker {
throw new RuntimeException(e);
}
- // Only pull records about data that has been ingested and is ready for replication
+ // Only pull replication records
s.fetchColumnFamily(ReplicationSection.COLF);
s.setRange(ReplicationSection.getRange());
Text row = new Text(), tableId = new Text();
for (Entry<Key,Value> entry : s) {
+ // Get a writer to the replication table
if (null == writer) {
// Ensures table exists and is properly configured
ReplicationTable.create(conn);
@@ -103,14 +106,15 @@ public class StatusMaker {
rowStr = rowStr.substring(ReplicationSection.getRowPrefix().length());
try {
- log.debug("Creating replication status record for {} on table {} with {}.", rowStr, tableId, Status.parseFrom(entry.getValue().get()).toString().replace("\n", ", "));
+ log.debug("Creating replication status record for {} on table {} with {}.", rowStr, tableId, TextFormat.shortDebugString(Status.parseFrom(entry.getValue().get())));
} catch (InvalidProtocolBufferException e) {
// TODO Auto-generated catch block
e.printStackTrace();
- }
+ }
Span workSpan = Trace.start("createStatusMutations");
try {
+ // Create entries in the replication table from the metadata table
addStatusRecord(rowStr, tableId, entry.getValue());
} finally {
workSpan.stop();
@@ -125,6 +129,12 @@ public class StatusMaker {
this.writer = bw;
}
+ /**
+ * Create a status record in the replication table
+ * @param file
+ * @param tableId
+ * @param v
+ */
protected void addStatusRecord(String file, Text tableId, Value v) {
// TODO come up with something that tries to avoid creating a new BatchWriter all the time
try {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
index 16e6757..ad2bf75 100644
--- a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
@@ -43,6 +43,7 @@ import org.apache.accumulo.core.security.Credentials;
import org.apache.accumulo.core.security.TablePermission;
import org.apache.accumulo.master.Master;
import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.server.replication.ReplicationWorkAssignerHelper;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
import org.apache.accumulo.server.zookeeper.ZooCache;
import org.apache.hadoop.fs.Path;
@@ -74,7 +75,7 @@ public class ReplicationWorkAssignerTest {
@Test
public void workQueuedUsingFileName() throws Exception {
- ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
+ ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
Text serializedTarget = target.toText();
DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
@@ -83,7 +84,7 @@ public class ReplicationWorkAssignerTest {
assigner.setWorkQueue(workQueue);
Path p = new Path("/accumulo/wal/tserver+port/" + UUID.randomUUID());
-
+
workQueue.addWork(p.getName() + "|" + serializedTarget.toString(), p.toString());
expectLastCall().once();
@@ -116,8 +117,10 @@ public class ReplicationWorkAssignerTest {
@Test
public void createWorkForFilesNeedingIt() throws Exception {
- ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
+ ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText();
+ String keyTarget1 = target1.getPeerName() + ReplicationWorkAssignerHelper.KEY_SEPARATOR + target1.getRemoteIdentifier(), keyTarget2 = target2
+ .getPeerName() + ReplicationWorkAssignerHelper.KEY_SEPARATOR + target2.getRemoteIdentifier();
MockInstance inst = new MockInstance(test.getMethodName());
Credentials creds = new Credentials("root", new PasswordToken(""));
@@ -155,25 +158,25 @@ public class ReplicationWorkAssignerTest {
// Make sure we expect the invocations in the correct order (accumulo is sorted)
if (file1.compareTo(file2) <= 0) {
- String key = filename1 + "|" + serializedTarget1;
+ String key = filename1 + "|" + keyTarget1;
expect(queuedWork.contains(key)).andReturn(false);
workQueue.addWork(key, file1);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
-
- key = filename2 + "|" + serializedTarget2;
+
+ key = filename2 + "|" + keyTarget2;
expect(queuedWork.contains(key)).andReturn(false);
workQueue.addWork(key, file2);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
} else {
- String key = filename2 + "|" + serializedTarget2;
+ String key = filename2 + "|" + keyTarget2;
expect(queuedWork.contains(key)).andReturn(false);
workQueue.addWork(key, file2);
expectLastCall().once();
expect(queuedWork.add(key)).andReturn(true).once();
- key = filename1 + "|" + serializedTarget1;
+ key = filename1 + "|" + keyTarget1;
expect(queuedWork.contains(key)).andReturn(false);
workQueue.addWork(key, file1);
expectLastCall().once();
@@ -189,7 +192,7 @@ public class ReplicationWorkAssignerTest {
@Test
public void doNotCreateWorkForFilesNotNeedingIt() throws Exception {
- ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
+ ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
Text serializedTarget1 = target1.toText(), serializedTarget2 = target2.toText();
MockInstance inst = new MockInstance(test.getMethodName());
@@ -229,7 +232,7 @@ public class ReplicationWorkAssignerTest {
replay(queuedWork, workQueue);
assigner.createWork();
-
+
verify(queuedWork, workQueue);
}
@@ -256,13 +259,13 @@ public class ReplicationWorkAssignerTest {
}
@Test
- public void workNotReAdded() throws Exception {
+ public void workNotReAdded() throws Exception {
Set<String> queuedWork = new HashSet<>();
assigner.setQueuedWork(queuedWork);
- ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
- Text serializedTarget = target.toText();
+ ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
+ String serializedTarget = target.getPeerName() + ReplicationWorkAssignerHelper.KEY_SEPARATOR + target.getRemoteIdentifier();
queuedWork.add("wal1|" + serializedTarget.toString());
@@ -281,7 +284,7 @@ public class ReplicationWorkAssignerTest {
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
String file1 = "/accumulo/wal/tserver+port/wal1";
Mutation m = new Mutation(file1);
- WorkSection.add(m, serializedTarget, StatusUtil.openWithUnknownLengthValue());
+ WorkSection.add(m, target.toText(), StatusUtil.openWithUnknownLengthValue());
bw.addMutation(m);
bw.close();
@@ -291,7 +294,7 @@ public class ReplicationWorkAssignerTest {
assigner.setMaxQueueSize(Integer.MAX_VALUE);
replay(workQueue);
-
+
assigner.createWork();
verify(workQueue);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 5e07a70..803419c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -116,6 +116,7 @@ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
import org.apache.accumulo.core.security.AuthorizationContainer;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SecurityUtil;
@@ -222,6 +223,7 @@ import org.apache.accumulo.tserver.metrics.TabletServerMBean;
import org.apache.accumulo.tserver.metrics.TabletServerMinCMetrics;
import org.apache.accumulo.tserver.metrics.TabletServerScanMetrics;
import org.apache.accumulo.tserver.metrics.TabletServerUpdateMetrics;
+import org.apache.accumulo.tserver.replication.ReplicationServicerHandler;
import org.apache.accumulo.tserver.replication.ReplicationWorker;
import org.apache.commons.collections.map.LRUMap;
import org.apache.hadoop.fs.FSError;
@@ -3017,6 +3019,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// used for stopping the server and MasterListener thread
private volatile boolean serverStopRequested = false;
+ private HostAndPort replicationAddress;
private HostAndPort clientAddress;
private TabletServerResourceManager resourceManager;
@@ -3028,6 +3031,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
private ZooLock tabletServerLock;
private TServer server;
+ private TServer replServer;
private DistributedWorkQueue bulkFailedCopyQ;
@@ -3117,6 +3121,18 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
return address;
}
+ private HostAndPort startReplicationService() throws UnknownHostException {
+ ReplicationServicer.Iface repl = TraceWrap.service(new ReplicationServicerHandler());
+ ReplicationServicer.Processor<ReplicationServicer.Iface> processor = new ReplicationServicer.Processor<ReplicationServicer.Iface>(repl);
+ AccumuloConfiguration conf = getSystemConfiguration();
+ Property maxMessageSizeProperty = (conf.get(Property.TSERV_MAX_MESSAGE_SIZE) != null ? Property.TSERV_MAX_MESSAGE_SIZE : Property.GENERAL_MAX_MESSAGE_SIZE);
+ ServerAddress sp = TServerUtils.startServer(conf, clientAddress.getHostText(), Property.REPLICATION_RECEIPT_SERVICE_PORT, processor,
+ "ReplicationServicerHandler", "Replication Servicer", null, Property.REPLICATION_MIN_THREADS, Property.REPLICATION_THREADCHECK, maxMessageSizeProperty);
+ this.replServer = sp.server;
+ log.info("Started replication service on " + sp.address);
+ return sp.address;
+ }
+
ZooLock getLock() {
return tabletServerLock;
}
@@ -3180,7 +3196,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
// main loop listens for client requests
public void run() {
SecurityUtil.serverLogin(ServerConfiguration.getSiteConfiguration());
-
+
try {
clientAddress = startTabletClientService();
} catch (UnknownHostException e1) {
@@ -3204,6 +3220,14 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
throw new RuntimeException(ex);
}
+ // Start the thrift service listening for incoming replication requests
+ try {
+ replicationAddress = startReplicationService();
+ } catch (UnknownHostException e) {
+ throw new RuntimeException("Failed to start replication service", e);
+ }
+
+ // Start the pool to handle outgoing replications
ThreadPoolExecutor replicationThreadPool = new SimpleThreadPool(getSystemConfiguration().getCount(Property.REPLICATION_WORKER_THREADS), "replication task");
replWorker.setExecutor(replicationThreadPool);
replWorker.run();
@@ -3296,6 +3320,8 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu
}
}
}
+ log.debug("Stopping Replication Server");
+ TServerUtils.stopTServer(this.replServer);
log.debug("Stopping Thrift Servers");
TServerUtils.stopTServer(server);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
new file mode 100644
index 0000000..40676f7
--- /dev/null
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystem.java
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.ZooKeeperInstance;
+import org.apache.accumulo.core.client.impl.ClientExecReturn;
+import org.apache.accumulo.core.client.impl.ReplicationClient;
+import org.apache.accumulo.core.client.impl.ServerConfigurationUtil;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.file.rfile.RFile;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.replication.thrift.KeyValues;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client;
+import org.apache.accumulo.core.replication.thrift.WalEdits;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.server.client.HdfsZooInstance;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.fs.VolumeManagerImpl;
+import org.apache.accumulo.tserver.log.DfsLogger;
+import org.apache.accumulo.tserver.log.DfsLogger.DFSLoggerInputStreams;
+import org.apache.accumulo.tserver.logger.LogFileKey;
+import org.apache.accumulo.tserver.logger.LogFileValue;
+import org.apache.hadoop.fs.Path;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ *
+ */
+public class AccumuloReplicaSystem implements ReplicaSystem {
+ private static final Logger log = LoggerFactory.getLogger(AccumuloReplicaSystem.class);
+ private static final String RFILE_SUFFIX = "." + RFile.EXTENSION;
+
+ private String instanceName, zookeepers;
+ private AccumuloConfiguration conf;
+ private VolumeManager fs;
+
+ @Override
+ public void configure(String configuration) {
+ Preconditions.checkNotNull(configuration);
+
+ // instance_name,zookeepers
+ int index = configuration.indexOf(',');
+ if (-1 == index) {
+ throw new IllegalArgumentException("Expected comma in configuration string");
+ }
+
+ instanceName = configuration.substring(0, index);
+ zookeepers = configuration.substring(index + 1);
+
+ conf = ServerConfiguration.getSiteConfiguration();
+
+ try {
+ fs = VolumeManagerImpl.get(conf);
+ } catch (IOException e) {
+ log.error("Could not connect to filesystem", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Status replicate(final Path p, final Status status, ReplicationTarget target) {
+ Instance localInstance = HdfsZooInstance.getInstance();
+ AccumuloConfiguration localConf = ServerConfigurationUtil.getConfiguration(localInstance);
+
+ Instance peerInstance = getPeerInstance(target);
+ // Remote identifier is an integer (table id) in this case.
+ final int remoteTableId = Integer.parseInt(target.getRemoteIdentifier());
+
+ // Attempt the replication of this status a number of times before giving up and
+ // trying to replicate it again later some other time.
+ for (int i = 0; i < localConf.getCount(Property.REPLICATION_WORK_ATTEMPTS); i++) {
+ String peerTserver;
+ try {
+ // Ask the master on the remote what TServer we should talk with to replicate the data
+ peerTserver = ReplicationClient.executeCoordinatorWithReturn(peerInstance, new ClientExecReturn<String,ReplicationCoordinator.Client>() {
+
+ @Override
+ public String execute(ReplicationCoordinator.Client client) throws Exception {
+ return client.getServicerAddress(remoteTableId);
+ }
+
+ });
+ } catch (AccumuloException | AccumuloSecurityException e) {
+ // No progress is made
+ log.error("Could not connect to master at {}, cannot proceed with replication. Will retry", target, e);
+ continue;
+ }
+
+ if (null == peerTserver) {
+ // Something went wrong, and we didn't get a valid tserver from the remote for some reason
+ log.warn("Did not receive tserver from master at {}, cannot proceed with replication. Will retry.", target);
+ continue;
+ }
+
+ // We have a tserver on the remote -- send the data its way.
+ Long entriesReplicated;
+ //TODO should chunk up the given file into some configurable sizes instead of just sending the entire file all at once
+ // configuration should probably just be size based.
+ final long sizeLimit = Long.MAX_VALUE;
+ try {
+ entriesReplicated = ReplicationClient.executeServicerWithReturn(peerInstance, peerTserver, new ClientExecReturn<Long,ReplicationServicer.Client>() {
+ @Override
+ public Long execute(Client client) throws Exception {
+ // RFiles have an extension, call everything else a WAL
+ if (p.getName().endsWith(RFILE_SUFFIX)) {
+ return client.replicateKeyValues(remoteTableId, getKeyValues(p, status, sizeLimit));
+ } else {
+ return client.replicateLog(remoteTableId, getWalEdits(p, status, sizeLimit));
+ }
+ }
+ });
+
+ log.debug("Replicated {} entries from {} to {} which is a part of {}", entriesReplicated, p, peerTserver, peerInstance.getInstanceName());
+
+ // Update the begin to account for what we replicated
+ Status updatedStatus = Status.newBuilder(status).setBegin(status.getBegin() + entriesReplicated).build();
+
+ return updatedStatus;
+ } catch (TTransportException | AccumuloException | AccumuloSecurityException e) {
+ log.warn("Could not connect to remote server {}, will retry", peerTserver, e);
+ UtilWaitThread.sleep(250);
+ }
+ }
+
+ // We made no status, punt on it for now, and let it re-queue itself for work
+ return status;
+ }
+
+ protected Instance getPeerInstance(ReplicationTarget target) {
+ return new ZooKeeperInstance(instanceName, zookeepers);
+ }
+
+ protected KeyValues getKeyValues(Path p, Status status, long sizeLimit) {
+ // TODO Implement me
+ throw new UnsupportedOperationException();
+ }
+
+ protected WalEdits getWalEdits(Path p, Status status, long sizeLimit) throws IOException {
+ DFSLoggerInputStreams streams = DfsLogger.readHeaderAndReturnStream(fs, p, conf);
+ DataInputStream wal = streams.getDecryptingInputStream();
+ LogFileKey key = new LogFileKey();
+ LogFileValue value = new LogFileValue();
+
+ // Read through the stuff we don't need to replicate
+ for (long i = 0; i < status.getBegin(); i++) {
+ try {
+ key.readFields(wal);
+ value.readFields(wal);
+ } catch (EOFException e) {
+ log.warn("Unexpectedly reached the end of file. Nothing more to replicate.");
+ return null;
+ }
+ }
+
+ WalEdits edits = new WalEdits();
+ edits.edits = new ArrayList<ByteBuffer>();
+ long size = 0l;
+ while (size < sizeLimit) {
+ try {
+ key.readFields(wal);
+ value.readFields(wal);
+ } catch (EOFException e) {
+ log.trace("Caught EOFException, no more data to replicate");
+ break;
+ }
+
+ switch (key.event) {
+ case MUTATION:
+ case MANY_MUTATIONS:
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(baos);
+ key.write(out);
+ value.write(out);
+ out.flush();
+ byte[] data = baos.toByteArray();
+ size += data.length;
+ edits.addToEdits(ByteBuffer.wrap(data));
+ break;
+ default:
+ log.trace("Ignorning WAL entry which doesn't contain mutations");
+ break;
+ }
+ }
+
+ log.debug("Returning {} bytes of WAL entries for replication for {}", size, p);
+
+ return edits;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
index c0aa31d..84782df 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationProcessor.java
@@ -38,10 +38,10 @@ import org.apache.accumulo.core.replication.ReplicationTarget;
import org.apache.accumulo.core.replication.StatusUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.fs.VolumeManager;
import org.apache.accumulo.server.replication.ReplicationTable;
import org.apache.accumulo.server.replication.ReplicationWorkAssignerHelper;
-import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor;
import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
@@ -56,20 +56,21 @@ import com.google.protobuf.InvalidProtocolBufferException;
public class ReplicationProcessor implements Processor {
private static final Logger log = LoggerFactory.getLogger(ReplicationProcessor.class);
- private Instance inst;
- private AccumuloConfiguration conf;
- private VolumeManager fs;
- private Credentials creds;
+ private final Instance inst;
+ private final AccumuloConfiguration conf;
+ private final VolumeManager fs;
+ private final Credentials creds;
- public ReplicationProcessor(Instance inst, AccumuloConfiguration conf, VolumeManager fs) {
+ public ReplicationProcessor(Instance inst, AccumuloConfiguration conf, VolumeManager fs, Credentials creds) {
+ this.inst = inst;
this.conf = conf;
this.fs = fs;
- creds = SystemCredentials.get();
+ this.creds = creds;
}
@Override
public ReplicationProcessor newProcessor() {
- return new ReplicationProcessor(inst, conf, fs);
+ return new ReplicationProcessor(inst, new ServerConfiguration(inst).getConfiguration(), fs, creds);
}
@Override
@@ -77,14 +78,11 @@ public class ReplicationProcessor implements Processor {
ReplicationTarget target = ReplicationWorkAssignerHelper.fromQueueKey(workID).getValue();
String file = new String(data);
+ log.debug("Received replication work for {} to {}", file, target);
+
// Find the configured replication peer so we know how to replicate to it
- Map<String,String> configuredPeers = conf.getAllPropertiesWithPrefix(Property.REPLICATION_PEERS);
- String peerType = configuredPeers.get(target.getPeerName());
- if (null == peerType) {
- String msg = "Cannot process replication for unknown peer: " + target.getPeerName();
- log.warn(msg);
- throw new IllegalArgumentException(msg);
- }
+ // Classname,Configuration
+ String peerType = getPeerType(target.getPeerName());
// Get the peer that we're replicating to
ReplicaSystem replica = ReplicaSystemFactory.get(peerType);
@@ -126,10 +124,25 @@ public class ReplicationProcessor implements Processor {
recordNewStatus(filePath, replicatedStatus, target);
}
+ log.debug("Did not replicate any new data for {} to {}", filePath, target);
+
// otherwise, we didn't actually replicate because there was error sending the data
// we can just not record any updates, and it will be picked up again by the work assigner
}
+ public String getPeerType(String peerName) {
+ // Find the configured replication peer so we know how to replicate to it
+ Map<String,String> configuredPeers = conf.getAllPropertiesWithPrefix(Property.REPLICATION_PEERS);
+ String peerType = configuredPeers.get(Property.REPLICATION_PEERS.getKey() + peerName);
+ if (null == peerType) {
+ String msg = "Cannot process replication for unknown peer: " + peerName;
+ log.warn(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ return peerType;
+ }
+
public Status getStatus(String file, ReplicationTarget target) throws TableNotFoundException, AccumuloException, AccumuloSecurityException, InvalidProtocolBufferException {
Scanner s = ReplicationTable.getScanner(inst.getConnector(creds.getPrincipal(), creds.getToken()));
s.setRange(Range.exact(file));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
new file mode 100644
index 0000000..5036cab
--- /dev/null
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationServicerHandler.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
+import org.apache.accumulo.core.replication.thrift.KeyValues;
+import org.apache.accumulo.core.replication.thrift.RemoteReplicationException;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer.Iface;
+import org.apache.accumulo.core.replication.thrift.WalEdits;
+import org.apache.thrift.TException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class ReplicationServicerHandler implements Iface {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationServicerHandler.class);
+
+ @Override
+ public long replicateLog(int remoteTableId, WalEdits data) throws RemoteReplicationException, TException {
+ log.error("Got replication request to tableID {} with {} edits", remoteTableId, data.getEditsSize());
+ return data.getEditsSize();
+ }
+
+ @Override
+ public long replicateKeyValues(int remoteTableId, KeyValues data) throws RemoteReplicationException, TException {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
index e00f343..63bc261 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/replication/ReplicationWorker.java
@@ -23,6 +23,7 @@ import org.apache.accumulo.core.client.Instance;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.server.fs.VolumeManager;
+import org.apache.accumulo.server.security.SystemCredentials;
import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
import org.apache.zookeeper.KeeperException;
@@ -49,7 +50,7 @@ public class ReplicationWorker implements Runnable {
@Override
public void run() {
try {
- new DistributedWorkQueue(ZooUtil.getRoot(inst) + Constants.ZREPLICATION, conf).startProcessing(new ReplicationProcessor(inst, conf, fs), executor);
+ new DistributedWorkQueue(ZooUtil.getRoot(inst) + Constants.ZREPLICATION, conf).startProcessing(new ReplicationProcessor(inst, conf, fs, SystemCredentials.get()), executor);
} catch (KeeperException | InterruptedException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
new file mode 100644
index 0000000..ae05800
--- /dev/null
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/ReplicationProcessorTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.tserver.replication;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.server.fs.VolumeManager;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class ReplicationProcessorTest {
+
+ @Test
+ public void peerTypeExtractionFromConfiguration() {
+ Instance inst = EasyMock.createMock(Instance.class);
+ VolumeManager fs = EasyMock.createMock(VolumeManager.class);
+ Credentials creds = new Credentials("foo", new PasswordToken("bar"));
+
+ Map<String,String> data = new HashMap<>();
+
+ String peerName = "peer";
+ String configuration = "java.lang.String,foo";
+ data.put(Property.REPLICATION_PEERS + peerName, configuration);
+ ConfigurationCopy conf = new ConfigurationCopy(data);
+
+ ReplicationProcessor proc = new ReplicationProcessor(inst, conf, fs, creds);
+
+ Assert.assertEquals(configuration, proc.getPeerType(peerName));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void noPeerConfigurationThrowsAnException() {
+ Instance inst = EasyMock.createMock(Instance.class);
+ VolumeManager fs = EasyMock.createMock(VolumeManager.class);
+ Credentials creds = new Credentials("foo", new PasswordToken("bar"));
+
+ Map<String,String> data = new HashMap<>();
+ ConfigurationCopy conf = new ConfigurationCopy(data);
+
+ ReplicationProcessor proc = new ReplicationProcessor(inst, conf, fs, creds);
+
+ proc.getPeerType("foo");
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/3579b674/server/tserver/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/server/tserver/src/test/resources/log4j.properties b/server/tserver/src/test/resources/log4j.properties
new file mode 100644
index 0000000..71c06e9
--- /dev/null
+++ b/server/tserver/src/test/resources/log4j.properties
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+log4j.rootLogger=DEBUG, CA
+log4j.appender.CA=org.apache.log4j.ConsoleAppender
+log4j.appender.CA.layout=org.apache.log4j.PatternLayout
+log4j.appender.CA.layout.ConversionPattern=%d{ISO8601} [%c{2}] %-5p: %m%n
+
+log4j.logger.org.apache.accumulo.core=DEBUG
+log4j.logger.org.apache.accumulo.core.client.impl.ServerClient=ERROR
+log4j.logger.org.apache.accumulo.core.util.shell.Shell.audit=off
+log4j.logger.org.apache.accumulo.core.util.shell.Shell=FATAL
+log4j.logger.org.apache.commons.vfs2.impl.DefaultFileSystemManager=WARN
+log4j.logger.org.apache.hadoop.io.compress.CodecPool=WARN
+log4j.logger.org.apache.hadoop.mapred=ERROR
+log4j.logger.org.apache.hadoop.tools.DistCp=WARN
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.util.ProcessTree=WARN
+log4j.logger.org.apache.zookeeper.ClientCnxn=FATAL
+log4j.logger.org.apache.zookeeper.ZooKeeper=WARN
+log4j.logger.org.apache.accumulo.core.file.rfile.bcfile=INFO
+log4j.logger.org.apache.accumulo.server.util.ReplicationTableUtil=TRACE
+log4j.logger.org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator=INFO
+log4j.logger.org.apache.accumulo.core.client.impl.ThriftScanner=INFO
[08/50] [abbrv] git commit: ACCUMULO-2581 Initial impl to use
DistributedWorkQueue to assign replication work.
Posted by el...@apache.org.
ACCUMULO-2581 Initial impl to use DistributedWorkQueue to assign replication work.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/31f4e83b
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/31f4e83b
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/31f4e83b
Branch: refs/heads/ACCUMULO-378
Commit: 31f4e83b94c4f150df2639f251d6835305fa531d
Parents: f584685
Author: Josh Elser <el...@apache.org>
Authored: Thu May 1 16:58:47 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu May 1 16:58:47 2014 -0400
----------------------------------------------------------------------
.../org/apache/accumulo/core/Constants.java | 2 +
.../org/apache/accumulo/core/conf/Property.java | 6 +-
.../accumulo/core/replication/StatusUtil.java | 24 ++
server/master/pom.xml | 5 +
.../replication/ReplicationWorkAssigner.java | 264 +++++++++++++++++++
.../accumulo/master/replication/Work.java | 98 +++++++
.../ReplicationWorkAssignerTest.java | 217 +++++++++++++++
.../accumulo/master/replication/WorkTest.java | 49 ++++
8 files changed, 664 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/core/src/main/java/org/apache/accumulo/core/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/Constants.java b/core/src/main/java/org/apache/accumulo/core/Constants.java
index 7d602bb..c87119c 100644
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@ -113,4 +113,6 @@ public class Constants {
public static final String[] PATH_PROPERTY_ENV_VARS = new String[] {"ACCUMULO_HOME", "ACCUMULO_CONF_DIR"};
public static final String HDFS_TABLES_DIR = "/tables";
+
+ public static final String ZREPLICATION = "/replication";
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 2ad1369..637600d 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -454,7 +454,11 @@ public enum Property {
REPLICATION_BATCH_SIZE("replication.batch.size", "1000", PropertyType.COUNT, "Maximum number of updates (WAL) or key-value pairs (RFile) to send in one replication task"),
@Experimental
REPLICATION_SEND_THREAD_POOL_SIZE("replication.send.threads", "1", PropertyType.COUNT, "Size of threadpool used to start replication to slaves"),
-
+ @Experimental
+ REPLICATION_MAX_WORK_QUEUE("replication.max.work.queue", "20000000", PropertyType.COUNT, "Upper bound of the number of files queued for replication"),
+ @Experimental
+ REPLICATION_WORK_ASSIGNMENT_SLEEP("replication.work.assignment.sleep", "30s", PropertyType.TIMEDURATION, "Amount of time to sleep between replication work assignment"),
+
;
private String key, defaultValue, description;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
index 3b8ccb6..74327fd 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/StatusUtil.java
@@ -21,6 +21,8 @@ import org.apache.accumulo.core.protobuf.ProtobufUtil;
import org.apache.accumulo.core.replication.proto.Replication.Status;
import org.apache.accumulo.core.replication.proto.Replication.Status.Builder;
+import com.google.protobuf.InvalidProtocolBufferException;
+
/**
* Helper methods to create Status protobuf messages
*/
@@ -161,6 +163,15 @@ public class StatusUtil {
}
/**
+ * @param v Value with serialized Status
+ * @return A Status created from the Value
+ * @throws InvalidProtocolBufferException
+ */
+ public static Status fromValue(Value v) throws InvalidProtocolBufferException {
+ return Status.parseFrom(v.get());
+ }
+
+ /**
* Is the given Status fully replicated and is its file ready for deletion on the source
* @param status a Status protobuf
* @return True if the file this Status references can be deleted.
@@ -181,4 +192,17 @@ public class StatusUtil {
return status.getBegin() >= status.getEnd();
}
}
+
+ /**
+ * Given the {@link Status}, is there replication work to be done
+ * @param status Status for a file
+ * @return true if replication work is required
+ */
+ public static boolean isWorkRequired(Status status) {
+ if (status.getInfiniteEnd()) {
+ return Long.MAX_VALUE == status.getBegin();
+ } else {
+ return status.getBegin() < status.getEnd();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/server/master/pom.xml
----------------------------------------------------------------------
diff --git a/server/master/pom.xml b/server/master/pom.xml
index f9b2990..3b9684c 100644
--- a/server/master/pom.xml
+++ b/server/master/pom.xml
@@ -84,5 +84,10 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.easymock</groupId>
+ <artifactId>easymock</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
new file mode 100644
index 0000000..f3a8825
--- /dev/null
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/ReplicationWorkAssigner.java
@@ -0,0 +1,264 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.accumulo.server.zookeeper.ZooCache;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * Read work records from the replication table, create work entries for other nodes to complete.
+ * <p>
+ * Uses the DistributedWorkQueue to make the work available for any tserver. This approach does not consider the locality of the tabletserver performing the
+ * work in relation to the data being replicated (local HDFS blocks).
+ */
+public class ReplicationWorkAssigner implements Runnable {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationWorkAssigner.class);
+
+ private Master master;
+ private Connector conn;
+
+ private AccumuloConfiguration conf;
+ private DistributedWorkQueue workQueue;
+ private Set<String> queuedWork;
+ private int maxQueueSize;
+ private ZooCache zooCache;
+
+ public ReplicationWorkAssigner(Master master, Connector conn) {
+ this.master = master;
+ this.conn = conn;
+ }
+
+ /*
+ * Getters/setters for testing purposes
+ */
+ protected Connector getConnector() {
+ return conn;
+ }
+
+ protected void setConnector(Connector conn) {
+ this.conn = conn;
+ }
+
+ protected AccumuloConfiguration getConf() {
+ return conf;
+ }
+
+ protected void setConf(AccumuloConfiguration conf) {
+ this.conf = conf;
+ }
+
+ protected DistributedWorkQueue getWorkQueue() {
+ return workQueue;
+ }
+
+ protected void setWorkQueue(DistributedWorkQueue workQueue) {
+ this.workQueue = workQueue;
+ }
+
+ protected Set<String> getQueuedWork() {
+ return queuedWork;
+ }
+
+ protected void setQueuedWork(Set<String> queuedWork) {
+ this.queuedWork = queuedWork;
+ }
+
+ protected int getMaxQueueSize() {
+ return maxQueueSize;
+ }
+
+ protected void setMaxQueueSize(int maxQueueSize) {
+ this.maxQueueSize = maxQueueSize;
+ }
+
+ protected ZooCache getZooCache() {
+ return zooCache;
+ }
+
+ protected void setZooCache(ZooCache zooCache) {
+ this.zooCache = zooCache;
+ }
+
+ /**
+ * Initialize the DistributedWorkQueue using the proper ZK location
+ *
+ * @param conf
+ */
+ protected void initializeWorkQueue(AccumuloConfiguration conf) {
+ workQueue = new DistributedWorkQueue(ZooUtil.getRoot(master.getInstance()) + Constants.ZREPLICATION, conf);
+ }
+
+ /**
+ * Initialize the queuedWork set with the work already sent out
+ */
+ protected void initializeQueuedWork() {
+ Preconditions.checkArgument(null == queuedWork, "Expected queuedWork to be null");
+ queuedWork = new HashSet<>();
+ try {
+ queuedWork.addAll(workQueue.getWorkQueued());
+ } catch (KeeperException | InterruptedException e) {
+ throw new RuntimeException("Error reading existing queued replication work", e);
+ }
+ }
+
+ @Override
+ public void run() {
+ while (master.stillMaster()) {
+ if (null == conf) {
+ conf = master.getConfiguration().getConfiguration();
+ }
+
+ // Get the maximum number of entries we want to queue work for (or the default)
+ maxQueueSize = conf.getCount(Property.REPLICATION_MAX_WORK_QUEUE);
+
+ if (null == workQueue) {
+ initializeWorkQueue(conf);
+ }
+
+ if (null == queuedWork) {
+ initializeQueuedWork();
+ }
+
+ if (null == zooCache) {
+ zooCache = new ZooCache();
+ }
+
+ // Scan over the work records, adding the work to the queue
+ createWork();
+
+ // Keep the state of the work we queued correct
+ cleanupFinishedWork();
+ }
+ }
+
+ /**
+ * Scan over the {@link WorkSection} of the replication table adding work for entries that
+ * have data to replicate and have not already been queued.
+ */
+ protected void createWork() {
+ // Create a batchscanner over the replication table's work entries
+ BatchScanner bs;
+ try {
+ bs = ReplicationTable.getBatchScanner(conn, 4);
+ } catch (TableNotFoundException e) {
+ log.warn("Could not find replication table", e);
+ return;
+ }
+
+ WorkSection.limit(bs);
+ bs.setRanges(Collections.singleton(new Range()));
+ Text buffer = new Text();
+ try {
+ for (Entry<Key,Value> entry : bs) {
+ // If we're not working off the entries, we need to not shoot ourselves in the foot by continuing
+ // to add more work entries
+ if (queuedWork.size() > maxQueueSize) {
+ log.warn("Queued replication work exceeds configured maximum ({}), sleeping to allow work to occur", maxQueueSize);
+ return;
+ }
+
+ WorkSection.getFile(entry.getKey(), buffer);
+ String file = buffer.toString();
+ Status status;
+ try {
+ status = StatusUtil.fromValue(entry.getValue());
+ } catch (InvalidProtocolBufferException e) {
+ log.warn("Could not deserialize protobuf from work entry for {}", file, e);
+ continue;
+ }
+
+ // If there is work to do
+ if (StatusUtil.isWorkRequired(status)) {
+ Path p = new Path(file);
+ String filename = p.getName();
+
+ // And, we haven't already queued this file up for work already
+ if (!queuedWork.contains(filename)) {
+ entry.getKey().getColumnQualifier(buffer);
+ queueWork(file, filename, buffer);
+ }
+ }
+ }
+ } finally {
+ if (null != bs) {
+ bs.close();
+ }
+ }
+ }
+
+ /**
+ * Distribute the work for the given path with filename
+ *
+ * @param path
+ * Full path to a file
+ * @param filename
+ * Filename
+ */
+ protected void queueWork(String path, String filename, Text serializedTarget) {
+ try {
+ String key = filename + "|" + serializedTarget;
+ workQueue.addWork(key, path.getBytes(StandardCharsets.UTF_8));
+ queuedWork.add(key);
+ } catch (KeeperException | InterruptedException e) {
+ log.warn("Could not queue work for {}", path, e);
+ }
+ }
+
+ /**
+ * Iterate over the queued work to remove entries that have been completed.
+ */
+ protected void cleanupFinishedWork() {
+ Iterator<String> work = queuedWork.iterator();
+ while (work.hasNext()) {
+ String filename = work.next();
+ // Null equates to the work was finished
+ if (null == zooCache.get(ZooUtil.getRoot(master.getInstance()) + Constants.ZREPLICATION + "/" + filename)) {
+ work.remove();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/server/master/src/main/java/org/apache/accumulo/master/replication/Work.java
----------------------------------------------------------------------
diff --git a/server/master/src/main/java/org/apache/accumulo/master/replication/Work.java b/server/master/src/main/java/org/apache/accumulo/master/replication/Work.java
new file mode 100644
index 0000000..cb45b44
--- /dev/null
+++ b/server/master/src/main/java/org/apache/accumulo/master/replication/Work.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+import com.google.protobuf.TextFormat;
+
+/**
+ * Encapsulates a file (path) and {@link Status}
+ */
+public class Work implements Writable {
+
+ private String file;
+
+ private Status status;
+
+ public Work() { }
+
+ public Work(String file, Status status) {
+ this.file = file;
+ this.status = status;
+ }
+
+ public String getFile() {
+ return file;
+ }
+
+ public void setFile(String file) {
+ this.file = file;
+ }
+
+ public Status getStatus() {
+ return status;
+ }
+
+ public void setStatus(Status status) {
+ this.status = status;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ WritableUtils.writeString(out, file);
+ byte[] bytes = status.toByteArray();
+ WritableUtils.writeVInt(out, bytes.length);
+ out.write(bytes);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ file = WritableUtils.readString(in);
+ int len = WritableUtils.readVInt(in);
+ byte[] bytes = new byte[len];
+ in.readFully(bytes);
+ status = Status.parseFrom(bytes);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o instanceof Work) {
+ Work other = (Work) o;
+ return file.equals(other.getFile()) && status.equals(other.getStatus());
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return file + " " + TextFormat.shortDebugString(status);
+ }
+
+ @Override
+ public int hashCode() {
+ return file.hashCode() ^ status.hashCode();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
new file mode 100644
index 0000000..b596126
--- /dev/null
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/ReplicationWorkAssignerTest.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.mock.MockInstance;
+import org.apache.accumulo.core.client.security.tokens.PasswordToken;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.replication.ReplicationSchema.WorkSection;
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.master.Master;
+import org.apache.accumulo.server.replication.ReplicationTable;
+import org.apache.accumulo.server.zookeeper.DistributedWorkQueue;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+/**
+ *
+ */
+public class ReplicationWorkAssignerTest {
+
+ @Rule
+ public TestName test = new TestName();
+
+ private Master master;
+ private Connector conn;
+ private ReplicationWorkAssigner assigner;
+
+ @Before
+ public void init() {
+ master = createMock(Master.class);
+ conn = createMock(Connector.class);
+ assigner = new ReplicationWorkAssigner(master, conn);
+ }
+
+ @Test
+ public void workQueuedUsingFileName() throws Exception {
+ ReplicationTarget target = new ReplicationTarget("cluster1", "table1");
+ Text serializedTarget = ReplicationTarget.toText(target);
+
+ DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
+ Set<String> queuedWork = new HashSet<>();
+ assigner.setQueuedWork(queuedWork);
+ assigner.setWorkQueue(workQueue);
+
+ Path p = new Path("/accumulo/wal/tserver+port/" + UUID.randomUUID());
+ assigner.queueWork(p.toString(), p.getName(), serializedTarget);
+
+ workQueue.addWork(p.getName(), p.toString().getBytes(StandardCharsets.UTF_8));
+ expectLastCall().once();
+
+ replay(workQueue);
+
+ Assert.assertEquals(1, queuedWork.size());
+ Assert.assertEquals(p.getName() + "|" + serializedTarget, queuedWork.iterator().next());
+ }
+
+ @Test
+ public void existingWorkIsReQueued() throws Exception {
+ DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
+
+ List<String> existingWork = Arrays.asList("/accumulo/wal/tserver+port/wal1", "/accumulo/wal/tserver+port/wal2");
+ expect(workQueue.getWorkQueued()).andReturn(existingWork);
+
+ replay(workQueue);
+
+ assigner.setWorkQueue(workQueue);
+ assigner.initializeQueuedWork();
+
+ Set<String> queuedWork = assigner.getQueuedWork();
+ Assert.assertEquals("Expected existing work and queued work to be the same size", existingWork.size(), queuedWork.size());
+ Assert.assertTrue("Expected all existing work to be queued", queuedWork.containsAll(existingWork));
+ }
+
+ @Test
+ public void createWorkForFilesNeedingIt() throws Exception {
+ ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
+ Text serializedTarget1 = ReplicationTarget.toText(target1), serializedTarget2 = ReplicationTarget.toText(target2);
+
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Credentials creds = new Credentials("root", new PasswordToken(""));
+ Connector conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+
+ // Set the connector
+ assigner.setConnector(conn);
+
+ // Create and grant ourselves write to the replication table
+ ReplicationTable.create(conn);
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE);
+
+ // Create two mutations, both of which need replication work done
+ BatchWriter bw = ReplicationTable.getBatchWriter(conn);
+ String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString();
+ String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
+ Mutation m = new Mutation(file1);
+ WorkSection.add(m, serializedTarget1, StatusUtil.openWithUnknownLengthValue());
+ bw.addMutation(m);
+
+ m = new Mutation(file2);
+ WorkSection.add(m, serializedTarget2, StatusUtil.openWithUnknownLengthValue());
+ bw.addMutation(m);
+
+ bw.close();
+
+ DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
+ @SuppressWarnings("unchecked")
+ HashSet<String> queuedWork = createMock(HashSet.class);
+ assigner.setQueuedWork(queuedWork);
+
+ expect(queuedWork.size()).andReturn(0).anyTimes();
+
+ // Make sure we expect the invocations in the correct order (accumulo is sorted)
+ if (file1.compareTo(file2) <= 0) {
+ String key = filename1 + "|" + serializedTarget1;
+ workQueue.addWork(key, file1.getBytes(StandardCharsets.UTF_8));
+ expectLastCall().once();
+ expect(queuedWork.add(key)).andReturn(true).once();
+
+ key = filename2 + "|" + serializedTarget2;
+ workQueue.addWork(key, file2.getBytes(StandardCharsets.UTF_8));
+ expectLastCall().once();
+ expect(queuedWork.add(key)).andReturn(true).once();
+ } else {
+ String key = filename2 + "|" + serializedTarget2;
+ workQueue.addWork(key, file2.getBytes(StandardCharsets.UTF_8));
+ expectLastCall().once();
+ expect(queuedWork.add(key)).andReturn(true).once();
+
+ key = filename1 + "|" + serializedTarget1;
+ workQueue.addWork(key, file1.getBytes(StandardCharsets.UTF_8));
+ expectLastCall().once();
+ expect(queuedWork.add(key)).andReturn(true).once();
+ }
+
+ replay(queuedWork, workQueue);
+
+ assigner.createWork();
+ }
+
+ @Test
+ public void doNotCreateWorkForFilesNotNeedingIt() throws Exception {
+ ReplicationTarget target1 = new ReplicationTarget("cluster1", "table1"), target2 = new ReplicationTarget("cluster1", "table2");
+ Text serializedTarget1 = ReplicationTarget.toText(target1), serializedTarget2 = ReplicationTarget.toText(target2);
+
+ MockInstance inst = new MockInstance(test.getMethodName());
+ Credentials creds = new Credentials("root", new PasswordToken(""));
+ Connector conn = inst.getConnector(creds.getPrincipal(), creds.getToken());
+
+ // Set the connector
+ assigner.setConnector(conn);
+
+ // Create and grant ourselves write to the replication table
+ ReplicationTable.create(conn);
+ conn.securityOperations().grantTablePermission("root", ReplicationTable.NAME, TablePermission.WRITE);
+
+ // Create two mutations, both of which need replication work done
+ BatchWriter bw = ReplicationTable.getBatchWriter(conn);
+ String filename1 = UUID.randomUUID().toString(), filename2 = UUID.randomUUID().toString();
+ String file1 = "/accumulo/wal/tserver+port/" + filename1, file2 = "/accumulo/wal/tserver+port/" + filename2;
+ Mutation m = new Mutation(file1);
+ WorkSection.add(m, serializedTarget1, StatusUtil.newFileValue());
+ bw.addMutation(m);
+
+ m = new Mutation(file2);
+ WorkSection.add(m, serializedTarget2, StatusUtil.newFileValue());
+ bw.addMutation(m);
+
+ bw.close();
+
+ DistributedWorkQueue workQueue = createMock(DistributedWorkQueue.class);
+ @SuppressWarnings("unchecked")
+ HashSet<String> queuedWork = createMock(HashSet.class);
+ assigner.setQueuedWork(queuedWork);
+
+ expect(queuedWork.size()).andReturn(0).anyTimes();
+
+ replay(queuedWork, workQueue);
+
+ assigner.createWork();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/31f4e83b/server/master/src/test/java/org/apache/accumulo/master/replication/WorkTest.java
----------------------------------------------------------------------
diff --git a/server/master/src/test/java/org/apache/accumulo/master/replication/WorkTest.java b/server/master/src/test/java/org/apache/accumulo/master/replication/WorkTest.java
new file mode 100644
index 0000000..09530a7
--- /dev/null
+++ b/server/master/src/test/java/org/apache/accumulo/master/replication/WorkTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.master.replication;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.accumulo.core.replication.StatusUtil;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class WorkTest {
+
+ @Test
+ public void serialization() throws IOException {
+ Work w = new Work("/foo/bar", StatusUtil.openWithUnknownLength());
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ w.write(new DataOutputStream(baos));
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+
+ Work newW = new Work();
+ newW.readFields(new DataInputStream(bais));
+
+ Assert.assertEquals(w, newW);
+ }
+
+}
[32/50] [abbrv] git commit: ACCUMULO-2583 First stab at setting up
the actual wire transfer to the replication peer
Posted by el...@apache.org.
ACCUMULO-2583 First stab at setting up the actual wire transfer to the replication peer
Changed a few things in the thrift idl to make a little more sense. Use the protobuf
Status message that we already have as the response object to the replication. This
allows the peer to not have to process the entire message at one time as an optimization (later).
Made a ReplicationClient in the same spirit as the MasterClient, but with a few notable
changes. Since we're operating outside of the local instance, we don't want to make
the same assertions about availability and blocking until we get a connection as is
the case in talking to the master. This pushes a bit of the logic into the replication
setup code, but this is desirable as we can better handle failure/retry logic as we intend.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/baa7a4f7
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/baa7a4f7
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/baa7a4f7
Branch: refs/heads/ACCUMULO-378
Commit: baa7a4f73739e652d72a47873da967eb2b7dd199
Parents: 59ecd21
Author: Josh Elser <el...@apache.org>
Authored: Wed May 7 00:16:42 2014 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Wed May 7 00:16:42 2014 -0400
----------------------------------------------------------------------
.../apache/accumulo/core/client/Connector.java | 9 +
.../client/admin/ReplicationOperations.java | 52 +
.../core/client/impl/ConnectorImpl.java | 12 +
.../accumulo/core/client/impl/MasterClient.java | 1 +
.../core/client/impl/ReplicationClient.java | 208 ++
.../client/impl/ReplicationOperationsImpl.java | 82 +
.../core/client/mock/MockConnector.java | 7 +
.../client/replication/PeerExistsException.java | 36 +
.../replication/PeerNotFoundException.java | 36 +
.../core/client/replication/ReplicaSystem.java | 45 +
.../replication/ReplicaSystemFactory.java | 61 +
.../org/apache/accumulo/core/conf/Property.java | 2 +-
.../core/replication/ReplicationTarget.java | 50 +-
.../thrift/RemoteReplicationCoordinator.java | 1010 ---------
.../thrift/RemoteReplicationServicer.java | 1917 ----------------
.../thrift/ReplicationCoordinator.java | 1010 +++++++++
.../replication/thrift/ReplicationServicer.java | 2145 ++++++++++++++++++
core/src/main/thrift/replication.thrift | 8 +-
.../core/replication/AccumuloReplicaSystem.java | 124 +
.../core/replication/ReplicationTargetTest.java | 13 +-
.../ReplicationWorkAssignerHelper.java | 69 +
.../MasterReplicationCoordinator.java | 4 +-
.../replication/ReplicationWorkAssigner.java | 10 +-
.../accumulo/master/replication/WorkMaker.java | 2 +-
.../ReplicationWorkAssignerTest.java | 9 +-
.../replication/ReplicationProcessor.java | 130 +-
.../tserver/replication/ReplicationWorker.java | 2 +-
.../test/replication/ReplicationWithGCIT.java | 2 +-
.../replication/ReplicationWithMakerTest.java | 2 +-
29 files changed, 4090 insertions(+), 2968 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/Connector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/Connector.java b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
index 4a2acff..301577c 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/Connector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/Connector.java
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client;
import org.apache.accumulo.core.client.admin.InstanceOperations;
import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.admin.ReplicationOperations;
import org.apache.accumulo.core.client.admin.SecurityOperations;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.security.Authorizations;
@@ -228,4 +229,12 @@ public abstract class Connector {
* @return an object to modify instance configuration
*/
public abstract InstanceOperations instanceOperations();
+
+ /**
+ * Retrieves a ReplicationOperations object to manage replication configuration.
+ *
+ * @return an object to modify replication configuration
+ * @since 1.7.0
+ */
+ public abstract ReplicationOperations replicationOperations();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/admin/ReplicationOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/ReplicationOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/ReplicationOperations.java
new file mode 100644
index 0000000..22cfcaf
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/ReplicationOperations.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.admin;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.replication.PeerExistsException;
+import org.apache.accumulo.core.client.replication.PeerNotFoundException;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+
+/**
+ * Supports replication configuration
+ * @since 1.7.0
+ */
+public interface ReplicationOperations {
+
+ /**
+ * Define a cluster with the given name using the given {@link ReplicaSystem}
+ * @param name Name of the cluster, used for configuring replication on tables
+ * @param system Type of system to be replicated to
+ */
+ public void addPeer(String name, ReplicaSystem system) throws AccumuloException, AccumuloSecurityException, PeerExistsException;
+
+ /**
+ * Define a cluster with the given name and the given name system
+ * @param name Unique name for the cluster
+ * @param replicaType {@link ReplicaSystem} class name to use to replicate the data
+ * @throws PeerExistsException
+ */
+ public void addPeer(String name, String replicaType) throws AccumuloException, AccumuloSecurityException, PeerExistsException;
+
+ /**
+ * Remove a cluster with the given name
+ * @param name Name of the cluster to remove
+ * @throws PeerNotFoundException
+ */
+ public void removePeer(String name) throws AccumuloException, AccumuloSecurityException, PeerNotFoundException;
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
index 0df35f6..e22be52 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ConnectorImpl.java
@@ -19,6 +19,7 @@ package org.apache.accumulo.core.client.impl;
import java.util.concurrent.TimeUnit;
import static com.google.common.base.Preconditions.checkArgument;
+
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.BatchDeleter;
@@ -35,6 +36,7 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.TableOfflineException;
import org.apache.accumulo.core.client.admin.InstanceOperations;
import org.apache.accumulo.core.client.admin.NamespaceOperations;
+import org.apache.accumulo.core.client.admin.ReplicationOperations;
import org.apache.accumulo.core.client.admin.SecurityOperations;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.client.impl.thrift.ClientService;
@@ -51,6 +53,7 @@ public class ConnectorImpl extends Connector {
private TableOperations tableops = null;
private NamespaceOperations namespaceops = null;
private InstanceOperations instanceops = null;
+ private ReplicationOperations replicationops = null;
public ConnectorImpl(final Instance instance, Credentials cred) throws AccumuloException, AccumuloSecurityException {
checkArgument(instance != null, "instance is null");
@@ -182,4 +185,13 @@ public class ConnectorImpl extends Connector {
return instanceops;
}
+
+ @Override
+ public synchronized ReplicationOperations replicationOperations() {
+ if (null == replicationops) {
+ replicationops = new ReplicationOperationsImpl(instance, credentials);
+ }
+
+ return replicationops;
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java b/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
index 1fa8f12..9807a82 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/MasterClient.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.core.client.impl;
import static com.google.common.base.Preconditions.checkArgument;
+
import java.net.UnknownHostException;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
new file mode 100644
index 0000000..df12ae8
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationClient.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.net.UnknownHostException;
+import java.util.List;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.client.impl.thrift.ThriftTableOperationException;
+import org.apache.accumulo.core.replication.thrift.ReplicationCoordinator;
+import org.apache.accumulo.core.replication.thrift.ReplicationServicer;
+import org.apache.accumulo.core.util.ThriftUtil;
+import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.thrift.TServiceClient;
+import org.apache.thrift.transport.TTransportException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReplicationClient {
+ private static final Logger log = LoggerFactory.getLogger(ReplicationClient.class);
+
+ /**
+ * @param instance
+ * Instance for the peer replicant
+ * @return Client to the ReplicationCoordinator service
+ */
+ public static ReplicationCoordinator.Client getCoordinatorConnectionWithRetry(Instance instance) {
+ checkArgument(instance != null, "instance is null");
+
+ while (true) {
+
+ ReplicationCoordinator.Client result = getCoordinatorConnection(instance);
+ if (result != null)
+ return result;
+ UtilWaitThread.sleep(250);
+ }
+
+ }
+
+ public static ReplicationCoordinator.Client getCoordinatorConnection(Instance instance) {
+ List<String> locations = instance.getMasterLocations();
+
+ if (locations.size() == 0) {
+ log.debug("No masters...");
+ return null;
+ }
+
+ String master = locations.get(0);
+ if (master.endsWith(":0"))
+ return null;
+
+ try {
+ // Master requests can take a long time: don't ever time out
+ ReplicationCoordinator.Client client = ThriftUtil.getClientNoTimeout(new ReplicationCoordinator.Client.Factory(), master,
+ ServerConfigurationUtil.getConfiguration(instance));
+ return client;
+ } catch (TTransportException tte) {
+ if (tte.getCause().getClass().equals(UnknownHostException.class)) {
+ // do not expect to recover from this
+ throw new RuntimeException(tte);
+ }
+ log.debug("Failed to connect to master={}, will retry... ", master, tte);
+ return null;
+ }
+ }
+
+ /**
+ * Attempt a single time to create a ReplicationServicer client to the given host
+ *
+ * @param inst
+ * Instance to the peer replicant
+ * @param server
+ * Server to connect to
+ * @return A ReplicationServicer client to the given host in the given instance
+ */
+ public static ReplicationServicer.Client getServicerConnection(Instance inst, String server) throws TTransportException {
+ checkNotNull(inst);
+ checkNotNull(server);
+
+ try {
+ return ThriftUtil.getClientNoTimeout(new ReplicationServicer.Client.Factory(), server, ServerConfigurationUtil.getConfiguration(inst));
+ } catch (TTransportException tte) {
+ log.debug("Failed to connect to servicer ({}), will retry...", tte);
+ throw tte;
+ }
+ }
+
+ public static void close(ReplicationCoordinator.Iface iface) {
+ TServiceClient client = (TServiceClient) iface;
+ if (client != null && client.getInputProtocol() != null && client.getInputProtocol().getTransport() != null) {
+ ThriftTransportPool.getInstance().returnTransport(client.getInputProtocol().getTransport());
+ } else {
+ log.debug("Attempt to close null connection to the remote system", new Exception());
+ }
+ }
+
+ public static void close(ReplicationServicer.Iface iface) {
+ TServiceClient client = (TServiceClient) iface;
+ if (client != null && client.getInputProtocol() != null && client.getInputProtocol().getTransport() != null) {
+ ThriftTransportPool.getInstance().returnTransport(client.getInputProtocol().getTransport());
+ } else {
+ log.debug("Attempt to close null connection to the remote system", new Exception());
+ }
+ }
+
+ public static <T> T executeCoordinatorWithReturn(Instance instance, ClientExecReturn<T,ReplicationCoordinator.Client> exec) throws AccumuloException,
+ AccumuloSecurityException {
+ ReplicationCoordinator.Client client = null;
+ while (true) {
+ try {
+ client = getCoordinatorConnectionWithRetry(instance);
+ return exec.execute(client);
+ } catch (TTransportException tte) {
+ log.debug("ReplicationClient coordinator request failed, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+ }
+
+ public static void executeCoordinator(Instance instance, ClientExec<ReplicationCoordinator.Client> exec) throws AccumuloException, AccumuloSecurityException {
+ ReplicationCoordinator.Client client = null;
+ try {
+ client = getCoordinatorConnectionWithRetry(instance);
+ exec.execute(client);
+ } catch (TTransportException tte) {
+ log.debug("ReplicationClient coordinator request failed, retrying ... ", tte);
+ UtilWaitThread.sleep(100);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+
+ public static <T> T executeServicerWithReturn(Instance instance, String tserver, ClientExecReturn<T,ReplicationServicer.Client> exec)
+ throws AccumuloException, AccumuloSecurityException, TTransportException {
+ ReplicationServicer.Client client = null;
+ while (true) {
+ try {
+ client = getServicerConnection(instance, tserver);
+ return exec.execute(client);
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+ }
+
+ public static void executeServicer(Instance instance, String tserver, ClientExec<ReplicationServicer.Client> exec) throws AccumuloException,
+ AccumuloSecurityException, TTransportException {
+ ReplicationServicer.Client client = null;
+ try {
+ client = getServicerConnection(instance, tserver);
+ exec.execute(client);
+ return;
+ } catch (ThriftSecurityException e) {
+ throw new AccumuloSecurityException(e.user, e.code, e);
+ } catch (AccumuloException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new AccumuloException(e);
+ } finally {
+ if (client != null)
+ close(client);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
new file mode 100644
index 0000000..5bbb1c1
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ReplicationOperationsImpl.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.admin.ReplicationOperations;
+import org.apache.accumulo.core.client.replication.PeerExistsException;
+import org.apache.accumulo.core.client.replication.PeerNotFoundException;
+import org.apache.accumulo.core.client.replication.ReplicaSystem;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.master.thrift.MasterClientService.Client;
+import org.apache.accumulo.core.security.Credentials;
+import org.apache.accumulo.trace.instrument.Tracer;
+
+/**
+ *
+ */
+public class ReplicationOperationsImpl implements ReplicationOperations {
+
+ private Instance inst;
+ private Credentials creds;
+
+ public ReplicationOperationsImpl(Instance inst, Credentials creds) {
+ this.inst = inst;
+ this.creds = creds;
+ }
+
+ @Override
+ public void addPeer(String name, ReplicaSystem system) throws AccumuloException, AccumuloSecurityException, PeerExistsException {
+ checkNotNull(name);
+ checkNotNull(system);
+
+ addPeer(name, system.getClass().getName());
+ }
+
+ @Override
+ public void addPeer(final String name, final String replicaType) throws AccumuloException, AccumuloSecurityException, PeerExistsException {
+ checkNotNull(name);
+ checkNotNull(replicaType);
+
+ MasterClient.execute(inst, new ClientExec<Client>() {
+
+ @Override
+ public void execute(Client client) throws Exception {
+ client.setSystemProperty(Tracer.traceInfo(), creds.toThrift(inst), Property.REPLICATION_PEERS.getKey() + name, replicaType);
+ }
+
+ });
+ }
+
+ @Override
+ public void removePeer(final String name) throws AccumuloException, AccumuloSecurityException, PeerNotFoundException {
+ checkNotNull(name);
+
+ MasterClient.execute(inst, new ClientExec<Client>() {
+
+ @Override
+ public void execute(Client client) throws Exception {
+ client.removeSystemProperty(Tracer.traceInfo(), creds.toThrift(inst), Property.REPLICATION_PEERS.getKey() + name);
+ }
+
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java
index 996198c..8613602 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockConnector.java
@@ -31,6 +31,7 @@ import org.apache.accumulo.core.client.MultiTableBatchWriter;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.InstanceOperations;
+import org.apache.accumulo.core.client.admin.ReplicationOperations;
import org.apache.accumulo.core.client.admin.SecurityOperations;
import org.apache.accumulo.core.client.admin.NamespaceOperations;
import org.apache.accumulo.core.client.admin.TableOperations;
@@ -147,5 +148,11 @@ public class MockConnector extends Connector {
// TODO add implementation
throw new UnsupportedOperationException();
}
+
+ @Override
+ public ReplicationOperations replicationOperations() {
+ // TODO add implementation
+ throw new UnsupportedOperationException();
+ }
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/replication/PeerExistsException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/replication/PeerExistsException.java b/core/src/main/java/org/apache/accumulo/core/client/replication/PeerExistsException.java
new file mode 100644
index 0000000..d205238
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/replication/PeerExistsException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.replication;
+
+/**
+ * The peer already exists
+ */
+public class PeerExistsException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public PeerExistsException(String peer) {
+ this(peer, (String) null);
+ }
+
+ public PeerExistsException(String peer, String message) {
+ super("Peer '" + peer + "' already exists" + (null == message || message.isEmpty() ? "" : message));
+ }
+
+ public PeerExistsException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/replication/PeerNotFoundException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/replication/PeerNotFoundException.java b/core/src/main/java/org/apache/accumulo/core/client/replication/PeerNotFoundException.java
new file mode 100644
index 0000000..1859c62
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/replication/PeerNotFoundException.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.replication;
+
+/**
+ * The peer already exists
+ */
+public class PeerNotFoundException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public PeerNotFoundException(String peer) {
+ this(peer, (String) null);
+ }
+
+ public PeerNotFoundException(String peer, String message) {
+ super("Peer '" + peer + "' not found " + (null == message || message.isEmpty() ? "" : message));
+ }
+
+ public PeerNotFoundException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystem.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystem.java b/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystem.java
new file mode 100644
index 0000000..220d7bb
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystem.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.replication;
+
+import org.apache.accumulo.core.replication.ReplicationTarget;
+import org.apache.accumulo.core.replication.proto.Replication.Status;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Encapsulation of a remote system which Accumulo can replicate data to
+ */
+public interface ReplicaSystem {
+
+ /**
+ * Replicate the given status to the target peer
+ * @param p Path to the resource we're reading from
+ * @param status Information to replicate
+ * @param target The peer
+ * @return A new Status for the progress that was made
+ */
+ public Status replicate(Path p, Status status, ReplicationTarget target);
+
+ /**
+ * Configure the implementation with necessary information from the system configuration
+ * <p>
+ * For example, we only need one implementation for Accumulo, but, for each peer,
+ * we have a ZK quorom and instance name
+ * @param configuration
+ */
+ public void configure(String configuration);
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystemFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystemFactory.java b/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystemFactory.java
new file mode 100644
index 0000000..3aa8ea4
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/client/replication/ReplicaSystemFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.client.replication;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+/**
+ *
+ */
+public class ReplicaSystemFactory {
+ private static final Logger log = LoggerFactory.getLogger(ReplicaSystemFactory.class);
+
+ /**
+ * @param value {@link ReplicaSystem} implementation class name
+ * @return A {@link ReplicaSystem} object from the given name
+ */
+ public static ReplicaSystem get(String value) {
+ Preconditions.checkNotNull(value);
+
+ int index = value.indexOf(',');
+ if (-1 == index) {
+ throw new IllegalArgumentException("Expected comma separator between replication system name and configuration");
+ }
+
+ String name = value.substring(0, index);
+ String configuration = value.substring(index + 1);
+
+ try {
+ Class<?> clz = Class.forName(name);
+ Object o = clz.newInstance();
+
+ if (ReplicaSystem.class.isAssignableFrom(o.getClass())) {
+ ReplicaSystem rs = (ReplicaSystem) o;
+ rs.configure(configuration);
+ return rs;
+ }
+
+ throw new RuntimeException("Class is not assignable to ReplicaSystem: " + name);
+ } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+ log.error("Error creating ReplicaSystem object", e);
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/conf/Property.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6700736..f31cd46 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -449,7 +449,7 @@ public enum Property {
@Experimental
REPLICATION_PREFIX("replication.", null, PropertyType.PREFIX, "Properties in this category affect the replication of data to other Accumulo instances."),
@Experimental
- REPLICATION_SLAVES("replication.slave.", null, PropertyType.PREFIX, "Properties in this category control what systems data can be replicated to"),
+ REPLICATION_PEERS("replication.peer.", null, PropertyType.PREFIX, "Properties in this category control what systems data can be replicated to"),
@Experimental
REPLICATION_BATCH_SIZE("replication.batch.size", "1000", PropertyType.COUNT, "Maximum number of updates (WAL) or key-value pairs (RFile) to send in one replication task"),
@Experimental
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
index a75d52a..3a60632 100644
--- a/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
+++ b/core/src/main/java/org/apache/accumulo/core/replication/ReplicationTarget.java
@@ -31,22 +31,22 @@ import org.apache.hadoop.io.WritableUtils;
*/
public class ReplicationTarget implements Writable {
- private String remoteName;
+ private String peerName;
private String remoteIdentifier;
public ReplicationTarget() { }
- public ReplicationTarget(String remoteName, String remoteIdentifier) {
- this.remoteName = remoteName;
+ public ReplicationTarget(String peerName, String remoteIdentifier) {
+ this.peerName = peerName;
this.remoteIdentifier = remoteIdentifier;
}
- public String getRemoteName() {
- return remoteName;
+ public String getPeerName() {
+ return peerName;
}
- public void setRemoteName(String remoteName) {
- this.remoteName = remoteName;
+ public void setPeerName(String peerName) {
+ this.peerName = peerName;
}
public String getRemoteIdentifier() {
@@ -59,11 +59,11 @@ public class ReplicationTarget implements Writable {
@Override
public void write(DataOutput out) throws IOException {
- if (null == remoteName) {
+ if (null == peerName) {
out.writeBoolean(false);
} else {
out.writeBoolean(true);
- WritableUtils.writeString(out, remoteName);
+ WritableUtils.writeString(out, peerName);
}
if (null == remoteIdentifier) {
@@ -77,7 +77,7 @@ public class ReplicationTarget implements Writable {
@Override
public void readFields(DataInput in) throws IOException {
if (in.readBoolean()) {
- this.remoteName = WritableUtils.readString(in);
+ this.peerName = WritableUtils.readString(in);
}
if (in.readBoolean()) {
this.remoteIdentifier = WritableUtils.readString(in);
@@ -87,13 +87,13 @@ public class ReplicationTarget implements Writable {
@Override
public String toString() {
StringBuilder sb = new StringBuilder(64);
- sb.append("Remote Name: ").append(remoteName).append(" Remote identifier: ").append(remoteIdentifier);
+ sb.append("Remote Name: ").append(peerName).append(" Remote identifier: ").append(remoteIdentifier);
return sb.toString();
}
@Override
public int hashCode() {
- return remoteName.hashCode() ^ remoteIdentifier.hashCode();
+ return peerName.hashCode() ^ remoteIdentifier.hashCode();
}
@Override
@@ -101,7 +101,7 @@ public class ReplicationTarget implements Writable {
if (o instanceof ReplicationTarget) {
ReplicationTarget other = (ReplicationTarget) o;
- return remoteName.equals(other.remoteName) && remoteIdentifier.equals(other.remoteIdentifier);
+ return peerName.equals(other.peerName) && remoteIdentifier.equals(other.remoteIdentifier);
}
return false;
@@ -127,16 +127,34 @@ public class ReplicationTarget implements Writable {
}
/**
+ * Deserialize a ReplicationTarget
+ * @param s Serialized copy
+ * @return the deserialized version
+ */
+ public static ReplicationTarget from(String s) {
+ ReplicationTarget target = new ReplicationTarget();
+ DataInputBuffer buffer = new DataInputBuffer();
+ buffer.reset(s.getBytes(), s.length());
+
+ try {
+ target.readFields(buffer);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return target;
+ }
+
+ /**
* Convenience method to serialize a ReplicationTarget to {@link Text} using the {@link Writable} methods without caring about
* performance penalties due to excessive object creation
- * @param target The object to serialize
* @return The serialized representation of the object
*/
- public static Text toText(ReplicationTarget target) {
+ public Text toText() {
DataOutputBuffer buffer = new DataOutputBuffer();
try {
- target.write(buffer);
+ this.write(buffer);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/baa7a4f7/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java b/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java
deleted file mode 100644
index 3d07b34..0000000
--- a/core/src/main/java/org/apache/accumulo/core/replication/thrift/RemoteReplicationCoordinator.java
+++ /dev/null
@@ -1,1010 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Autogenerated by Thrift Compiler (0.9.0)
- *
- * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
- * @generated
- */
-package org.apache.accumulo.core.replication.thrift;
-
-import org.apache.thrift.scheme.IScheme;
-import org.apache.thrift.scheme.SchemeFactory;
-import org.apache.thrift.scheme.StandardScheme;
-
-import org.apache.thrift.scheme.TupleScheme;
-import org.apache.thrift.protocol.TTupleProtocol;
-import org.apache.thrift.protocol.TProtocolException;
-import org.apache.thrift.EncodingUtils;
-import org.apache.thrift.TException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.EnumMap;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.EnumSet;
-import java.util.Collections;
-import java.util.BitSet;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings("all") public class RemoteReplicationCoordinator {
-
- public interface Iface {
-
- public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, org.apache.thrift.TException;
-
- }
-
- public interface AsyncIface {
-
- public void getServicerAddress(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<AsyncClient.getServicerAddress_call> resultHandler) throws org.apache.thrift.TException;
-
- }
-
- public static class Client extends org.apache.thrift.TServiceClient implements Iface {
- public static class Factory implements org.apache.thrift.TServiceClientFactory<Client> {
- public Factory() {}
- public Client getClient(org.apache.thrift.protocol.TProtocol prot) {
- return new Client(prot);
- }
- public Client getClient(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- return new Client(iprot, oprot);
- }
- }
-
- public Client(org.apache.thrift.protocol.TProtocol prot)
- {
- super(prot, prot);
- }
-
- public Client(org.apache.thrift.protocol.TProtocol iprot, org.apache.thrift.protocol.TProtocol oprot) {
- super(iprot, oprot);
- }
-
- public String getServicerAddress(int remoteTableId) throws NoServersAvailableException, org.apache.thrift.TException
- {
- send_getServicerAddress(remoteTableId);
- return recv_getServicerAddress();
- }
-
- public void send_getServicerAddress(int remoteTableId) throws org.apache.thrift.TException
- {
- getServicerAddress_args args = new getServicerAddress_args();
- args.setRemoteTableId(remoteTableId);
- sendBase("getServicerAddress", args);
- }
-
- public String recv_getServicerAddress() throws NoServersAvailableException, org.apache.thrift.TException
- {
- getServicerAddress_result result = new getServicerAddress_result();
- receiveBase(result, "getServicerAddress");
- if (result.isSetSuccess()) {
- return result.success;
- }
- if (result.e != null) {
- throw result.e;
- }
- throw new org.apache.thrift.TApplicationException(org.apache.thrift.TApplicationException.MISSING_RESULT, "getServicerAddress failed: unknown result");
- }
-
- }
- public static class AsyncClient extends org.apache.thrift.async.TAsyncClient implements AsyncIface {
- public static class Factory implements org.apache.thrift.async.TAsyncClientFactory<AsyncClient> {
- private org.apache.thrift.async.TAsyncClientManager clientManager;
- private org.apache.thrift.protocol.TProtocolFactory protocolFactory;
- public Factory(org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.protocol.TProtocolFactory protocolFactory) {
- this.clientManager = clientManager;
- this.protocolFactory = protocolFactory;
- }
- public AsyncClient getAsyncClient(org.apache.thrift.transport.TNonblockingTransport transport) {
- return new AsyncClient(protocolFactory, clientManager, transport);
- }
- }
-
- public AsyncClient(org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.async.TAsyncClientManager clientManager, org.apache.thrift.transport.TNonblockingTransport transport) {
- super(protocolFactory, clientManager, transport);
- }
-
- public void getServicerAddress(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<getServicerAddress_call> resultHandler) throws org.apache.thrift.TException {
- checkReady();
- getServicerAddress_call method_call = new getServicerAddress_call(remoteTableId, resultHandler, this, ___protocolFactory, ___transport);
- this.___currentMethod = method_call;
- ___manager.call(method_call);
- }
-
- public static class getServicerAddress_call extends org.apache.thrift.async.TAsyncMethodCall {
- private int remoteTableId;
- public getServicerAddress_call(int remoteTableId, org.apache.thrift.async.AsyncMethodCallback<getServicerAddress_call> resultHandler, org.apache.thrift.async.TAsyncClient client, org.apache.thrift.protocol.TProtocolFactory protocolFactory, org.apache.thrift.transport.TNonblockingTransport transport) throws org.apache.thrift.TException {
- super(client, protocolFactory, transport, resultHandler, false);
- this.remoteTableId = remoteTableId;
- }
-
- public void write_args(org.apache.thrift.protocol.TProtocol prot) throws org.apache.thrift.TException {
- prot.writeMessageBegin(new org.apache.thrift.protocol.TMessage("getServicerAddress", org.apache.thrift.protocol.TMessageType.CALL, 0));
- getServicerAddress_args args = new getServicerAddress_args();
- args.setRemoteTableId(remoteTableId);
- args.write(prot);
- prot.writeMessageEnd();
- }
-
- public String getResult() throws NoServersAvailableException, org.apache.thrift.TException {
- if (getState() != org.apache.thrift.async.TAsyncMethodCall.State.RESPONSE_READ) {
- throw new IllegalStateException("Method call not finished!");
- }
- org.apache.thrift.transport.TMemoryInputTransport memoryTransport = new org.apache.thrift.transport.TMemoryInputTransport(getFrameBuffer().array());
- org.apache.thrift.protocol.TProtocol prot = client.getProtocolFactory().getProtocol(memoryTransport);
- return (new Client(prot)).recv_getServicerAddress();
- }
- }
-
- }
-
- public static class Processor<I extends Iface> extends org.apache.thrift.TBaseProcessor<I> implements org.apache.thrift.TProcessor {
- private static final Logger LOGGER = LoggerFactory.getLogger(Processor.class.getName());
- public Processor(I iface) {
- super(iface, getProcessMap(new HashMap<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>>()));
- }
-
- protected Processor(I iface, Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- super(iface, getProcessMap(processMap));
- }
-
- private static <I extends Iface> Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> getProcessMap(Map<String, org.apache.thrift.ProcessFunction<I, ? extends org.apache.thrift.TBase>> processMap) {
- processMap.put("getServicerAddress", new getServicerAddress());
- return processMap;
- }
-
- public static class getServicerAddress<I extends Iface> extends org.apache.thrift.ProcessFunction<I, getServicerAddress_args> {
- public getServicerAddress() {
- super("getServicerAddress");
- }
-
- public getServicerAddress_args getEmptyArgsInstance() {
- return new getServicerAddress_args();
- }
-
- protected boolean isOneway() {
- return false;
- }
-
- public getServicerAddress_result getResult(I iface, getServicerAddress_args args) throws org.apache.thrift.TException {
- getServicerAddress_result result = new getServicerAddress_result();
- try {
- result.success = iface.getServicerAddress(args.remoteTableId);
- } catch (NoServersAvailableException e) {
- result.e = e;
- }
- return result;
- }
- }
-
- }
-
- public static class getServicerAddress_args implements org.apache.thrift.TBase<getServicerAddress_args, getServicerAddress_args._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getServicerAddress_args");
-
- private static final org.apache.thrift.protocol.TField REMOTE_TABLE_ID_FIELD_DESC = new org.apache.thrift.protocol.TField("remoteTableId", org.apache.thrift.protocol.TType.I32, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new getServicerAddress_argsStandardSchemeFactory());
- schemes.put(TupleScheme.class, new getServicerAddress_argsTupleSchemeFactory());
- }
-
- public int remoteTableId; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- REMOTE_TABLE_ID((short)1, "remoteTableId");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 1: // REMOTE_TABLE_ID
- return REMOTE_TABLE_ID;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- private static final int __REMOTETABLEID_ISSET_ID = 0;
- private byte __isset_bitfield = 0;
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.REMOTE_TABLE_ID, new org.apache.thrift.meta_data.FieldMetaData("remoteTableId", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I32)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getServicerAddress_args.class, metaDataMap);
- }
-
- public getServicerAddress_args() {
- }
-
- public getServicerAddress_args(
- int remoteTableId)
- {
- this();
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public getServicerAddress_args(getServicerAddress_args other) {
- __isset_bitfield = other.__isset_bitfield;
- this.remoteTableId = other.remoteTableId;
- }
-
- public getServicerAddress_args deepCopy() {
- return new getServicerAddress_args(this);
- }
-
- @Override
- public void clear() {
- setRemoteTableIdIsSet(false);
- this.remoteTableId = 0;
- }
-
- public int getRemoteTableId() {
- return this.remoteTableId;
- }
-
- public getServicerAddress_args setRemoteTableId(int remoteTableId) {
- this.remoteTableId = remoteTableId;
- setRemoteTableIdIsSet(true);
- return this;
- }
-
- public void unsetRemoteTableId() {
- __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- /** Returns true if field remoteTableId is set (has been assigned a value) and false otherwise */
- public boolean isSetRemoteTableId() {
- return EncodingUtils.testBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID);
- }
-
- public void setRemoteTableIdIsSet(boolean value) {
- __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __REMOTETABLEID_ISSET_ID, value);
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case REMOTE_TABLE_ID:
- if (value == null) {
- unsetRemoteTableId();
- } else {
- setRemoteTableId((Integer)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case REMOTE_TABLE_ID:
- return Integer.valueOf(getRemoteTableId());
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case REMOTE_TABLE_ID:
- return isSetRemoteTableId();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof getServicerAddress_args)
- return this.equals((getServicerAddress_args)that);
- return false;
- }
-
- public boolean equals(getServicerAddress_args that) {
- if (that == null)
- return false;
-
- boolean this_present_remoteTableId = true;
- boolean that_present_remoteTableId = true;
- if (this_present_remoteTableId || that_present_remoteTableId) {
- if (!(this_present_remoteTableId && that_present_remoteTableId))
- return false;
- if (this.remoteTableId != that.remoteTableId)
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(getServicerAddress_args other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- getServicerAddress_args typedOther = (getServicerAddress_args)other;
-
- lastComparison = Boolean.valueOf(isSetRemoteTableId()).compareTo(typedOther.isSetRemoteTableId());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetRemoteTableId()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.remoteTableId, typedOther.remoteTableId);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("getServicerAddress_args(");
- boolean first = true;
-
- sb.append("remoteTableId:");
- sb.append(this.remoteTableId);
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- // it doesn't seem like you should have to do this, but java serialization is wacky, and doesn't call the default constructor.
- __isset_bitfield = 0;
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class getServicerAddress_argsStandardSchemeFactory implements SchemeFactory {
- public getServicerAddress_argsStandardScheme getScheme() {
- return new getServicerAddress_argsStandardScheme();
- }
- }
-
- private static class getServicerAddress_argsStandardScheme extends StandardScheme<getServicerAddress_args> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, getServicerAddress_args struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 1: // REMOTE_TABLE_ID
- if (schemeField.type == org.apache.thrift.protocol.TType.I32) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, getServicerAddress_args struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- oprot.writeFieldBegin(REMOTE_TABLE_ID_FIELD_DESC);
- oprot.writeI32(struct.remoteTableId);
- oprot.writeFieldEnd();
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class getServicerAddress_argsTupleSchemeFactory implements SchemeFactory {
- public getServicerAddress_argsTupleScheme getScheme() {
- return new getServicerAddress_argsTupleScheme();
- }
- }
-
- private static class getServicerAddress_argsTupleScheme extends TupleScheme<getServicerAddress_args> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_args struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetRemoteTableId()) {
- optionals.set(0);
- }
- oprot.writeBitSet(optionals, 1);
- if (struct.isSetRemoteTableId()) {
- oprot.writeI32(struct.remoteTableId);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_args struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(1);
- if (incoming.get(0)) {
- struct.remoteTableId = iprot.readI32();
- struct.setRemoteTableIdIsSet(true);
- }
- }
- }
-
- }
-
- public static class getServicerAddress_result implements org.apache.thrift.TBase<getServicerAddress_result, getServicerAddress_result._Fields>, java.io.Serializable, Cloneable {
- private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new org.apache.thrift.protocol.TStruct("getServicerAddress_result");
-
- private static final org.apache.thrift.protocol.TField SUCCESS_FIELD_DESC = new org.apache.thrift.protocol.TField("success", org.apache.thrift.protocol.TType.STRING, (short)0);
- private static final org.apache.thrift.protocol.TField E_FIELD_DESC = new org.apache.thrift.protocol.TField("e", org.apache.thrift.protocol.TType.STRUCT, (short)1);
-
- private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>();
- static {
- schemes.put(StandardScheme.class, new getServicerAddress_resultStandardSchemeFactory());
- schemes.put(TupleScheme.class, new getServicerAddress_resultTupleSchemeFactory());
- }
-
- public String success; // required
- public NoServersAvailableException e; // required
-
- /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */
- @SuppressWarnings("all") public enum _Fields implements org.apache.thrift.TFieldIdEnum {
- SUCCESS((short)0, "success"),
- E((short)1, "e");
-
- private static final Map<String, _Fields> byName = new HashMap<String, _Fields>();
-
- static {
- for (_Fields field : EnumSet.allOf(_Fields.class)) {
- byName.put(field.getFieldName(), field);
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, or null if its not found.
- */
- public static _Fields findByThriftId(int fieldId) {
- switch(fieldId) {
- case 0: // SUCCESS
- return SUCCESS;
- case 1: // E
- return E;
- default:
- return null;
- }
- }
-
- /**
- * Find the _Fields constant that matches fieldId, throwing an exception
- * if it is not found.
- */
- public static _Fields findByThriftIdOrThrow(int fieldId) {
- _Fields fields = findByThriftId(fieldId);
- if (fields == null) throw new IllegalArgumentException("Field " + fieldId + " doesn't exist!");
- return fields;
- }
-
- /**
- * Find the _Fields constant that matches name, or null if its not found.
- */
- public static _Fields findByName(String name) {
- return byName.get(name);
- }
-
- private final short _thriftId;
- private final String _fieldName;
-
- _Fields(short thriftId, String fieldName) {
- _thriftId = thriftId;
- _fieldName = fieldName;
- }
-
- public short getThriftFieldId() {
- return _thriftId;
- }
-
- public String getFieldName() {
- return _fieldName;
- }
- }
-
- // isset id assignments
- public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
- static {
- Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
- tmpMap.put(_Fields.SUCCESS, new org.apache.thrift.meta_data.FieldMetaData("success", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING)));
- tmpMap.put(_Fields.E, new org.apache.thrift.meta_data.FieldMetaData("e", org.apache.thrift.TFieldRequirementType.DEFAULT,
- new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT)));
- metaDataMap = Collections.unmodifiableMap(tmpMap);
- org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(getServicerAddress_result.class, metaDataMap);
- }
-
- public getServicerAddress_result() {
- }
-
- public getServicerAddress_result(
- String success,
- NoServersAvailableException e)
- {
- this();
- this.success = success;
- this.e = e;
- }
-
- /**
- * Performs a deep copy on <i>other</i>.
- */
- public getServicerAddress_result(getServicerAddress_result other) {
- if (other.isSetSuccess()) {
- this.success = other.success;
- }
- if (other.isSetE()) {
- this.e = new NoServersAvailableException(other.e);
- }
- }
-
- public getServicerAddress_result deepCopy() {
- return new getServicerAddress_result(this);
- }
-
- @Override
- public void clear() {
- this.success = null;
- this.e = null;
- }
-
- public String getSuccess() {
- return this.success;
- }
-
- public getServicerAddress_result setSuccess(String success) {
- this.success = success;
- return this;
- }
-
- public void unsetSuccess() {
- this.success = null;
- }
-
- /** Returns true if field success is set (has been assigned a value) and false otherwise */
- public boolean isSetSuccess() {
- return this.success != null;
- }
-
- public void setSuccessIsSet(boolean value) {
- if (!value) {
- this.success = null;
- }
- }
-
- public NoServersAvailableException getE() {
- return this.e;
- }
-
- public getServicerAddress_result setE(NoServersAvailableException e) {
- this.e = e;
- return this;
- }
-
- public void unsetE() {
- this.e = null;
- }
-
- /** Returns true if field e is set (has been assigned a value) and false otherwise */
- public boolean isSetE() {
- return this.e != null;
- }
-
- public void setEIsSet(boolean value) {
- if (!value) {
- this.e = null;
- }
- }
-
- public void setFieldValue(_Fields field, Object value) {
- switch (field) {
- case SUCCESS:
- if (value == null) {
- unsetSuccess();
- } else {
- setSuccess((String)value);
- }
- break;
-
- case E:
- if (value == null) {
- unsetE();
- } else {
- setE((NoServersAvailableException)value);
- }
- break;
-
- }
- }
-
- public Object getFieldValue(_Fields field) {
- switch (field) {
- case SUCCESS:
- return getSuccess();
-
- case E:
- return getE();
-
- }
- throw new IllegalStateException();
- }
-
- /** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */
- public boolean isSet(_Fields field) {
- if (field == null) {
- throw new IllegalArgumentException();
- }
-
- switch (field) {
- case SUCCESS:
- return isSetSuccess();
- case E:
- return isSetE();
- }
- throw new IllegalStateException();
- }
-
- @Override
- public boolean equals(Object that) {
- if (that == null)
- return false;
- if (that instanceof getServicerAddress_result)
- return this.equals((getServicerAddress_result)that);
- return false;
- }
-
- public boolean equals(getServicerAddress_result that) {
- if (that == null)
- return false;
-
- boolean this_present_success = true && this.isSetSuccess();
- boolean that_present_success = true && that.isSetSuccess();
- if (this_present_success || that_present_success) {
- if (!(this_present_success && that_present_success))
- return false;
- if (!this.success.equals(that.success))
- return false;
- }
-
- boolean this_present_e = true && this.isSetE();
- boolean that_present_e = true && that.isSetE();
- if (this_present_e || that_present_e) {
- if (!(this_present_e && that_present_e))
- return false;
- if (!this.e.equals(that.e))
- return false;
- }
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return 0;
- }
-
- public int compareTo(getServicerAddress_result other) {
- if (!getClass().equals(other.getClass())) {
- return getClass().getName().compareTo(other.getClass().getName());
- }
-
- int lastComparison = 0;
- getServicerAddress_result typedOther = (getServicerAddress_result)other;
-
- lastComparison = Boolean.valueOf(isSetSuccess()).compareTo(typedOther.isSetSuccess());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetSuccess()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.success, typedOther.success);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- lastComparison = Boolean.valueOf(isSetE()).compareTo(typedOther.isSetE());
- if (lastComparison != 0) {
- return lastComparison;
- }
- if (isSetE()) {
- lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.e, typedOther.e);
- if (lastComparison != 0) {
- return lastComparison;
- }
- }
- return 0;
- }
-
- public _Fields fieldForId(int fieldId) {
- return _Fields.findByThriftId(fieldId);
- }
-
- public void read(org.apache.thrift.protocol.TProtocol iprot) throws org.apache.thrift.TException {
- schemes.get(iprot.getScheme()).getScheme().read(iprot, this);
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot) throws org.apache.thrift.TException {
- schemes.get(oprot.getScheme()).getScheme().write(oprot, this);
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder("getServicerAddress_result(");
- boolean first = true;
-
- sb.append("success:");
- if (this.success == null) {
- sb.append("null");
- } else {
- sb.append(this.success);
- }
- first = false;
- if (!first) sb.append(", ");
- sb.append("e:");
- if (this.e == null) {
- sb.append("null");
- } else {
- sb.append(this.e);
- }
- first = false;
- sb.append(")");
- return sb.toString();
- }
-
- public void validate() throws org.apache.thrift.TException {
- // check for required fields
- // check for sub-struct validity
- }
-
- private void writeObject(java.io.ObjectOutputStream out) throws java.io.IOException {
- try {
- write(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(out)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException {
- try {
- read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
- } catch (org.apache.thrift.TException te) {
- throw new java.io.IOException(te);
- }
- }
-
- private static class getServicerAddress_resultStandardSchemeFactory implements SchemeFactory {
- public getServicerAddress_resultStandardScheme getScheme() {
- return new getServicerAddress_resultStandardScheme();
- }
- }
-
- private static class getServicerAddress_resultStandardScheme extends StandardScheme<getServicerAddress_result> {
-
- public void read(org.apache.thrift.protocol.TProtocol iprot, getServicerAddress_result struct) throws org.apache.thrift.TException {
- org.apache.thrift.protocol.TField schemeField;
- iprot.readStructBegin();
- while (true)
- {
- schemeField = iprot.readFieldBegin();
- if (schemeField.type == org.apache.thrift.protocol.TType.STOP) {
- break;
- }
- switch (schemeField.id) {
- case 0: // SUCCESS
- if (schemeField.type == org.apache.thrift.protocol.TType.STRING) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- case 1: // E
- if (schemeField.type == org.apache.thrift.protocol.TType.STRUCT) {
- struct.e = new NoServersAvailableException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- } else {
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- break;
- default:
- org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type);
- }
- iprot.readFieldEnd();
- }
- iprot.readStructEnd();
-
- // check for required fields of primitive type, which can't be checked in the validate method
- struct.validate();
- }
-
- public void write(org.apache.thrift.protocol.TProtocol oprot, getServicerAddress_result struct) throws org.apache.thrift.TException {
- struct.validate();
-
- oprot.writeStructBegin(STRUCT_DESC);
- if (struct.success != null) {
- oprot.writeFieldBegin(SUCCESS_FIELD_DESC);
- oprot.writeString(struct.success);
- oprot.writeFieldEnd();
- }
- if (struct.e != null) {
- oprot.writeFieldBegin(E_FIELD_DESC);
- struct.e.write(oprot);
- oprot.writeFieldEnd();
- }
- oprot.writeFieldStop();
- oprot.writeStructEnd();
- }
-
- }
-
- private static class getServicerAddress_resultTupleSchemeFactory implements SchemeFactory {
- public getServicerAddress_resultTupleScheme getScheme() {
- return new getServicerAddress_resultTupleScheme();
- }
- }
-
- private static class getServicerAddress_resultTupleScheme extends TupleScheme<getServicerAddress_result> {
-
- @Override
- public void write(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_result struct) throws org.apache.thrift.TException {
- TTupleProtocol oprot = (TTupleProtocol) prot;
- BitSet optionals = new BitSet();
- if (struct.isSetSuccess()) {
- optionals.set(0);
- }
- if (struct.isSetE()) {
- optionals.set(1);
- }
- oprot.writeBitSet(optionals, 2);
- if (struct.isSetSuccess()) {
- oprot.writeString(struct.success);
- }
- if (struct.isSetE()) {
- struct.e.write(oprot);
- }
- }
-
- @Override
- public void read(org.apache.thrift.protocol.TProtocol prot, getServicerAddress_result struct) throws org.apache.thrift.TException {
- TTupleProtocol iprot = (TTupleProtocol) prot;
- BitSet incoming = iprot.readBitSet(2);
- if (incoming.get(0)) {
- struct.success = iprot.readString();
- struct.setSuccessIsSet(true);
- }
- if (incoming.get(1)) {
- struct.e = new NoServersAvailableException();
- struct.e.read(iprot);
- struct.setEIsSet(true);
- }
- }
- }
-
- }
-
-}
[37/50] [abbrv] ACCUMULO-1327 converted latex manual to asciidoc
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/design.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/design.tex b/docs/src/main/latex/accumulo_user_manual/chapters/design.tex
deleted file mode 100644
index 8507f24..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/design.tex
+++ /dev/null
@@ -1,186 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Accumulo Design}
-
-\section{Data Model}
-
-Accumulo provides a richer data model than simple key-value stores, but is not a
-fully relational database. Data is represented as key-value pairs, where the key and
-value are comprised of the following elements:
-
-\begin{center}
-$\begin{array}{|c|c|c|c|c|c|} \hline
-\multicolumn{5}{|c|}{\mbox{Key}} & \multirow{3}{*}{\mbox{Value}}\\ \cline{1-5}
-\multirow{2}{*}{\mbox{Row ID}}& \multicolumn{3}{|c|}{\mbox{Column}} & \multirow{2}{*}{\mbox{Timestamp}} & \\ \cline{2-4}
-& \mbox{Family} & \mbox{Qualifier} & \mbox{Visibility} & & \\ \hline
-\end{array}$
-\end{center}
-
-All elements of the Key and the Value are represented as byte arrays except for
-Timestamp, which is a Long. Accumulo sorts keys by element and lexicographically
-in ascending order. Timestamps are sorted in descending order so that later
-versions of the same Key appear first in a sequential scan. Tables consist of a set of
-sorted key-value pairs.
-
-\section{Architecture}
-
-Accumulo is a distributed data storage and retrieval system and as such consists of
-several architectural components, some of which run on many individual servers.
-Much of the work Accumulo does involves maintaining certain properties of the
-data, such as organization, availability, and integrity, across many commodity-class
-machines.
-
-\section{Components}
-
-An instance of Accumulo includes many TabletServers, one Garbage Collector process,
-one Master server and many Clients.
-
-\subsection{Tablet Server}
-
-The TabletServer manages some subset of all the tablets (partitions of tables). This includes receiving writes from clients, persisting writes to a
-write-ahead log, sorting new key-value pairs in memory, periodically
-flushing sorted key-value pairs to new files in HDFS, and responding
-to reads from clients, forming a merge-sorted view of all keys and
-values from all the files it has created and the sorted in-memory
-store.
-
-TabletServers also perform recovery of a tablet
-that was previously on a server that failed, reapplying any writes
-found in the write-ahead log to the tablet.
-
-\subsection{Garbage Collector}
-
-Accumulo processes will share files stored in HDFS. Periodically, the Garbage
-Collector will identify files that are no longer needed by any process, and
-delete them. Multiple garbage collectors can be run to provide hot-standby support.
-They will perform leader election among themselves to choose a single active instance.
-
-\subsection{Master}
-
-The Accumulo Master is responsible for detecting and responding to TabletServer
-failure. It tries to balance the load across TabletServer by assigning tablets carefully
-and instructing TabletServers to unload tablets when necessary. The Master ensures all
-tablets are assigned to one TabletServer each, and handles table creation, alteration,
-and deletion requests from clients. The Master also coordinates startup, graceful
-shutdown and recovery of changes in write-ahead logs when Tablet servers fail.
-
-Multiple masters may be run. The masters will choose among themselves a single master,
-and the others will become backups if the master should fail.
-
-\subsection{Tracer}
-
-The Accumulo Tracer process supports the distributed timing API provided by Accumulo.
-One to many of these processes can be run on a cluster which will write the timing
-information to a given Accumulo table for future reference. Seeing the section on
-Tracing for more information on this support.
-
-\subsection{Monitor}
-
-The Accumulo Monitor is a web application that provides a wealth of information about
-the state of an instance. The Monitor shows graphs and tables which contain information
-about read/write rates, cache hit/miss rates, and Accumulo table information such as scan
-rate and active/queued compactions. Additionally, the Monitor should always be the first
-point of entry when attempting to debug an Accumulo problem as it will show high-level problems
-in addition to aggregated errors from all nodes in the cluster. See the section on Monitoring
-for more information.
-
-Multiple Monitors can be run to provide hot-standby support in the face of failure. Due to the
-forwarding of logs from remote hosts to the Monitor, only one Monitor process should be active
-at one time. Leader election will be performed internally to choose the active Monitor.
-
-\subsection{Client}
-
-Accumulo includes a client library that is linked to every application. The client
-library contains logic for finding servers managing a particular tablet, and
-communicating with TabletServers to write and retrieve key-value pairs.
-
-\section{Data Management}
-
-Accumulo stores data in tables, which are partitioned into tablets. Tablets are
-partitioned on row boundaries so that all of the columns and values for a particular
-row are found together within the same tablet. The Master assigns Tablets to one
-TabletServer at a time. This enables row-level transactions to take place without
-using distributed locking or some other complicated synchronization mechanism. As
-clients insert and query data, and as machines are added and removed from the
-cluster, the Master migrates tablets to ensure they remain available and that the
-ingest and query load is balanced across the cluster.
-
-\begin{center}
-\includegraphics[scale=0.4]{images/data_distribution.png}
-\end{center}
-
-\section{Tablet Service}
-
-
-When a write arrives at a TabletServer it is written to a Write-Ahead Log and
-then inserted into a sorted data structure in memory called a MemTable. When the
-MemTable reaches a certain size the TabletServer writes out the sorted key-value
-pairs to a file in HDFS called Indexed Sequential Access Method (ISAM)
-file. This process is called a minor compaction. A new MemTable is then created
-and the fact of the compaction is recorded in the Write-Ahead Log.
-
-When a request to read data arrives at a TabletServer, the TabletServer does a
-binary search across the MemTable as well as the in-memory indexes associated
-with each ISAM file to find the relevant values. If clients are performing a
-scan, several key-value pairs are returned to the client in order from the
-MemTable and the set of ISAM files by performing a merge-sort as they are read.
-
-\section{Compactions}
-
-In order to manage the number of files per tablet, periodically the TabletServer
-performs Major Compactions of files within a tablet, in which some set of ISAM
-files are combined into one file. The previous files will eventually be removed
-by the Garbage Collector. This also provides an opportunity to permanently
-remove deleted key-value pairs by omitting key-value pairs suppressed by a
-delete entry when the new file is created.
-
-\section{Splitting}
-
-When a table is created it has one tablet. As the table grows its initial
-tablet eventually splits into two tablets. Its likely that one of these
-tablets will migrate to another tablet server. As the table continues to grow,
-its tablets will continue to split and be migrated. The decision to
-automatically split a tablet is based on the size of a tablets files. The
-size threshold at which a tablet splits is configurable per table. In addition
-to automatic splitting, a user can manually add split points to a table to
-create new tablets. Manually splitting a new table can parallelize reads and
-writes giving better initial performance without waiting for automatic
-splitting.
-
-As data is deleted from a table, tablets may shrink. Over time this can lead
-to small or empty tablets. To deal with this, merging of tablets was
-introduced in Accumulo 1.4. This is discussed in more detail later.
-
-\section{Fault-Tolerance}
-
-If a TabletServer fails, the Master detects it and automatically reassigns the tablets
-assigned from the failed server to other servers. Any key-value pairs that were in
-memory at the time the TabletServer fails are automatically reapplied from the Write-Ahead
-Log(WAL) to prevent any loss of data.
-
-Tablet servers write their WALs directly to HDFS so the logs are available to all tablet
-servers for recovery. To make the recovery process efficient, the updates within a log are
-grouped by tablet. TabletServers can quickly apply the mutations from the sorted logs
-that are destined for the tablets they have now been assigned.
-
-TabletServer failures are noted on the Master's monitor page, accessible via\\
-\mbox{http://master-address:50095/monitor}.
-
-\begin{center}
-\includegraphics[scale=0.4]{images/failure_handling.png}
-\end{center}
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/development_clients.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/development_clients.tex b/docs/src/main/latex/accumulo_user_manual/chapters/development_clients.tex
deleted file mode 100644
index fb7195d..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/development_clients.tex
+++ /dev/null
@@ -1,109 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Development Clients}
-
-Normally, Accumulo consists of lots of moving parts. Even a stand-alone version of
-Accumulo requires Hadoop, Zookeeper, the Accumulo master, a tablet server, etc. If
-you want to write a unit test that uses Accumulo, you need a lot of infrastructure
-in place before your test can run.
-
-\section{Mock Accumulo}
-
-Mock Accumulo supplies mock implementations for much of the client API. It presently
-does not enforce users, logins, permissions, etc. It does support Iterators and Combiners.
-Note that MockAccumulo holds all data in memory, and will not retain any data or
-settings between runs.
-
-While normal interaction with the Accumulo client looks like this:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Instance instance = new ZooKeeperInstance(...);
-Connector conn = instance.getConnector(user, passwordToken);
-\end{verbatim}\endgroup
-
-To interact with the MockAccumulo, just replace the ZooKeeperInstance with MockInstance:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Instance instance = new MockInstance();
-\end{verbatim}\endgroup
-
-In fact, you can use the "fake" option to the Accumulo shell and interact with
-MockAccumulo:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ ./bin/accumulo shell --fake -u root -p ''
-
-Shell - Apache Accumulo Interactive Shell
--
-- version: 1.6
-- instance name: fake
-- instance id: mock-instance-id
--
-- type 'help' for a list of available commands
--
-root@fake> createtable test
-root@fake test> insert row1 cf cq value
-root@fake test> insert row2 cf cq value2
-root@fake test> insert row3 cf cq value3
-root@fake test> scan
-row1 cf:cq [] value
-row2 cf:cq [] value2
-row3 cf:cq [] value3
-root@fake test> scan -b row2 -e row2
-row2 cf:cq [] value2
-root@fake test>
-\end{verbatim}\endgroup
-
-When testing Map Reduce jobs, you can also set the Mock Accumulo on the AccumuloInputFormat
-and AccumuloOutputFormat classes:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// ... set up job configuration
-AccumuloInputFormat.setMockInstance(job, "mockInstance");
-AccumuloOutputFormat.setMockInstance(job, "mockInstance");
-\end{verbatim}\endgroup
-
-\section{Mini Accumulo Cluster}
-
-While the Mock Accumulo provides a lightweight implementation of the client API for unit
-testing, it is often necessary to write more realistic end-to-end integration tests that
-take advantage of the entire ecosystem. The Mini Accumulo Cluster makes this possible by
-configuring and starting Zookeeper, initializing Accumulo, and starting the Master as well
-as some Tablet Servers. It runs against the local filesystem instead of having to start
-up HDFS.
-
-To start it up, you will need to supply an empty directory and a root password as arguments:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-File tempDirectory = // JUnit and Guava supply mechanisms for creating temp directories
-MiniAccumuloCluster accumulo = new MiniAccumuloCluster(tempDirectory, "password");
-accumulo.start();
-\end{verbatim}\endgroup
-
-Once we have our mini cluster running, we will want to interact with the Accumulo client API:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers());
-Connector conn = instance.getConnector("root", new PasswordToken("password"));
-\end{verbatim}\endgroup
-
-Upon completion of our development code, we will want to shutdown our MiniAccumuloCluster:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-accumulo.stop()
-// delete your temporary folder
-\end{verbatim}\endgroup
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/high_speed_ingest.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/high_speed_ingest.tex b/docs/src/main/latex/accumulo_user_manual/chapters/high_speed_ingest.tex
deleted file mode 100644
index ab766d0..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/high_speed_ingest.tex
+++ /dev/null
@@ -1,133 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{High-Speed Ingest}
-
-Accumulo is often used as part of a larger data processing and storage system. To
-maximize the performance of a parallel system involving Accumulo, the ingestion
-and query components should be designed to provide enough parallelism and
-concurrency to avoid creating bottlenecks for users and other systems writing to
-and reading from Accumulo. There are several ways to achieve high ingest
-performance.
-
-\section{Pre-Splitting New Tables}
-
-New tables consist of a single tablet by default. As mutations are applied, the table
-grows and splits into multiple tablets which are balanced by the Master across
-TabletServers. This implies that the aggregate ingest rate will be limited to fewer
-servers than are available within the cluster until the table has reached the point
-where there are tablets on every TabletServer.
-
-Pre-splitting a table ensures that there are as many tablets as desired available
-before ingest begins to take advantage of all the parallelism possible with the cluster
-hardware. Tables can be split anytime by using the shell:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance mytable> addsplits -sf /local_splitfile -t mytable
-\end{verbatim}\endgroup
-
-For the purposes of providing parallelism to ingest it is not necessary to create more
-tablets than there are physical machines within the cluster as the aggregate ingest
-rate is a function of the number of physical machines. Note that the aggregate ingest
-rate is still subject to the number of machines running ingest clients, and the
-distribution of rowIDs across the table. The aggregation ingest rate will be
-suboptimal if there are many inserts into a small number of rowIDs.
-
-\section{Multiple Ingester Clients}
-
-Accumulo is capable of scaling to very high rates of ingest, which is dependent upon
-not just the number of TabletServers in operation but also the number of ingest
-clients. This is because a single client, while capable of batching mutations and
-sending them to all TabletServers, is ultimately limited by the amount of data that
-can be processed on a single machine. The aggregate ingest rate will scale linearly
-with the number of clients up to the point at which either the aggregate I/O of
-TabletServers or total network bandwidth capacity is reached.
-
-In operational settings where high rates of ingest are paramount, clusters are often
-configured to dedicate some number of machines solely to running Ingester Clients.
-The exact ratio of clients to TabletServers necessary for optimum ingestion rates
-will vary according to the distribution of resources per machine and by data type.
-
-\section{Bulk Ingest}
-
-Accumulo supports the ability to import files produced by an external process such
-as MapReduce into an existing table. In some cases it may be faster to load data this
-way rather than via ingesting through clients using BatchWriters. This allows a large
-number of machines to format data the way Accumulo expects. The new files can
-then simply be introduced to Accumulo via a shell command.
-
-To configure MapReduce to format data in preparation for bulk loading, the job
-should be set to use a range partitioner instead of the default hash partitioner. The
-range partitioner uses the split points of the Accumulo table that will receive the
-data. The split points can be obtained from the shell and used by the MapReduce
-RangePartitioner. Note that this is only useful if the existing table is already split
-into multiple tablets.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance mytable> getsplits
-aa
-ab
-ac
-...
-zx
-zy
-zz
-\end{verbatim}\endgroup
-
-Run the MapReduce job, using the AccumuloFileOutputFormat to create the files to
-be introduced to Accumulo. Once this is complete, the files can be added to
-Accumulo via the shell:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance mytable> importdirectory /files_dir /failures
-\end{verbatim}\endgroup
-
-Note that the paths referenced are directories within the same HDFS instance over
-which Accumulo is running. Accumulo places any files that failed to be added to the
-second directory specified.
-
-A complete example of using Bulk Ingest can be found at\\
-accumulo/docs/examples/README.bulkIngest
-
-\section{Logical Time for Bulk Ingest}
-
-Logical time is important for bulk imported data, for which the client code may
-be choosing a timestamp. At bulk import time, the user can choose to enable
-logical time for the set of files being imported. When its enabled, Accumulo
-uses a specialized system iterator to lazily set times in a bulk imported file.
-This mechanism guarantees that times set by unsynchronized multi-node
-applications (such as those running on MapReduce) will maintain some semblance
-of causal ordering. This mitigates the problem of the time being wrong on the
-system that created the file for bulk import. These times are not set when the
-file is imported, but whenever it is read by scans or compactions. At import, a
-time is obtained and always used by the specialized system iterator to set that
-time.
-
-The timestamp assigned by Accumulo will be the same for every key in the file.
-This could cause problems if the file contains multiple keys that are identical
-except for the timestamp. In this case, the sort order of the keys will be
-undefined. This could occur if an insert and an update were in the same bulk
-import file.
-
-\section{MapReduce Ingest}
-It is possible to efficiently write many mutations to Accumulo in parallel via a
-MapReduce job. In this scenario the MapReduce is written to process data that lives
-in HDFS and write mutations to Accumulo using the AccumuloOutputFormat. See
-the MapReduce section under Analytics for details.
-
-An example of using MapReduce can be found under\\
-accumulo/docs/examples/README.mapred
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/introduction.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/introduction.tex b/docs/src/main/latex/accumulo_user_manual/chapters/introduction.tex
deleted file mode 100644
index 09d9f14..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/introduction.tex
+++ /dev/null
@@ -1,27 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Introduction}
-Apache Accumulo is a highly scalable structured store based on Google's BigTable.
-Accumulo is written in Java and operates over the Hadoop Distributed File System
-(HDFS), which is part of the popular Apache Hadoop project. Accumulo supports
-efficient storage and retrieval of structured data, including queries for ranges, and
-provides support for using Accumulo tables as input and output for MapReduce
-jobs.
-
-Accumulo features automatic load-balancing and partitioning, data compression
-and fine-grained security labels.
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/multivolume.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/multivolume.tex b/docs/src/main/latex/accumulo_user_manual/chapters/multivolume.tex
deleted file mode 100644
index 0a0e6fe..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/multivolume.tex
+++ /dev/null
@@ -1,85 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Multi-Volume Installations}
-
-This is an advanced configuration setting for very large clusters
-under a lot of write pressure.
-
-The HDFS NameNode holds all of the metadata about the files in
-HDFS. For fast performance, all of this information needs to be stored
-in memory. A single NameNode with 64G of memory can store the
-metadata for tens of millions of files.However, when scaling beyond a
-thousand nodes, an active Accumulo system can generate lots of updates
-to the file system, especially when data is being ingested. The large
-number of write transactions to the NameNode, and the speed of a
-single edit log, can become the limiting factor for large scale
-Accumulo installations.
-
-You can see the effect of slow write transactions when the Accumulo
-Garbage Collector takes a long time (more than 5 minutes) to delete
-the files Accumulo no longer needs. If your Garbage Collector
-routinely runs in less than a minute, the NameNode is performing well.
-
-However, if you do begin to experience slow-down and poor GC
-performance, Accumulo can be configured to use multiple NameNode
-servers. The configuration ``instance.volumes'' should be set to a
-comma-separated list, using full URI references to different NameNode
-servers:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- <property>
- <name>instance.volumes</name>
- <value>hdfs://ns1:9001,hdfs://ns2:9001</value>
- </property>
-\end{verbatim}\endgroup
-
-The introduction of multiple volume support in 1.6 changed the way Accumulo
-stores pointers to files. It now stores fully qualified URI references to
-files. Before 1.6, Accumulo stored paths that were relative to a table
-directory. After an upgrade these relative paths will still exist and are
-resolved using instance.dfs.dir, instance.dfs.uri, and Hadoop configuration in
-the same way they were before 1.6.
-
-If the URI for a namenode changes (e.g. namenode was running on host1 and its
-moved to host2), then Accumulo will no longer function. Even if Hadoop and
-Accumulo configurations are changed, the fully qualified URIs stored in
-Accumulo will still contain the old URI. To handle this Accumulo has the
-following configuration property for replacing URI stored in its metadata. The
-example configuration below will replace ns1 with nsA and ns2 with nsB in
-Accumulo metadata. For this property to take affect, Accumulo will need to be
-restarted.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
- <property>
- <name>instance.volumes.replacements</name>
- <value>hdfs://ns1:9001 hdfs://nsA:9001, hdfs://ns2:9001 hdfs://nsB:9001</value>
- </property>
-\end{verbatim}\endgroup
-
-Using viewfs or HA namenode, introduced in Hadoop 2, offers another option for
-managing the fully qualified URIs stored in Accumulo. Viewfs and HA namenode
-both introduce a level of indirection in the Hadoop configuration. For
-example assume viewfs:///nn1 maps to hdfs://nn1 in the Hadoop configuration.
-If viewfs://nn1 is used by Accumulo, then its easy to map viewfs://nn1 to
-hdfs://nnA by changing the Hadoop configuration w/o doing anything to Accumulo.
-A production system should probably use a HA namenode. Viewfs may be useful on
-a test system with a single non HA namenode.
-
-You may also want to configure your cluster to use Federation,
-available in Hadoop 2.0, which allows DataNodes to respond to multiple
-NameNode servers, so you do not have to partition your DataNodes by
-NameNode.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/security.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/security.tex b/docs/src/main/latex/accumulo_user_manual/chapters/security.tex
deleted file mode 100644
index 83cfb21..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/security.tex
+++ /dev/null
@@ -1,179 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Security}
-
-Accumulo extends the BigTable data model to implement a security mechanism
-known as cell-level security. Every key-value pair has its own security label, stored
-under the column visibility element of the key, which is used to determine whether
-a given user meets the security requirements to read the value. This enables data of
-various security levels to be stored within the same row, and users of varying
-degrees of access to query the same table, while preserving data confidentiality.
-
-\section{Security Label Expressions}
-
-When mutations are applied, users can specify a security label for each value. This is
-done as the Mutation is created by passing a ColumnVisibility object to the put()
-method:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Text rowID = new Text("row1");
-Text colFam = new Text("myColFam");
-Text colQual = new Text("myColQual");
-ColumnVisibility colVis = new ColumnVisibility("public");
-long timestamp = System.currentTimeMillis();
-
-Value value = new Value("myValue");
-
-Mutation mutation = new Mutation(rowID);
-mutation.put(colFam, colQual, colVis, timestamp, value);
-\end{verbatim}\endgroup
-
-\section{Security Label Expression Syntax}
-
-Security labels consist of a set of user-defined tokens that are required to read the
-value the label is associated with. The set of tokens required can be specified using
-syntax that supports logical AND and OR combinations of tokens, as well as nesting
-groups of tokens together.
-
-For example, suppose within our organization we want to label our data values with
-security labels defined in terms of user roles. We might have tokens such as:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-admin
-audit
-system
-\end{verbatim}\endgroup
-
-These can be specified alone or combined using logical operators:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// Users must have admin privileges:
-admin
-
-// Users must have admin and audit privileges
-admin&audit
-
-// Users with either admin or audit privileges
-admin|audit
-
-// Users must have audit and one or both of admin or system
-(admin|system)&audit
-\end{verbatim}\endgroup
-
-When both \verb^|^ and \verb^&^ operators are used, parentheses must be used to specify
-precedence of the operators.
-
-\section{Authorization}
-
-When clients attempt to read data from Accumulo, any security labels present are
-examined against the set of authorizations passed by the client code when the
-Scanner or BatchScanner are created. If the authorizations are determined to be
-insufficient to satisfy the security label, the value is suppressed from the set of
-results sent back to the client.
-
-Authorizations are specified as a comma-separated list of tokens the user possesses:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-// user possess both admin and system level access
-Authorization auths = new Authorization("admin","system");
-
-Scanner s = connector.createScanner("table", auths);
-\end{verbatim}\endgroup
-
-\section{User Authorizations}
-
-Each Accumulo user has a set of associated security labels. To manipulate
-these in the shell while using the default authorizor, use the setuaths and getauths commands.
-These may also be modified for the default authorizor using the java security operations API.
-
-When a user creates a scanner a set of Authorizations is passed. If the
-authorizations passed to the scanner are not a subset of the users
-authorizations, then an exception will be thrown.
-
-To prevent users from writing data they can not read, add the visibility
-constraint to a table. Use the -evc option in the createtable shell command to
-enable this constraint. For existing tables use the following shell command to
-enable the visibility constraint. Ensure the constraint number does not
-conflict with any existing constraints.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-config -t table -s table.constraint.1=org.apache.accumulo.core.security.VisibilityConstraint
-\end{verbatim}\endgroup
-
-Any user with the alter table permission can add or remove this constraint.
-This constraint is not applied to bulk imported data, if this a concern then
-disable the bulk import permission.
-
-\section{Pluggable Security}
-
-New in 1.5 of Accumulo is a pluggable security mechanism. It can be broken into three actions-
-authentication, authorization, and permission handling. By default all of these are handled in
-Zookeeper, which is how things were handled in Accumulo 1.4 and before. It is worth noting at this
-point, that it is a new feature in 1.5 and may be adjusted in future releases without the standard
-deprecation cycle.
-
-Authentication simply handles the ability for a user to verify their integrity. A combination of
-principal and authentication token are used to verify a user is who they say they are. An
-authentication token should be constructed, either directly through it's constructor, but it is
-advised to use the init(Property) method to populate an authentication token. It is expected that a
-user knows what the appropriate token to use for their system is. The default token is
-PasswordToken.
-
-Once a user is authenticated by the Authenticator, the user has access to the other actions within
-Accumulo. All actions in Accumulo are ACLed, and this ACL check is handled by the Permission
-Handler. This is what manages all of the permissions, which are divided in system and per table
-level. From there, if a user is doing an action which requires authorizations, the Authorizor is
-queried to determine what authorizations the user has.
-
-This setup allows a variety of different mechanisms to be used for handling different aspects of
-Accumulo's security. A system like Kerberos can be used for authentication, then a system like LDAP
-could be used to determine if a user has a specific permission, and then it may default back to the
-default ZookeeperAuthorizor to determine what Authorizations a user is ultimately allowed to use.
-This is a pluggable system so custom components can be created depending on your need.
-
-\section{Secure Authorizations Handling}
-
-For applications serving many users, it is not expected that an Accumulo user
-will be created for each application user. In this case an Accumulo user with
-all authorizations needed by any of the applications users must be created. To
-service queries, the application should create a scanner with the application
-user's authorizations. These authorizations could be obtained from a trusted 3rd
-party.
-
-Often production systems will integrate with Public-Key Infrastructure (PKI) and
-designate client code within the query layer to negotiate with PKI servers in order
-to authenticate users and retrieve their authorization tokens (credentials). This
-requires users to specify only the information necessary to authenticate themselves
-to the system. Once user identity is established, their credentials can be accessed by
-the client code and passed to Accumulo outside of the reach of the user.
-
-\section{Query Services Layer}
-
-Since the primary method of interaction with Accumulo is through the Java API,
-production environments often call for the implementation of a Query layer. This
-can be done using web services in containers such as Apache Tomcat, but is not a
-requirement. The Query Services Layer provides a mechanism for providing a
-platform on which user facing applications can be built. This allows the application
-designers to isolate potentially complex query logic, and enables a convenient point
-at which to perform essential security functions.
-
-Several production environments choose to implement authentication at this layer,
-where users identifiers are used to retrieve their access credentials which are then
-cached within the query layer and presented to Accumulo through the
-Authorizations mechanism.
-
-Typically, the query services layer sits between Accumulo and user workstations.
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/shell.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/shell.tex b/docs/src/main/latex/accumulo_user_manual/chapters/shell.tex
deleted file mode 100644
index f3c11ff..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/shell.tex
+++ /dev/null
@@ -1,138 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Accumulo Shell}
-Accumulo provides a simple shell that can be used to examine the contents and
-configuration settings of tables, insert/update/delete values, and change
-configuration settings.
-
-The shell can be started by the following command:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-$ACCUMULO_HOME/bin/accumulo shell -u [username]
-\end{verbatim}\endgroup
-
-The shell will prompt for the corresponding password to the username specified
-and then display the following prompt:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Shell - Apache Accumulo Interactive Shell
--
-- version 1.5
-- instance name: myinstance
-- instance id: 00000000-0000-0000-0000-000000000000
--
-- type 'help' for a list of available commands
--
-\end{verbatim}\endgroup
-
-\section{Basic Administration}
-
-The Accumulo shell can be used to create and delete tables, as well as to configure
-table and instance specific options.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> tables
-accumulo.metadata
-accumulo.root
-
-root@myinstance> createtable mytable
-
-root@myinstance mytable>
-
-root@myinstance mytable> tables
-accumulo.metadata
-accumulo.root
-mytable
-
-root@myinstance mytable> createtable testtable
-
-root@myinstance testtable>
-
-root@myinstance testtable> deletetable testtable
-deletetable { testtable } (yes|no)? yes
-Table: [testtable] has been deleted.
-
-root@myinstance>
-\end{verbatim}\endgroup
-
-The Shell can also be used to insert updates and scan tables. This is useful for
-inspecting tables.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance mytable> scan
-
-root@myinstance mytable> insert row1 colf colq value1
-insert successful
-
-root@myinstance mytable> scan
-row1 colf:colq [] value1
-\end{verbatim}\endgroup
-
-The value in brackets "[]" would be the visibility labels. Since none were used, this is empty for this row.
-You can use the "-st" option to scan to see the timestamp for the cell, too.
-
-\section{Table Maintenance}
-
-The \textbf{compact} command instructs Accumulo to schedule a compaction of the table during which
-files are consolidated and deleted entries are removed.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance mytable> compact -t mytable
-07 16:13:53,201 [shell.Shell] INFO : Compaction of table mytable started for given range
-\end{verbatim}\endgroup
-
-The \textbf{flush} command instructs Accumulo to write all entries currently in memory for a given table
-to disk.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance mytable> flush -t mytable
-07 16:14:19,351 [shell.Shell] INFO : Flush of table mytable
-initiated...
-\end{verbatim}\endgroup
-
-\section{User Administration}
-
-The Shell can be used to add, remove, and grant privileges to users.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance mytable> createuser bob
-Enter new password for 'bob': *********
-Please confirm new password for 'bob': *********
-
-root@myinstance mytable> authenticate bob
-Enter current password for 'bob': *********
-Valid
-
-root@myinstance mytable> grant System.CREATE_TABLE -s -u bob
-
-root@myinstance mytable> user bob
-Enter current password for 'bob': *********
-
-bob@myinstance mytable> userpermissions
-System permissions: System.CREATE_TABLE
-Table permissions (accumulo.metadata): Table.READ
-Table permissions (mytable): NONE
-
-bob@myinstance mytable> createtable bobstable
-bob@myinstance bobstable>
-
-bob@myinstance bobstable> user root
-Enter current password for 'root': *********
-
-root@myinstance bobstable> revoke System.CREATE_TABLE -s -u bob
-\end{verbatim}\endgroup
-
http://git-wip-us.apache.org/repos/asf/accumulo/blob/900d6abb/docs/src/main/latex/accumulo_user_manual/chapters/table_configuration.tex
----------------------------------------------------------------------
diff --git a/docs/src/main/latex/accumulo_user_manual/chapters/table_configuration.tex b/docs/src/main/latex/accumulo_user_manual/chapters/table_configuration.tex
deleted file mode 100644
index a19cb52..0000000
--- a/docs/src/main/latex/accumulo_user_manual/chapters/table_configuration.tex
+++ /dev/null
@@ -1,663 +0,0 @@
-
-% Licensed to the Apache Software Foundation (ASF) under one or more
-% contributor license agreements. See the NOTICE file distributed with
-% this work for additional information regarding copyright ownership.
-% The ASF licenses this file to You under the Apache License, Version 2.0
-% (the "License"); you may not use this file except in compliance with
-% the License. You may obtain a copy of the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS,
-% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-% See the License for the specific language governing permissions and
-% limitations under the License.
-
-\chapter{Table Configuration}
-
-Accumulo tables have a few options that can be configured to alter the default
-behavior of Accumulo as well as improve performance based on the data stored.
-These include locality groups, constraints, bloom filters, iterators, and block
-cache. For a complete list of available configuration options, see
-Appendix~\ref{app:config}.
-
-\section{Locality Groups}
-Accumulo supports storing sets of column families separately on disk to allow
-clients to efficiently scan over columns that are frequently used together and to avoid
-scanning over column families that are not requested. After a locality group is set,
-Scanner and BatchScanner operations will automatically take advantage of them
-whenever the fetchColumnFamilies() method is used.
-
-By default, tables place all column families into the same ``default'' locality group.
-Additional locality groups can be configured anytime via the shell or
-programmatically as follows:
-
-\subsection{Managing Locality Groups via the Shell}
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-usage: setgroups <group>=<col fam>{,<col fam>}{ <group>=<col fam>{,<col
-fam>}} [-?] -t <table>
-
-user@myinstance mytable> setgroups group_one=colf1,colf2 -t mytable
-
-user@myinstance mytable> getgroups -t mytable
-\end{verbatim}\endgroup
-
-\subsection{Managing Locality Groups via the Client API}
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Connector conn;
-
-HashMap<String,Set<Text>> localityGroups = new HashMap<String, Set<Text>>();
-
-HashSet<Text> metadataColumns = new HashSet<Text>();
-metadataColumns.add(new Text("domain"));
-metadataColumns.add(new Text("link"));
-
-HashSet<Text> contentColumns = new HashSet<Text>();
-contentColumns.add(new Text("body"));
-contentColumns.add(new Text("images"));
-
-localityGroups.put("metadata", metadataColumns);
-localityGroups.put("content", contentColumns);
-
-conn.tableOperations().setLocalityGroups("mytable", localityGroups);
-
-// existing locality groups can be obtained as follows
-Map<String, Set<Text>> groups =
- conn.tableOperations().getLocalityGroups("mytable");
-\end{verbatim}\endgroup
-
-The assignment of Column Families to Locality Groups can be changed anytime. The
-physical movement of column families into their new locality groups takes place via
-the periodic Major Compaction process that takes place continuously in the
-background. Major Compaction can also be scheduled to take place immediately
-through the shell:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance mytable> compact -t mytable
-\end{verbatim}\endgroup
-
-\section{Constraints}
-
-Accumulo supports constraints applied on mutations at insert time. This can be
-used to disallow certain inserts according to a user defined policy. Any mutation
-that fails to meet the requirements of the constraint is rejected and sent back to the
-client.
-
-Constraints can be enabled by setting a table property as follows:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance mytable> constraint -t mytable -a com.test.ExampleConstraint com.test.AnotherConstraint
-user@myinstance mytable> constraint -l
-com.test.ExampleConstraint=1
-com.test.AnotherConstraint=2
-\end{verbatim}\endgroup
-
-Currently there are no general-purpose constraints provided with the Accumulo
-distribution. New constraints can be created by writing a Java class that implements
-the following interface:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-org.apache.accumulo.core.constraints.Constraint
-\end{verbatim}\endgroup
-
-To deploy a new constraint, create a jar file containing the class implementing the
-new constraint and place it in the lib directory of the Accumulo installation. New
-constraint jars can be added to Accumulo and enabled without restarting but any
-change to an existing constraint class requires Accumulo to be restarted.
-
-An example of constraints can be found in\\
-\texttt{accumulo/docs/examples/README.constraints} with corresponding code under\\
-\texttt{accumulo/examples/simple/src/main/java/accumulo/examples/simple/constraints} .
-
-\section{Bloom Filters}
-As mutations are applied to an Accumulo table, several files are created per tablet. If
-bloom filters are enabled, Accumulo will create and load a small data structure into
-memory to determine whether a file contains a given key before opening the file.
-This can speed up lookups considerably.
-
-To enable bloom filters, enter the following command in the Shell:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance> config -t mytable -s table.bloom.enabled=true
-\end{verbatim}\endgroup
-
-An extensive example of using Bloom Filters can be found at\\
-\texttt{accumulo/docs/examples/README.bloom} .
-
-\section{Iterators}
-Iterators provide a modular mechanism for adding functionality to be executed by
-TabletServers when scanning or compacting data. This allows users to efficiently
-summarize, filter, and aggregate data. In fact, the built-in features of cell-level
-security and column fetching are implemented using Iterators.
-Some useful Iterators are provided with Accumulo and can be found in the org.apache.accumulo.core.iterators.user package.
-In each case, any custom Iterators must be included in Accumulo's classpath,
-typically by including a jar in \texttt{\$ACCUMULO\_HOME/lib} or
-\texttt{\$ACCUMULO\_HOME/lib/ext}, although the VFS classloader allows for
-classpath manipulation using a variety of schemes including URLs and HDFS URIs.
-
-\subsection{Setting Iterators via the Shell}
-
-Iterators can be configured on a table at scan, minor compaction and/or major
-compaction scopes. If the Iterator implements the OptionDescriber interface, the
-setiter command can be used which will interactively prompt the user to provide
-values for the given necessary options.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-usage: setiter [-?] -ageoff | -agg | -class <name> | -regex |
--reqvis | -vers [-majc] [-minc] [-n <itername>] -p <pri>
-[-scan] [-t <table>]
-
-user@myinstance mytable> setiter -t mytable -scan -p 15 -n myiter -class com.company.MyIterator
-\end{verbatim}\endgroup
-
-The config command can always be used to manually configure iterators which is useful
-in cases where the Iterator does not implement the OptionDescriber interface.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-config -t mytable -s table.iterator.scan.myiter=15,com.company.MyIterator
-config -t mytable -s table.iterator.minc.myiter=15,com.company.MyIterator
-config -t mytable -s table.iterator.majc.myiter=15,com.company.MyIterator
-config -t mytable -s table.iterator.scan.myiter.opt.myoptionname=myoptionvalue
-config -t mytable -s table.iterator.minc.myiter.opt.myoptionname=myoptionvalue
-config -t mytable -s table.iterator.majc.myiter.opt.myoptionname=myoptionvalue
-\end{verbatim}\endgroup
-
-\subsection{Setting Iterators Programmatically}
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-scanner.addIterator(new IteratorSetting(
- 15, // priority
- "myiter", // name this iterator
- "com.company.MyIterator" // class name
-));
-\end{verbatim}\endgroup
-
-Some iterators take additional parameters from client code, as in the following
-example:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-IteratorSetting iter = new IteratorSetting(...);
-iter.addOption("myoptionname", "myoptionvalue");
-scanner.addIterator(iter)
-\end{verbatim}\endgroup
-
-Tables support separate Iterator settings to be applied at scan time, upon minor
-compaction and upon major compaction. For most uses, tables will have identical
-iterator settings for all three to avoid inconsistent results.
-
-\subsection{Versioning Iterators and Timestamps}
-
-Accumulo provides the capability to manage versioned data through the use of
-timestamps within the Key. If a timestamp is not specified in the key created by the
-client then the system will set the timestamp to the current time. Two keys with
-identical rowIDs and columns but different timestamps are considered two versions
-of the same key. If two inserts are made into Accumulo with the same rowID,
-column, and timestamp, then the behavior is non-deterministic.
-
-Timestamps are sorted in descending order, so the most recent data comes first.
-Accumulo can be configured to return the top k versions, or versions later than a
-given date. The default is to return the one most recent version.
-
-The version policy can be changed by changing the VersioningIterator options for a
-table as follows:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance mytable> config -t mytable -s table.iterator.scan.vers.opt.maxVersions=3
-
-user@myinstance mytable> config -t mytable -s table.iterator.minc.vers.opt.maxVersions=3
-
-user@myinstance mytable> config -t mytable -s table.iterator.majc.vers.opt.maxVersions=3
-\end{verbatim}\endgroup
-
-When a table is created, by default its configured to use the
-VersioningIterator and keep one version. A table can be created without the
-VersioningIterator with the -ndi option in the shell. Also the Java API
-has the following method
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-connector.tableOperations.create(String tableName, boolean limitVersion).
-\end{verbatim}\endgroup
-
-
-\subsubsection{Logical Time}
-
-Accumulo 1.2 introduces the concept of logical time. This ensures that timestamps
-set by Accumulo always move forward. This helps avoid problems caused by
-TabletServers that have different time settings. The per tablet counter gives unique
-one up time stamps on a per mutation basis. When using time in milliseconds, if
-two things arrive within the same millisecond then both receive the same
-timestamp. When using time in milliseconds, Accumulo set times will still
-always move forward and never backwards.
-
-A table can be configured to use logical timestamps at creation time as follows:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance> createtable -tl logical
-\end{verbatim}\endgroup
-
-\subsubsection{Deletes}
-Deletes are special keys in Accumulo that get sorted along will all the other data.
-When a delete key is inserted, Accumulo will not show anything that has a
-timestamp less than or equal to the delete key. During major compaction, any keys
-older than a delete key are omitted from the new file created, and the omitted keys
-are removed from disk as part of the regular garbage collection process.
-
-\subsection{Filters}
-When scanning over a set of key-value pairs it is possible to apply an arbitrary
-filtering policy through the use of a Filter. Filters are types of iterators that return
-only key-value pairs that satisfy the filter logic. Accumulo has a few built-in filters
-that can be configured on any table: AgeOff, ColumnAgeOff, Timestamp, NoVis, and RegEx. More can be added
-by writing a Java class that extends the\\
-org.apache.accumulo.core.iterators.Filter class.
-
-The AgeOff filter can be configured to remove data older than a certain date or a fixed
-amount of time from the present. The following example sets a table to delete
-everything inserted over 30 seconds ago:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@myinstance> createtable filtertest
-user@myinstance filtertest> setiter -t filtertest -scan -minc -majc -p 10 -n myfilter -ageoff
-AgeOffFilter removes entries with timestamps more than <ttl> milliseconds old
-----------> set org.apache.accumulo.core.iterators.user.AgeOffFilter parameter negate, default false
- keeps k/v that pass accept method, true rejects k/v that pass accept method:
-----------> set org.apache.accumulo.core.iterators.user.AgeOffFilter parameter ttl, time to
- live (milliseconds): 3000
-----------> set org.apache.accumulo.core.iterators.user.AgeOffFilter parameter currentTime, if set,
- use the given value as the absolute time in milliseconds as the current time of day:
-user@myinstance filtertest>
-user@myinstance filtertest> scan
-user@myinstance filtertest> insert foo a b c
-user@myinstance filtertest> scan
-foo a:b [] c
-user@myinstance filtertest> sleep 4
-user@myinstance filtertest> scan
-user@myinstance filtertest>
-\end{verbatim}\endgroup
-
-To see the iterator settings for a table, use:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-user@example filtertest> config -t filtertest -f iterator
----------+---------------------------------------------+------------------
-SCOPE | NAME | VALUE
----------+---------------------------------------------+------------------
-table | table.iterator.majc.myfilter .............. | 10,org.apache.accumulo.core.iterators.user.AgeOffFilter
-table | table.iterator.majc.myfilter.opt.ttl ...... | 3000
-table | table.iterator.majc.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
-table | table.iterator.majc.vers.opt.maxVersions .. | 1
-table | table.iterator.minc.myfilter .............. | 10,org.apache.accumulo.core.iterators.user.AgeOffFilter
-table | table.iterator.minc.myfilter.opt.ttl ...... | 3000
-table | table.iterator.minc.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
-table | table.iterator.minc.vers.opt.maxVersions .. | 1
-table | table.iterator.scan.myfilter .............. | 10,org.apache.accumulo.core.iterators.user.AgeOffFilter
-table | table.iterator.scan.myfilter.opt.ttl ...... | 3000
-table | table.iterator.scan.vers .................. | 20,org.apache.accumulo.core.iterators.VersioningIterator
-table | table.iterator.scan.vers.opt.maxVersions .. | 1
----------+---------------------------------------------+------------------
-\end{verbatim}\endgroup
-
-\subsection{Combiners}
-
-Accumulo allows Combiners to be configured on tables and column
-families. When a Combiner is set it is applied across the values
-associated with any keys that share rowID, column family, and column qualifier.
-This is similar to the reduce step in MapReduce, which applied some function to all
-the values associated with a particular key.
-
-For example, if a summing combiner were configured on a table and the following
-mutations were inserted:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-Row Family Qualifier Timestamp Value
-rowID1 colfA colqA 20100101 1
-rowID1 colfA colqA 20100102 1
-\end{verbatim}\endgroup
-
-The table would reflect only one aggregate value:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-rowID1 colfA colqA - 2
-\end{verbatim}\endgroup
-
-Combiners can be enabled for a table using the setiter command in the shell. Below is an example.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@a14 perDayCounts> setiter -t perDayCounts -p 10 -scan -minc -majc -n daycount
- -class org.apache.accumulo.core.iterators.user.SummingCombiner
-TypedValueCombiner can interpret Values as a variety of number encodings
- (VLong, Long, or String) before combining
-----------> set SummingCombiner parameter columns,
- <col fam>[:<col qual>]{,<col fam>[:<col qual>]} : day
-----------> set SummingCombiner parameter type, <VARNUM|LONG|STRING>: STRING
-
-root@a14 perDayCounts> insert foo day 20080101 1
-root@a14 perDayCounts> insert foo day 20080101 1
-root@a14 perDayCounts> insert foo day 20080103 1
-root@a14 perDayCounts> insert bar day 20080101 1
-root@a14 perDayCounts> insert bar day 20080101 1
-
-root@a14 perDayCounts> scan
-bar day:20080101 [] 2
-foo day:20080101 [] 2
-foo day:20080103 [] 1
-\end{verbatim}\endgroup
-
-Accumulo includes some useful Combiners out of the box. To find these look in
-the\\ \texttt{org.apache.accumulo.core.iterators.user} package.
-
-Additional Combiners can be added by creating a Java class that extends\\
-\texttt{org.apache.accumulo.core.iterators.Combiner} and adding a jar containing that
-class to Accumulo's lib/ext directory.
-
-An example of a Combiner can be found under
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-accumulo/examples/simple/src/main/java/org/apache/accumulo/examples/simple/combiner/StatsCombiner.java
-\end{verbatim}\endgroup
-
-
-\section{Block Cache}
-
-In order to increase throughput of commonly accessed entries, Accumulo employs a block cache.
-This block cache buffers data in memory so that it doesn't have to be read off of disk.
-The RFile format that Accumulo prefers is a mix of index blocks and data blocks, where the index blocks are used to find the appropriate data blocks.
-Typical queries to Accumulo result in a binary search over several index blocks followed by a linear scan of one or more data blocks.
-
-The block cache can be configured on a per-table basis, and all tablets hosted on a tablet server share a single resource pool.
-To configure the size of the tablet server's block cache, set the following properties:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-tserver.cache.data.size: Specifies the size of the cache for file data blocks.
-tserver.cache.index.size: Specifies the size of the cache for file indices.
-\end{verbatim}\endgroup
-
-To enable the block cache for your table, set the following properties:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-table.cache.block.enable: Determines whether file (data) block cache is enabled.
-table.cache.index.enable: Determines whether index cache is enabled.
-\end{verbatim}\endgroup
-
-The block cache can have a significant effect on alleviating hot spots, as well as reducing query latency.
-It is enabled by default for the metadata tables.
-
-\section{Compaction}
-
-As data is written to Accumulo it is buffered in memory. The data buffered in
-memory is eventually written to HDFS on a per tablet basis. Files can also be
-added to tablets directly by bulk import. In the background tablet servers run
-major compactions to merge multiple files into one. The tablet server has to
-decide which tablets to compact and which files within a tablet to compact.
-This decision is made using the compaction ratio, which is configurable on a
-per table basis. To configure this ratio modify the following property:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-table.compaction.major.ratio
-\end{verbatim}\endgroup
-
-Increasing this ratio will result in more files per tablet and less compaction
-work. More files per tablet means more higher query latency. So adjusting
-this ratio is a trade off between ingest and query performance. The ratio
-defaults to 3.
-
-The way the ratio works is that a set of files is compacted into one file if the
-sum of the sizes of the files in the set is larger than the ratio multiplied by
-the size of the largest file in the set. If this is not true for the set of all
-files in a tablet, the largest file is removed from consideration, and the
-remaining files are considered for compaction. This is repeated until a
-compaction is triggered or there are no files left to consider.
-
-The number of background threads tablet servers use to run major compactions is
-configurable. To configure this modify the following property:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-tserver.compaction.major.concurrent.max
-\end{verbatim}\endgroup
-
-Also, the number of threads tablet servers use for minor compactions is
-configurable. To configure this modify the following property:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-tserver.compaction.minor.concurrent.max
-\end{verbatim}\endgroup
-
-The numbers of minor and major compactions running and queued is visible on the
-Accumulo monitor page. This allows you to see if compactions are backing up
-and adjustments to the above settings are needed. When adjusting the number of
-threads available for compactions, consider the number of cores and other tasks
-running on the nodes such as maps and reduces.
-
-If major compactions are not keeping up, then the number of files per tablet
-will grow to a point such that query performance starts to suffer. One way to
-handle this situation is to increase the compaction ratio. For example, if the
-compaction ratio were set to 1, then every new file added to a tablet by minor
-compaction would immediately queue the tablet for major compaction. So if a
-tablet has a 200M file and minor compaction writes a 1M file, then the major
-compaction will attempt to merge the 200M and 1M file. If the tablet server
-has lots of tablets trying to do this sort of thing, then major compactions
-will back up and the number of files per tablet will start to grow, assuming
-data is being continuously written. Increasing the compaction ratio will
-alleviate backups by lowering the amount of major compaction work that needs to
-be done.
-
-Another option to deal with the files per tablet growing too large is to adjust
-the following property:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-table.file.max
-\end{verbatim}\endgroup
-
-When a tablet reaches this number of files and needs to flush its in-memory
-data to disk, it will choose to do a merging minor compaction. A merging minor
-compaction will merge the tablet's smallest file with the data in memory at
-minor compaction time. Therefore the number of files will not grow beyond this
-limit. This will make minor compactions take longer, which will cause ingest
-performance to decrease. This can cause ingest to slow down until major
-compactions have enough time to catch up. When adjusting this property, also
-consider adjusting the compaction ratio. Ideally, merging minor compactions
-never need to occur and major compactions will keep up. It is possible to
-configure the file max and compaction ratio such that only merging minor
-compactions occur and major compactions never occur. This should be avoided
-because doing only merging minor compactions causes $O(N^2)$ work to be done.
-The amount of work done by major compactions is $O(N*\log_R(N))$ where
-\textit{R} is the compaction ratio.
-
-Compactions can be initiated manually for a table. To initiate a minor
-compaction, use the flush command in the shell. To initiate a major compaction,
-use the compact command in the shell. The compact command will compact all
-tablets in a table to one file. Even tablets with one file are compacted. This
-is useful for the case where a major compaction filter is configured for a
-table. In 1.4 the ability to compact a range of a table was added. To use this
-feature specify start and stop rows for the compact command. This will only
-compact tablets that overlap the given row range.
-
-\section{Pre-splitting tables}
-
-Accumulo will balance and distribute tables across servers. Before a
-table gets large, it will be maintained as a single tablet on a single
-server. This limits the speed at which data can be added or queried
-to the speed of a single node. To improve performance when the a table
-is new, or small, you can add split points and generate new tablets.
-
-In the shell:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> createtable newTable
-root@myinstance> addsplits -t newTable g n t
-\end{verbatim}\endgroup
-
-This will create a new table with 4 tablets. The table will be split
-on the letters ``g'', ``n'', and ``t'' which will work nicely if the
-row data start with lower-case alphabetic characters. If your row
-data includes binary information or numeric information, or if the
-distribution of the row information is not flat, then you would pick
-different split points. Now ingest and query can proceed on 4 nodes
-which can improve performance.
-
-\section{Merging tablets}
-
-Over time, a table can get very large, so large that it has hundreds
-of thousands of split points. Once there are enough tablets to spread
-a table across the entire cluster, additional splits may not improve
-performance, and may create unnecessary bookkeeping. The distribution
-of data may change over time. For example, if row data contains date
-information, and data is continually added and removed to maintain a
-window of current information, tablets for older rows may be empty.
-
-Accumulo supports tablet merging, which can be used to reduce
-the number of split points. The following command will merge all rows
-from ``A'' to ``Z'' into a single tablet:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> merge -t myTable -s A -e Z
-\end{verbatim}\endgroup
-
-If the result of a merge produces a tablet that is larger than the
-configured split size, the tablet may be split by the tablet server.
-Be sure to increase your tablet size prior to any merges if the goal
-is to have larger tablets:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> config -t myTable -s table.split.threshold=2G
-\end{verbatim}\endgroup
-
-In order to merge small tablets, you can ask Accumulo to merge
-sections of a table smaller than a given size.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> merge -t myTable -s 100M
-\end{verbatim}\endgroup
-
-By default, small tablets will not be merged into tablets that are
-already larger than the given size. This can leave isolated small
-tablets. To force small tablets to be merged into larger tablets use
-the ``-{}-force'' option:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> merge -t myTable -s 100M --force
-\end{verbatim}\endgroup
-
-Merging away small tablets works on one section at a time. If your
-table contains many sections of small split points, or you are
-attempting to change the split size of the entire table, it will be
-faster to set the split point and merge the entire table:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> config -t myTable -s table.split.threshold=256M
-root@myinstance> merge -t myTable
-\end{verbatim}\endgroup
-
-\section{Delete Range}
-
-Consider an indexing scheme that uses date information in each row.
-For example ``20110823-15:20:25.013'' might be a row that specifies a
-date and time. In some cases, we might like to delete rows based on
-this date, say to remove all the data older than the current year.
-Accumulo supports a delete range operation which efficiently
-removes data between two rows. For example:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> deleterange -t myTable -s 2010 -e 2011
-\end{verbatim}\endgroup
-
-This will delete all rows starting with ``2010'' and it will stop at
-any row starting ``2011''. You can delete any data prior to 2011
-with:
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@myinstance> deleterange -t myTable -e 2011 --force
-\end{verbatim}\endgroup
-
-The shell will not allow you to delete an unbounded range (no start)
-unless you provide the ``-{}-force'' option.
-
-Range deletion is implemented using splits at the given start/end
-positions, and will affect the number of splits in the table.
-
-\section{Cloning Tables}
-
-A new table can be created that points to an existing table's data. This is a
-very quick metadata operation, no data is actually copied. The cloned table
-and the source table can change independently after the clone operation. One
-use case for this feature is testing. For example to test a new filtering
-iterator, clone the table, add the filter to the clone, and force a major
-compaction. To perform a test on less data, clone a table and then use delete
-range to efficiently remove a lot of data from the clone. Another use case is
-generating a snapshot to guard against human error. To create a snapshot,
-clone a table and then disable write permissions on the clone.
-
-The clone operation will point to the source table's files. This is why the
-flush option is present and is enabled by default in the shell. If the flush
-option is not enabled, then any data the source table currently has in memory
-will not exist in the clone.
-
-A cloned table copies the configuration of the source table. However the
-permissions of the source table are not copied to the clone. After a clone is
-created, only the user that created the clone can read and write to it.
-
-In the following example we see that data inserted after the clone operation is
-not visible in the clone.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@a14> createtable people
-root@a14 people> insert 890435 name last Doe
-root@a14 people> insert 890435 name first John
-root@a14 people> clonetable people test
-root@a14 people> insert 890436 name first Jane
-root@a14 people> insert 890436 name last Doe
-root@a14 people> scan
-890435 name:first [] John
-890435 name:last [] Doe
-890436 name:first [] Jane
-890436 name:last [] Doe
-root@a14 people> table test
-root@a14 test> scan
-890435 name:first [] John
-890435 name:last [] Doe
-root@a14 test>
-\end{verbatim}\endgroup
-
-The du command in the shell shows how much space a table is using in HDFS.
-This command can also show how much overlapping space two cloned tables have in
-HDFS. In the example below du shows table ci is using 428M. Then ci is cloned
-to cic and du shows that both tables share 428M. After three entries are
-inserted into cic and its flushed, du shows the two tables still share 428M but
-cic has 226 bytes to itself. Finally, table cic is compacted and then du shows
-that each table uses 428M.
-
-\begingroup\fontsize{8pt}{8pt}\selectfont\begin{verbatim}
-root@a14> du ci
- 428,482,573 [ci]
-root@a14> clonetable ci cic
-root@a14> du ci cic
- 428,482,573 [ci, cic]
-root@a14> table cic
-root@a14 cic> insert r1 cf1 cq1 v1
-root@a14 cic> insert r1 cf1 cq2 v2
-root@a14 cic> insert r1 cf1 cq3 v3
-root@a14 cic> flush -t cic -w
-27 15:00:13,908 [shell.Shell] INFO : Flush of table cic completed.
-root@a14 cic> du ci cic
- 428,482,573 [ci, cic]
- 226 [cic]
-root@a14 cic> compact -t cic -w
-27 15:00:35,871 [shell.Shell] INFO : Compacting table ...
-27 15:03:03,303 [shell.Shell] INFO : Compaction of table cic completed for given range
-root@a14 cic> du ci cic
- 428,482,573 [ci]
- 428,482,612 [cic]
-root@a14 cic>
-\end{verbatim}\endgroup
-
-\section{Exporting Tables}
-
-Accumulo supports exporting tables for the purpose of copying tables to another
-cluster. Exporting and importing tables preserves the tables configuration,
-splits, and logical time. Tables are exported and then copied via the hadoop
-distcp command. To export a table, it must be offline and stay offline while
-discp runs. The reason it needs to stay offline is to prevent files from being
-deleted. A table can be cloned and the clone taken offline inorder to avoid
-losing access to the table. See docs/examples/README.export for an example.