You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2018/05/08 16:15:42 UTC
[accumulo] branch master updated (2d869d3 -> 49192c9)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from 2d869d3 Fixed more javadoc errors
add 80c2e9d fixes #449 fix two bugs with WAL recovery (#458)
add e56340c Fixes #449 suggested updates to PR apache/accumulo#458
add a8fead2 fixes #449 code review updates (#458)
add b8f574f fixes #477 Stop putting -1 in WAL (#458)
new 49192c9 Merge branch '1.9'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/accumulo/tserver/TabletServer.java | 10 +-
...utationReceiver.java => CloseableIterator.java} | 8 +-
.../org/apache/accumulo/tserver/log/DfsLogger.java | 8 +-
.../{MultiReader.java => RecoveryLogReader.java} | 162 ++++++++-
.../accumulo/tserver/log/RecoveryLogsIterator.java | 100 ++++++
.../accumulo/tserver/log/SortedLogRecovery.java | 378 +++++++++++----------
.../apache/accumulo/tserver/logger/LogFileKey.java | 46 +--
.../apache/accumulo/tserver/logger/LogReader.java | 16 +-
.../tserver/replication/AccumuloReplicaSystem.java | 6 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
.../accumulo/tserver/log/LogEventsTest.java} | 19 +-
.../accumulo/tserver/log/LogFileKeyTest.java | 98 ++++++
...ReaderTest.java => RecoveryLogsReaderTest.java} | 43 ++-
.../tserver/log/SortedLogRecoveryTest.java | 296 +++++++++++++++-
.../accumulo/tserver/logger/LogFileTest.java | 14 +-
.../replication/AccumuloReplicaSystemTest.java | 33 +-
.../BatchWriterReplicationReplayerTest.java | 4 +-
.../UnusedWalDoesntCloseReplicationStatusIT.java | 2 +-
18 files changed, 951 insertions(+), 294 deletions(-)
copy server/tserver/src/main/java/org/apache/accumulo/tserver/log/{MutationReceiver.java => CloseableIterator.java} (82%)
rename server/tserver/src/main/java/org/apache/accumulo/tserver/log/{MultiReader.java => RecoveryLogReader.java} (52%)
create mode 100644 server/tserver/src/main/java/org/apache/accumulo/tserver/log/RecoveryLogsIterator.java
copy server/tserver/src/{main/java/org/apache/accumulo/tserver/log/MutationReceiver.java => test/java/org/apache/accumulo/tserver/log/LogEventsTest.java} (61%)
create mode 100644 server/tserver/src/test/java/org/apache/accumulo/tserver/log/LogFileKeyTest.java
rename server/tserver/src/test/java/org/apache/accumulo/tserver/log/{MultiReaderTest.java => RecoveryLogsReaderTest.java} (77%)
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.
[accumulo] 01/01: Merge branch '1.9'
Posted by kt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 49192c9fe45de9efdac88afd0c3c96eb82822721
Merge: 2d869d3 b8f574f
Author: Keith Turner <kt...@apache.org>
AuthorDate: Tue May 8 12:13:44 2018 -0400
Merge branch '1.9'
.../org/apache/accumulo/tserver/TabletServer.java | 10 +-
.../accumulo/tserver/log/CloseableIterator.java | 25 ++
.../org/apache/accumulo/tserver/log/DfsLogger.java | 8 +-
.../{MultiReader.java => RecoveryLogReader.java} | 162 ++++++++-
.../accumulo/tserver/log/RecoveryLogsIterator.java | 100 ++++++
.../accumulo/tserver/log/SortedLogRecovery.java | 378 +++++++++++----------
.../apache/accumulo/tserver/logger/LogFileKey.java | 46 +--
.../apache/accumulo/tserver/logger/LogReader.java | 16 +-
.../tserver/replication/AccumuloReplicaSystem.java | 6 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
.../apache/accumulo/tserver/log/LogEventsTest.java | 36 ++
.../accumulo/tserver/log/LogFileKeyTest.java | 98 ++++++
...ReaderTest.java => RecoveryLogsReaderTest.java} | 43 ++-
.../tserver/log/SortedLogRecoveryTest.java | 296 +++++++++++++++-
.../accumulo/tserver/logger/LogFileTest.java | 14 +-
.../replication/AccumuloReplicaSystemTest.java | 33 +-
.../BatchWriterReplicationReplayerTest.java | 4 +-
.../UnusedWalDoesntCloseReplicationStatusIT.java | 2 +-
18 files changed, 991 insertions(+), 288 deletions(-)
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
index 914544a,bdf5210..708f843
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java
@@@ -35,7 -35,6 +35,8 @@@ import java.util.Objects
import java.util.Set;
import java.util.TreeMap;
+import org.apache.accumulo.core.client.impl.Table;
++import org.apache.accumulo.core.client.impl.Table.ID;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.data.impl.KeyExtent;
@@@ -675,6 -679,85 +681,85 @@@ public class SortedLogRecoveryTest
Assert.assertEquals(m, mutations.get(0));
}
+ @Test
+ public void testLeaveAndComeBack() throws IOException {
+ /**
+ * This test recreates the situation in bug #449 (Github issues).
+ */
+ Mutation m1 = new ServerMutation(new Text("r1"));
+ m1.put("f1", "q1", "v1");
+
+ Mutation m2 = new ServerMutation(new Text("r2"));
+ m2.put("f1", "q1", "v2");
+
+ KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+ createKeyValue(DEFINE_TABLET, 100, 10, extent), createKeyValue(MUTATION, 100, 10, m1),
+ createKeyValue(COMPACTION_START, 101, 10, "/t/f1"),
+ createKeyValue(COMPACTION_FINISH, 102, 10, null)};
+
+ KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+ createKeyValue(DEFINE_TABLET, 1, 20, extent), createKeyValue(MUTATION, 1, 20, m2)};
+
+ Arrays.sort(entries1);
+ Arrays.sort(entries2);
+ Map<String,KeyValue[]> logs = new TreeMap<>();
+ logs.put("entries1", entries1);
+ logs.put("entries2", entries2);
+
+ List<Mutation> mutations = recover(logs, extent);
+
+ Assert.assertEquals(1, mutations.size());
+ Assert.assertEquals(m2, mutations.get(0));
+ }
+
+ @Test
+ public void testMultipleTablets() throws IOException {
- KeyExtent e1 = new KeyExtent("1", new Text("m"), null);
- KeyExtent e2 = new KeyExtent("1", null, new Text("m"));
++ KeyExtent e1 = new KeyExtent(ID.of("1"), new Text("m"), null);
++ KeyExtent e2 = new KeyExtent(ID.of("1"), null, new Text("m"));
+
+ Mutation m1 = new ServerMutation(new Text("b"));
+ m1.put("f1", "q1", "v1");
+
+ Mutation m2 = new ServerMutation(new Text("b"));
+ m2.put("f1", "q2", "v2");
+
+ Mutation m3 = new ServerMutation(new Text("s"));
+ m3.put("f1", "q1", "v3");
+
+ Mutation m4 = new ServerMutation(new Text("s"));
+ m4.put("f1", "q2", "v4");
+
+ KeyValue entries1[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+ createKeyValue(DEFINE_TABLET, 7, 10, e1), createKeyValue(DEFINE_TABLET, 5, 11, e2),
+ createKeyValue(MUTATION, 8, 10, m1), createKeyValue(COMPACTION_START, 9, 10, "/t/f1"),
+ createKeyValue(MUTATION, 10, 10, m2), createKeyValue(COMPACTION_FINISH, 11, 10, null),
+ createKeyValue(MUTATION, 6, 11, m3), createKeyValue(COMPACTION_START, 7, 11, "/t/f2"),
+ createKeyValue(MUTATION, 8, 11, m4)};
+
+ Arrays.sort(entries1);
+
+ Map<String,KeyValue[]> logs = new TreeMap<>();
+ logs.put("entries1", entries1);
+
+ List<Mutation> mutations1 = recover(logs, e1);
+ Assert.assertEquals(1, mutations1.size());
+ Assert.assertEquals(m2, mutations1.get(0));
+
+ List<Mutation> mutations2 = recover(logs, e2);
+ Assert.assertEquals(2, mutations2.size());
+ Assert.assertEquals(m3, mutations2.get(0));
+ Assert.assertEquals(m4, mutations2.get(1));
+
+ KeyValue entries2[] = new KeyValue[] {createKeyValue(OPEN, 0, -1, "1"),
+ createKeyValue(DEFINE_TABLET, 9, 11, e2), createKeyValue(COMPACTION_FINISH, 10, 11, null)};
+ Arrays.sort(entries2);
+ logs.put("entries2", entries2);
+
+ mutations2 = recover(logs, e2);
+ Assert.assertEquals(1, mutations2.size());
+ Assert.assertEquals(m4, mutations2.get(0));
+ }
+
private void runPathTest(boolean startMatches, String compactionStartFile, String... tabletFiles)
throws IOException {
Mutation m1 = new ServerMutation(new Text("row1"));
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java
index 2e8d220,5e26a9d..22c83cc
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/logger/LogFileTest.java
@@@ -80,9 -79,9 +80,9 @@@ public class LogFileTest
readWrite(COMPACTION_START, 3, 4, "some file", null, null, key, value);
assertEquals(key.event, COMPACTION_START);
assertEquals(key.seq, 3);
- assertEquals(key.tid, 4);
+ assertEquals(key.tabletId, 4);
assertEquals(key.filename, "some file");
- KeyExtent tablet = new KeyExtent("table", new Text("bbbb"), new Text("aaaa"));
+ KeyExtent tablet = new KeyExtent(Table.ID.of("table"), new Text("bbbb"), new Text("aaaa"));
readWrite(DEFINE_TABLET, 5, 6, null, tablet, null, key, value);
assertEquals(key.event, DEFINE_TABLET);
assertEquals(key.seq, 5);
diff --cc server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java
index b270428,107060b..25cf68b
--- a/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java
+++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/replication/AccumuloReplicaSystemTest.java
@@@ -78,9 -78,9 +78,8 @@@ public class AccumuloReplicaSystemTest
* are solely for testing that each LogEvents is handled, order is not important.
*/
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("1", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
- key.tid = 1;
-
+ key.tabletId = 1;
-
key.write(dos);
value.write(dos);
@@@ -93,8 -93,8 +92,8 @@@
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("2", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
- key.tid = 2;
+ key.tabletId = 2;
value.mutations = Collections.emptyList();
key.write(dos);
@@@ -124,8 -124,8 +123,8 @@@
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("1", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
- key.tid = 3;
+ key.tabletId = 3;
value.mutations = Collections.emptyList();
key.write(dos);
@@@ -140,9 -140,9 +139,9 @@@
key.tablet = null;
key.event = LogEvents.MUTATION;
- key.tid = 3;
+ key.tabletId = 3;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
- value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
+ value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
@@@ -186,8 -187,8 +185,8 @@@
* are solely for testing that each LogEvents is handled, order is not important.
*/
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("1", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
- key.tid = 1;
+ key.tabletId = 1;
key.write(dos);
value.write(dos);
@@@ -201,8 -202,8 +200,8 @@@
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("2", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
- key.tid = 2;
+ key.tabletId = 2;
value.mutations = Collections.emptyList();
key.write(dos);
@@@ -232,8 -233,8 +231,8 @@@
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("1", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
- key.tid = 3;
+ key.tabletId = 3;
value.mutations = Collections.emptyList();
key.write(dos);
@@@ -248,9 -249,9 +247,9 @@@
key.tablet = null;
key.event = LogEvents.MUTATION;
- key.tid = 3;
+ key.tabletId = 3;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
- value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
+ value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
@@@ -392,8 -395,8 +391,8 @@@
* are solely for testing that each LogEvents is handled, order is not important.
*/
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent("1", null, null);
+ key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
- key.tid = 1;
+ key.tabletId = 1;
key.write(dos);
value.write(dos);
@@@ -408,9 -411,9 +407,9 @@@
key.tablet = null;
key.event = LogEvents.MUTATION;
- key.tid = 1;
+ key.tabletId = 1;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
- value.mutations = Arrays.<Mutation> asList(new ServerMutation(new Text("row")));
+ value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
diff --cc test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
index a2025bf,bdde3f7..1146c8e
--- a/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/replication/UnusedWalDoesntCloseReplicationStatusIT.java
@@@ -117,9 -116,9 +117,9 @@@ public class UnusedWalDoesntCloseReplic
value.write(out);
key.event = LogEvents.DEFINE_TABLET;
- key.tablet = new KeyExtent(Integer.toString(fakeTableId), null, null);
- key.seq = 1l;
+ key.tablet = new KeyExtent(Table.ID.of(Integer.toString(fakeTableId)), null, null);
+ key.seq = 1L;
- key.tid = 1;
+ key.tabletId = 1;
key.write(dos);
value.write(dos);
--
To stop receiving notification emails like this one, please contact
kturner@apache.org.