You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mm...@apache.org on 2021/02/16 20:29:23 UTC
[accumulo] branch main updated (0d5ea4b -> 925d17b)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git.
from 0d5ea4b Prevent deleted tablets from being flushed (#1899)
add 4b3a3b9 Improvements to TabletStateChangeIteratorIT (#1933)
new 3a1d740 Merge remote-tracking branch 'upstream/1.10' into main
new 925d17b Improvements to TabletStateChangeIteratorIT. Closes #1932
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../functional/TabletStateChangeIteratorIT.java | 62 ++++++++++++++++++----
1 file changed, 51 insertions(+), 11 deletions(-)
[accumulo] 01/02: Merge remote-tracking branch 'upstream/1.10' into
main
Posted by mm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3a1d7404b2dc8d910c8a340c19a5326ae2e3cd23
Merge: 0d5ea4b 4b3a3b9
Author: Mike Miller <mm...@apache.org>
AuthorDate: Tue Feb 16 15:13:04 2021 -0500
Merge remote-tracking branch 'upstream/1.10' into main
[accumulo] 02/02: Improvements to TabletStateChangeIteratorIT.
Closes #1932
Posted by mm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 925d17bdd72cf1bbdd0d693d31e58b02fb7d6eab
Author: Mike Miller <mm...@apache.org>
AuthorDate: Tue Feb 16 10:37:37 2021 -0500
Improvements to TabletStateChangeIteratorIT. Closes #1932
* Modified copyTable method to gather info about the metadata first
before creating the copy, preventing information about the copy itself
from being seen.
* Added debugging information
* Changed createTable method to pre split tables
---
.../functional/TabletStateChangeIteratorIT.java | 62 ++++++++++++++++++----
1 file changed, 51 insertions(+), 11 deletions(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 0a66f3d..f7597fa 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@ -20,10 +20,12 @@ package org.apache.accumulo.test.functional;
import static org.junit.Assert.assertEquals;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
@@ -43,6 +45,7 @@ import org.apache.accumulo.core.client.RowIterator;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableExistsException;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.clientImpl.ClientInfo;
import org.apache.accumulo.core.clientImpl.Tables;
@@ -68,6 +71,8 @@ import org.apache.accumulo.server.manager.state.MetaDataTableScanner;
import org.apache.accumulo.server.manager.state.TabletStateChangeIterator;
import org.apache.hadoop.io.Text;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
@@ -76,6 +81,7 @@ import com.google.common.collect.Sets;
* in the metadata table when there is no work to be done on the tablet (see ACCUMULO-3580)
*/
public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
+ private final static Logger log = LoggerFactory.getLogger(TabletStateChangeIteratorIT.class);
@Override
public int defaultTimeoutSeconds() {
@@ -105,9 +111,12 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
copyTable(client, MetadataTable.NAME, metaCopy1);
State state = new State(client);
- while (findTabletsNeedingAttention(client, metaCopy1, state) > 0) {
+ int tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, state);
+ while (tabletsInFlux > 0) {
+ log.debug("Waiting for {} tablets for {}", tabletsInFlux, metaCopy1);
UtilWaitThread.sleep(500);
copyTable(client, MetadataTable.NAME, metaCopy1);
+ tabletsInFlux = findTabletsNeedingAttention(client, metaCopy1, state);
}
assertEquals("No tables should need attention", 0,
findTabletsNeedingAttention(client, metaCopy1, state));
@@ -196,31 +205,39 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
private int findTabletsNeedingAttention(AccumuloClient client, String table, State state)
throws TableNotFoundException {
int results = 0;
+ List<Key> resultList = new ArrayList<>();
try (Scanner scanner = client.createScanner(table, Authorizations.EMPTY)) {
MetaDataTableScanner.configureScanner(scanner, state);
+ log.debug("Current state = {}", state);
scanner.updateScanIteratorOption("tabletChange", "debug", "1");
for (Entry<Key,Value> e : scanner) {
- if (e != null)
+ if (e != null) {
results++;
+ resultList.add(e.getKey());
+ }
}
}
-
+ log.debug("Tablets in flux: {}", resultList);
return results;
}
private void createTable(AccumuloClient client, String t, boolean online)
throws AccumuloSecurityException, AccumuloException, TableNotFoundException,
TableExistsException {
- client.tableOperations().create(t);
- client.tableOperations().online(t, true);
SortedSet<Text> partitionKeys = new TreeSet<>();
partitionKeys.add(new Text("some split"));
- client.tableOperations().addSplits(t, partitionKeys);
+ NewTableConfiguration ntc = new NewTableConfiguration().withSplits(partitionKeys);
+ client.tableOperations().create(t, ntc);
+ client.tableOperations().online(t, true);
if (!online) {
client.tableOperations().offline(t, true);
}
}
+ /**
+ * Create a copy of the source table by first gathering all the rows of the source in a list of
+ * mutations. Then create the copy of the table and apply the mutations to the copy.
+ */
private void copyTable(AccumuloClient client, String source, String copy)
throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
TableExistsException {
@@ -230,10 +247,10 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
// ignored
}
- client.tableOperations().create(copy);
+ log.info("Gathering rows to copy {} ", source);
+ List<Mutation> mutations = new ArrayList<>();
- try (Scanner scanner = client.createScanner(source, Authorizations.EMPTY);
- BatchWriter writer = client.createBatchWriter(copy, new BatchWriterConfig())) {
+ try (Scanner scanner = client.createScanner(source, Authorizations.EMPTY)) {
RowIterator rows = new RowIterator(new IsolatedScanner(scanner));
while (rows.hasNext()) {
@@ -250,9 +267,22 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
k.getTimestamp(), entry.getValue());
}
+ mutations.add(m);
+ }
+ }
+
+ // metadata should be stable with only 7 rows (1 replication + 2 for each table)
+ log.debug("Gathered {} rows to create copy {}", mutations.size(), copy);
+ assertEquals("Metadata should have 7 rows (1 repl + 2 for each table)", 7, mutations.size());
+ client.tableOperations().create(copy);
+
+ try (BatchWriter writer = client.createBatchWriter(copy, new BatchWriterConfig())) {
+ for (Mutation m : mutations) {
writer.addMutation(m);
}
}
+
+ log.info("Finished creating copy " + copy);
}
private void dropTables(AccumuloClient client, String... tables)
@@ -262,7 +292,7 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
}
}
- private class State implements CurrentState {
+ private static class State implements CurrentState {
final AccumuloClient client;
@@ -270,6 +300,9 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
this.client = client;
}
+ private Set<TServerInstance> tservers;
+ private Set<TableId> onlineTables;
+
@Override
public Set<TServerInstance> onlineTabletServers() {
HashSet<TServerInstance> tservers = new HashSet<>();
@@ -285,6 +318,7 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
throw new RuntimeException(e);
}
}
+ this.tservers = Collections.unmodifiableSet(tservers);
return tservers;
}
@@ -292,8 +326,9 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
public Set<TableId> onlineTables() {
ClientContext context = (ClientContext) client;
Set<TableId> onlineTables = Tables.getIdToNameMap(context).keySet();
- return Sets.filter(onlineTables,
+ this.onlineTables = Sets.filter(onlineTables,
tableId -> Tables.getTableState(context, tableId) == TableState.ONLINE);
+ return this.onlineTables;
}
@Override
@@ -315,6 +350,11 @@ public class TabletStateChangeIteratorIT extends AccumuloClusterHarness {
public ManagerState getManagerState() {
return ManagerState.NORMAL;
}
+
+ @Override
+ public String toString() {
+ return "tservers: " + tservers + " onlineTables: " + onlineTables;
+ }
}
}