You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2016/02/12 01:37:25 UTC
[8/8] accumulo git commit: Merge branch '1.7'
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/a6e6e672
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/a6e6e672
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/a6e6e672
Branch: refs/heads/master
Commit: a6e6e6723d99d6203170047b7df6a4feab4aba07
Parents: 8949fd4 69185ed
Author: Christopher Tubbs <ct...@apache.org>
Authored: Thu Feb 11 19:36:43 2016 -0500
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Thu Feb 11 19:36:43 2016 -0500
----------------------------------------------------------------------
NOTICE | 2 +-
assemble/bin-NOTICE | 2 +-
docs/src/main/asciidoc/accumulo_user_manual.asciidoc | 2 +-
.../apache/accumulo/server/master/state/CurrentState.java | 5 ++++-
.../server/master/state/MetaDataTableScanner.java | 2 +-
server/base/src/main/javadoc/META-INF/NOTICE | 2 +-
.../src/main/java/org/apache/accumulo/master/Master.java | 10 +++-------
server/monitor/src/main/javadoc/META-INF/NOTICE | 2 +-
server/native/src/main/resources/NOTICE | 2 +-
.../test/functional/TabletStateChangeIteratorIT.java | 4 ++--
.../org/apache/accumulo/test/master/MergeStateIT.java | 4 ++--
11 files changed, 18 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a6e6e672/docs/src/main/asciidoc/accumulo_user_manual.asciidoc
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a6e6e672/server/base/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a6e6e672/server/master/src/main/java/org/apache/accumulo/master/Master.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a6e6e672/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
index 82be2d2,0000000..cd44774
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/TabletStateChangeIteratorIT.java
@@@ -1,247 -1,0 +1,247 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableExistsException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.impl.Tables;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.master.thrift.MasterState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.zookeeper.ZooUtil;
+import org.apache.accumulo.fate.zookeeper.ZooCache;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.server.master.state.CurrentState;
+import org.apache.accumulo.server.master.state.MergeInfo;
+import org.apache.accumulo.server.master.state.MetaDataTableScanner;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletStateChangeIterator;
+import org.apache.accumulo.server.zookeeper.ZooLock;
+import org.apache.hadoop.io.Text;
+import org.junit.Test;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Sets;
+
+/**
+ * Test to ensure that the {@link TabletStateChangeIterator} properly skips over tablet information in the metadata table when there is no work to be done on
+ * the tablet (see ACCUMULO-3580)
+ */
+public class TabletStateChangeIteratorIT extends SharedMiniClusterBase {
+
+ @Override
+ public int defaultTimeoutSeconds() {
+ return 2 * 60;
+ }
+
+ @Test
+ public void test() throws AccumuloException, AccumuloSecurityException, TableExistsException, TableNotFoundException {
+ String[] tables = getUniqueNames(4);
+ final String t1 = tables[0];
+ final String t2 = tables[1];
+ final String t3 = tables[2];
+ final String cloned = tables[3];
+
+ // create some metadata
+ createTable(t1, true);
+ createTable(t2, false);
+ createTable(t3, true);
+
+ // examine a clone of the metadata table, so we can manipulate it
+ cloneMetadataTable(cloned);
+
+ State state = new State();
+ assertEquals("No tables should need attention", 0, findTabletsNeedingAttention(cloned, state));
+
+ // test the assigned case (no location)
+ removeLocation(cloned, t3);
+ assertEquals("Should have two tablets without a loc", 2, findTabletsNeedingAttention(cloned, state));
+
+ // test the cases where the assignment is to a dead tserver
+ getConnector().tableOperations().delete(cloned);
+ cloneMetadataTable(cloned);
+ reassignLocation(cloned, t3);
+ assertEquals("Should have one tablet that needs to be unassigned", 1, findTabletsNeedingAttention(cloned, state));
+
+ // test the cases where there is ongoing merges
+ state = new State() {
+ @Override
+ public Collection<MergeInfo> merges() {
+ String tableIdToModify = getConnector().tableOperations().tableIdMap().get(t3);
+ return Collections.singletonList(new MergeInfo(new KeyExtent(new Text(tableIdToModify), null, null), MergeInfo.Operation.MERGE));
+ }
+ };
+ assertEquals("Should have 2 tablets that need to be chopped or unassigned", 1, findTabletsNeedingAttention(cloned, state));
+
+ // test the bad tablet location state case (inconsistent metadata)
+ state = new State();
+ cloneMetadataTable(cloned);
+ addDuplicateLocation(cloned, t3);
+ assertEquals("Should have 1 tablet that needs a metadata repair", 1, findTabletsNeedingAttention(cloned, state));
+
+ // clean up
+ dropTables(t1, t2, t3, cloned);
+ }
+
+ private void addDuplicateLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException {
+ String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify);
+ Mutation m = new Mutation(new KeyExtent(new Text(tableIdToModify), null, null).getMetadataEntry());
+ m.put(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME, new Text("1234567"), new Value("fake:9005".getBytes(UTF_8)));
+ BatchWriter bw = getConnector().createBatchWriter(table, null);
+ bw.addMutation(m);
+ bw.close();
+ }
+
+ private void reassignLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException {
+ String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify);
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+ scanner.setRange(new KeyExtent(new Text(tableIdToModify), null, null).toMetadataRange());
+ scanner.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ Entry<Key,Value> entry = scanner.iterator().next();
+ Mutation m = new Mutation(entry.getKey().getRow());
+ m.putDelete(entry.getKey().getColumnFamily(), entry.getKey().getColumnQualifier(), entry.getKey().getTimestamp());
+ m.put(entry.getKey().getColumnFamily(), new Text("1234567"), entry.getKey().getTimestamp() + 1, new Value("fake:9005".getBytes(UTF_8)));
+ scanner.close();
+ BatchWriter bw = getConnector().createBatchWriter(table, null);
+ bw.addMutation(m);
+ bw.close();
+ }
+
+ private void removeLocation(String table, String tableNameToModify) throws TableNotFoundException, MutationsRejectedException {
+ String tableIdToModify = getConnector().tableOperations().tableIdMap().get(tableNameToModify);
+ BatchDeleter deleter = getConnector().createBatchDeleter(table, Authorizations.EMPTY, 1, new BatchWriterConfig());
+ deleter.setRanges(Collections.singleton(new KeyExtent(new Text(tableIdToModify), null, null).toMetadataRange()));
+ deleter.fetchColumnFamily(MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME);
+ deleter.delete();
+ deleter.close();
+ }
+
+ private int findTabletsNeedingAttention(String table, State state) throws TableNotFoundException {
+ int results = 0;
+ Scanner scanner = getConnector().createScanner(table, Authorizations.EMPTY);
+ MetaDataTableScanner.configureScanner(scanner, state);
+ scanner.updateScanIteratorOption("tabletChange", "debug", "1");
+ for (Entry<Key,Value> e : scanner) {
+ if (e != null)
+ results++;
+ }
+ return results;
+ }
+
+ private void createTable(String t, boolean online) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, TableExistsException {
+ Connector conn = getConnector();
+ conn.tableOperations().create(t);
+ conn.tableOperations().online(t, true);
+ SortedSet<Text> partitionKeys = new TreeSet<Text>();
+ partitionKeys.add(new Text("some split"));
+ conn.tableOperations().addSplits(t, partitionKeys);
+ if (!online) {
+ conn.tableOperations().offline(t, true);
+ }
+ }
+
+ private void cloneMetadataTable(String cloned) throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException {
+ try {
+ dropTables(cloned);
+ } catch (TableNotFoundException ex) {
+ // ignored
+ }
+ getConnector().tableOperations().clone(MetadataTable.NAME, cloned, true, null, null);
+ }
+
+ private void dropTables(String... tables) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ for (String t : tables) {
+ getConnector().tableOperations().delete(t);
+ }
+ }
+
+ private class State implements CurrentState {
+
+ @Override
+ public Set<TServerInstance> onlineTabletServers() {
+ HashSet<TServerInstance> tservers = new HashSet<TServerInstance>();
+ for (String tserver : getConnector().instanceOperations().getTabletServers()) {
+ try {
+ String zPath = ZooUtil.getRoot(getConnector().getInstance()) + Constants.ZTSERVERS + "/" + tserver;
+ long sessionId = ZooLock.getSessionId(new ZooCache(getCluster().getZooKeepers(), getConnector().getInstance().getZooKeepersSessionTimeOut()), zPath);
+ tservers.add(new TServerInstance(tserver, sessionId));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return tservers;
+ }
+
+ @Override
+ public Set<String> onlineTables() {
+ HashSet<String> onlineTables = new HashSet<String>(getConnector().tableOperations().tableIdMap().values());
+ return Sets.filter(onlineTables, new Predicate<String>() {
+ @Override
+ public boolean apply(String tableId) {
+ return Tables.getTableState(getConnector().getInstance(), tableId) == TableState.ONLINE;
+ }
+ });
+ }
+
+ @Override
+ public Collection<MergeInfo> merges() {
+ return Collections.emptySet();
+ }
+
+ @Override
- public Collection<KeyExtent> migrations() {
- return Collections.emptyList();
++ public Set<KeyExtent> migrationsSnapshot() {
++ return Collections.emptySet();
+ }
+
+ @Override
+ public Set<TServerInstance> shutdownServers() {
+ return Collections.emptySet();
+ }
+
+ @Override
+ public MasterState getMasterState() {
+ return MasterState.NORMAL;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/a6e6e672/test/src/main/java/org/apache/accumulo/test/master/MergeStateIT.java
----------------------------------------------------------------------
diff --cc test/src/main/java/org/apache/accumulo/test/master/MergeStateIT.java
index fb702a2,0000000..5ead27d
mode 100644,000000..100644
--- a/test/src/main/java/org/apache/accumulo/test/master/MergeStateIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/master/MergeStateIT.java
@@@ -1,204 -1,0 +1,204 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.test.master;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.BatchDeleter;
+import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.BatchWriterConfig;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.MasterState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ChoppedColumnFamily;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.master.state.MergeStats;
+import org.apache.accumulo.server.AccumuloServerContext;
+import org.apache.accumulo.server.master.state.Assignment;
+import org.apache.accumulo.server.master.state.CurrentState;
+import org.apache.accumulo.server.master.state.MergeInfo;
+import org.apache.accumulo.server.master.state.MergeState;
+import org.apache.accumulo.server.master.state.MetaDataStateStore;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletLocationState;
+import org.apache.accumulo.test.functional.ConfigurableMacBase;
+import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.net.HostAndPort;
+
+public class MergeStateIT extends ConfigurableMacBase {
+
+ private static class MockCurrentState implements CurrentState {
+
+ TServerInstance someTServer = new TServerInstance(HostAndPort.fromParts("127.0.0.1", 1234), 0x123456);
+ MergeInfo mergeInfo;
+
+ MockCurrentState(MergeInfo info) {
+ this.mergeInfo = info;
+ }
+
+ @Override
+ public Set<String> onlineTables() {
+ return Collections.singleton("t");
+ }
+
+ @Override
+ public Set<TServerInstance> onlineTabletServers() {
+ return Collections.singleton(someTServer);
+ }
+
+ @Override
+ public Collection<MergeInfo> merges() {
+ return Collections.singleton(mergeInfo);
+ }
+
+ @Override
- public Collection<KeyExtent> migrations() {
- return Collections.emptyList();
++ public Set<KeyExtent> migrationsSnapshot() {
++ return Collections.emptySet();
+ }
+
+ @Override
+ public MasterState getMasterState() {
+ return MasterState.NORMAL;
+ }
+
+ @Override
+ public Set<TServerInstance> shutdownServers() {
+ return Collections.emptySet();
+ }
+ }
+
+ private static void update(Connector c, Mutation m) throws TableNotFoundException, MutationsRejectedException {
+ BatchWriter bw = c.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+ bw.addMutation(m);
+ bw.close();
+ }
+
+ @Test
+ public void test() throws Exception {
+ AccumuloServerContext context = EasyMock.createMock(AccumuloServerContext.class);
+ Connector connector = getConnector();
+ EasyMock.expect(context.getConnector()).andReturn(connector).anyTimes();
+ EasyMock.replay(context);
+ connector.securityOperations().grantTablePermission(connector.whoami(), MetadataTable.NAME, TablePermission.WRITE);
+ BatchWriter bw = connector.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
+
+ // Create a fake METADATA table with these splits
+ String splits[] = {"a", "e", "j", "o", "t", "z"};
+ // create metadata for a table "t" with the splits above
+ Text tableId = new Text("t");
+ Text pr = null;
+ for (String s : splits) {
+ Text split = new Text(s);
+ Mutation prevRow = KeyExtent.getPrevRowUpdateMutation(new KeyExtent(tableId, split, pr));
+ prevRow.put(TabletsSection.CurrentLocationColumnFamily.NAME, new Text("123456"), new Value("127.0.0.1:1234".getBytes()));
+ ChoppedColumnFamily.CHOPPED_COLUMN.put(prevRow, new Value("junk".getBytes()));
+ bw.addMutation(prevRow);
+ pr = split;
+ }
+ // Add the default tablet
+ Mutation defaultTablet = KeyExtent.getPrevRowUpdateMutation(new KeyExtent(tableId, null, pr));
+ defaultTablet.put(TabletsSection.CurrentLocationColumnFamily.NAME, new Text("123456"), new Value("127.0.0.1:1234".getBytes()));
+ bw.addMutation(defaultTablet);
+ bw.close();
+
+ // Read out the TabletLocationStates
+ MockCurrentState state = new MockCurrentState(new MergeInfo(new KeyExtent(tableId, new Text("p"), new Text("e")), MergeInfo.Operation.MERGE));
+
+ // Verify the tablet state: hosted, and count
+ MetaDataStateStore metaDataStateStore = new MetaDataStateStore(context, state);
+ int count = 0;
+ for (TabletLocationState tss : metaDataStateStore) {
+ if (tss != null)
+ count++;
+ }
+ Assert.assertEquals(0, count); // the normal case is to skip tablets in a good state
+
+ // Create the hole
+ // Split the tablet at one end of the range
+ Mutation m = new KeyExtent(tableId, new Text("t"), new Text("p")).getPrevRowUpdateMutation();
+ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
+ TabletsSection.TabletColumnFamily.OLD_PREV_ROW_COLUMN.put(m, KeyExtent.encodePrevEndRow(new Text("o")));
+ update(connector, m);
+
+ // do the state check
+ MergeStats stats = scan(state, metaDataStateStore);
+ MergeState newState = stats.nextMergeState(connector, state);
+ Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, newState);
+
+ // unassign the tablets
+ BatchDeleter deleter = connector.createBatchDeleter(MetadataTable.NAME, Authorizations.EMPTY, 1000, new BatchWriterConfig());
+ deleter.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ deleter.setRanges(Collections.singletonList(new Range()));
+ deleter.delete();
+
+ // now we should be ready to merge but, we have inconsistent metadata
+ stats = scan(state, metaDataStateStore);
+ Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
+
+ // finish the split
+ KeyExtent tablet = new KeyExtent(tableId, new Text("p"), new Text("o"));
+ m = tablet.getPrevRowUpdateMutation();
+ TabletsSection.TabletColumnFamily.SPLIT_RATIO_COLUMN.put(m, new Value("0.5".getBytes()));
+ update(connector, m);
+ metaDataStateStore.setLocations(Collections.singletonList(new Assignment(tablet, state.someTServer)));
+
+ // onos... there's a new tablet online
+ stats = scan(state, metaDataStateStore);
+ Assert.assertEquals(MergeState.WAITING_FOR_CHOPPED, stats.nextMergeState(connector, state));
+
+ // chop it
+ m = tablet.getPrevRowUpdateMutation();
+ ChoppedColumnFamily.CHOPPED_COLUMN.put(m, new Value("junk".getBytes()));
+ update(connector, m);
+
+ stats = scan(state, metaDataStateStore);
+ Assert.assertEquals(MergeState.WAITING_FOR_OFFLINE, stats.nextMergeState(connector, state));
+
+ // take it offline
+ m = tablet.getPrevRowUpdateMutation();
+ Collection<Collection<String>> walogs = Collections.emptyList();
+ metaDataStateStore.unassign(Collections.singletonList(new TabletLocationState(tablet, null, state.someTServer, null, walogs, false)), null);
+
+ // now we can split
+ stats = scan(state, metaDataStateStore);
+ Assert.assertEquals(MergeState.MERGING, stats.nextMergeState(connector, state));
+
+ }
+
+ private MergeStats scan(MockCurrentState state, MetaDataStateStore metaDataStateStore) {
+ MergeStats stats = new MergeStats(state.mergeInfo);
+ stats.getMergeInfo().setState(MergeState.WAITING_FOR_OFFLINE);
+ for (TabletLocationState tss : metaDataStateStore) {
+ stats.update(tss.extent, tss.getState(state.onlineTabletServers()), tss.chopped, false);
+ }
+ return stats;
+ }
+}