You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ec...@apache.org on 2013/12/19 21:33:33 UTC
[08/13] git commit: ACCUMULO-2057 handle duplicate last locations
gracefully
ACCUMULO-2057 handle duplicate last locations gracefully
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/76842b29
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/76842b29
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/76842b29
Branch: refs/heads/1.5.1-SNAPSHOT
Commit: 76842b29458571fdd66e5b3440f7fd0075d10542
Parents: a480f63 2235b50
Author: Eric Newton <er...@gmail.com>
Authored: Thu Dec 19 15:31:59 2013 -0500
Committer: Eric Newton <er...@gmail.com>
Committed: Thu Dec 19 15:31:59 2013 -0500
----------------------------------------------------------------------
.../accumulo/server/master/state/MetaDataTableScanner.java | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/76842b29/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
----------------------------------------------------------------------
diff --cc server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
index bf1da22,0000000..2458a07
mode 100644,000000..100644
--- a/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
+++ b/server/src/main/java/org/apache/accumulo/server/master/state/MetaDataTableScanner.java
@@@ -1,185 -1,0 +1,183 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.state;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.BatchScanner;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.client.ScannerBase;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.server.master.state.TabletLocationState.BadLocationStateException;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Logger;
+
+public class MetaDataTableScanner implements Iterator<TabletLocationState> {
+ private static final Logger log = Logger.getLogger(MetaDataTableScanner.class);
+
+ BatchScanner mdScanner;
+ Iterator<Entry<Key,Value>> iter;
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range, CurrentState state) {
+ // scan over metadata table, looking for tablets in the wrong state based on the live servers and online tables
+ try {
+ Connector connector = instance.getConnector(auths.getPrincipal(), CredentialHelper.extractToken(auths));
+ mdScanner = connector.createBatchScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS, 8);
+ configureScanner(mdScanner, state);
+ mdScanner.setRanges(Collections.singletonList(range));
+ iter = mdScanner.iterator();
+ } catch (Exception ex) {
+ mdScanner.close();
+ throw new RuntimeException(ex);
+ }
+ }
+
+ static public void configureScanner(ScannerBase scanner, CurrentState state) {
+ Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);
+ scanner.fetchColumnFamily(Constants.METADATA_CHOPPED_COLUMN_FAMILY);
+ scanner.addScanIterator(new IteratorSetting(1000, "wholeRows", WholeRowIterator.class));
+ IteratorSetting tabletChange = new IteratorSetting(1001, "tabletChange", TabletStateChangeIterator.class);
+ if (state != null) {
+ TabletStateChangeIterator.setCurrentServers(tabletChange, state.onlineTabletServers());
+ TabletStateChangeIterator.setOnlineTables(tabletChange, state.onlineTables());
+ TabletStateChangeIterator.setMerges(tabletChange, state.merges());
+ }
+ scanner.addScanIterator(tabletChange);
+ }
+
+ public MetaDataTableScanner(Instance instance, TCredentials auths, Range range) {
+ this(instance, auths, range, null);
+ }
+
+ public void close() {
+ if (iter != null) {
+ mdScanner.close();
+ iter = null;
+ }
+ }
+
+ public void finalize() {
+ close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (iter == null)
+ return false;
+ boolean result = iter.hasNext();
+ if (!result) {
+ close();
+ }
+ return result;
+ }
+
+ @Override
+ public TabletLocationState next() {
+ try {
+ return fetch();
+ } catch (RuntimeException ex) {
+ // something is wrong with the records in the !METADATA table, just skip over it
+ log.error(ex, ex);
+ mdScanner.close();
+ return null;
+ }
+ }
+
+ public static TabletLocationState createTabletLocationState(Key k, Value v) throws IOException, BadLocationStateException {
+ final SortedMap<Key,Value> decodedRow = WholeRowIterator.decodeRow(k, v);
+ KeyExtent extent = null;
+ TServerInstance future = null;
+ TServerInstance current = null;
+ TServerInstance last = null;
++ long lastTimestamp = 0;
+ List<Collection<String>> walogs = new ArrayList<Collection<String>>();
+ boolean chopped = false;
+
+ for (Entry<Key,Value> entry : decodedRow.entrySet()) {
+ Key key = entry.getKey();
+ Text row = key.getRow();
+ Text cf = key.getColumnFamily();
+ Text cq = key.getColumnQualifier();
+
+ if (cf.compareTo(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (future != null) {
+ throw new BadLocationStateException("found two assignments for the same extent " + key.getRow() + ": " + future + " and " + location);
+ }
+ future = location;
+ } else if (cf.compareTo(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) == 0) {
+ TServerInstance location = new TServerInstance(entry.getValue(), cq);
+ if (current != null) {
+ throw new BadLocationStateException("found two locations for the same extent " + key.getRow() + ": " + current + " and " + location);
+ }
+ current = location;
+ } else if (cf.compareTo(Constants.METADATA_LOG_COLUMN_FAMILY) == 0) {
+ String[] split = entry.getValue().toString().split("\\|")[0].split(";");
+ walogs.add(Arrays.asList(split));
+ } else if (cf.compareTo(Constants.METADATA_LAST_LOCATION_COLUMN_FAMILY) == 0) {
- TServerInstance location = new TServerInstance(entry.getValue(), cq);
- if (last != null) {
- throw new BadLocationStateException("found two last locations for the same extent " + key.getRow() + ": " + last + " and " + location);
- }
- last = new TServerInstance(entry.getValue(), cq);
++ if (lastTimestamp < entry.getKey().getTimestamp())
++ last = new TServerInstance(entry.getValue(), cq);
+ } else if (cf.compareTo(Constants.METADATA_CHOPPED_COLUMN_FAMILY) == 0) {
+ chopped = true;
+ } else if (Constants.METADATA_PREV_ROW_COLUMN.equals(cf, cq)) {
+ extent = new KeyExtent(row, entry.getValue());
+ }
+ }
+ if (extent == null) {
+ log.warn("No prev-row for key extent: " + decodedRow);
+ return null;
+ }
+ return new TabletLocationState(extent, future, current, last, walogs, chopped);
+ }
+
+ private TabletLocationState fetch() {
+ try {
+ Entry<Key,Value> e = iter.next();
+ return createTabletLocationState(e.getKey(), e.getValue());
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ } catch (BadLocationStateException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Unimplemented");
+ }
+}