You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by ct...@apache.org on 2013/07/17 04:33:17 UTC
[15/50] [abbrv] git commit: ACCUMULO-1481 : Add tests for
splitting/merging root table;
refactor to consolidate metadata constants and structures in an organized way;
begin consolidating metadata ops into a servicer interface to abstract the
code that ac
ACCUMULO-1481 : Add tests for splitting/merging root table; refactor to consolidate metadata constants and structures in an organized way; begin consolidating metadata ops into a servicer interface to abstract the code that actually does the servicing of metadata for all tables; remove some special cases for checking for metadata tables/root tablet that no longer apply; make mock work with root table; fix SimpleGarbageCollector to confirm candidates for deletion by checking the root table as well as the regular metadata table
git-svn-id: https://svn.apache.org/repos/asf/accumulo/trunk@1499510 13f79535-47bb-0310-9956-ffa450edef68
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/446a37a9
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/446a37a9
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/446a37a9
Branch: refs/heads/ACCUMULO-1496
Commit: 446a37a9795f2df7adc841154ca05add79cf286e
Parents: a5872e6
Author: Christopher Tubbs <ct...@apache.org>
Authored: Wed Jul 3 18:32:51 2013 +0000
Committer: Christopher Tubbs <ct...@apache.org>
Committed: Wed Jul 3 18:32:51 2013 +0000
----------------------------------------------------------------------
.../accumulo/core/client/ZooKeeperInstance.java | 2 +-
.../core/client/admin/TableOperationsImpl.java | 33 +-
.../client/impl/MetadataLocationObtainer.java | 206 ---
.../core/client/impl/OfflineScanner.java | 14 +-
.../core/client/impl/RootTabletLocator.java | 2 +-
.../core/client/impl/ScannerOptions.java | 8 +-
.../core/client/impl/TabletLocator.java | 11 +-
.../core/client/impl/TabletLocatorImpl.java | 2 +-
.../impl/TabletServerBatchReaderIterator.java | 4 +-
.../client/impl/TabletServerBatchWriter.java | 10 +-
.../core/client/impl/ThriftScanner.java | 18 +-
.../accumulo/core/client/impl/Writer.java | 6 +-
.../core/client/mapred/InputFormatBase.java | 19 +-
.../core/client/mapreduce/InputFormatBase.java | 19 +-
.../mapreduce/lib/util/InputConfigurator.java | 2 +-
.../accumulo/core/client/mock/MockAccumulo.java | 4 +-
.../core/client/mock/MockTableOperations.java | 19 +-
.../apache/accumulo/core/data/KeyExtent.java | 22 +-
.../core/metadata/MetadataLocationObtainer.java | 269 ++++
.../core/metadata/MetadataServicer.java | 67 +
.../accumulo/core/metadata/MetadataTable.java | 25 +
.../accumulo/core/metadata/RootTable.java | 46 +
.../core/metadata/ServicerForMetadataTable.java | 32 +
.../core/metadata/ServicerForRootTable.java | 49 +
.../core/metadata/ServicerForUserTables.java | 32 +
.../core/metadata/TableMetadataServicer.java | 143 ++
.../core/metadata/schema/DataFileValue.java | 96 ++
.../core/metadata/schema/MetadataSchema.java | 236 ++++
.../org/apache/accumulo/core/util/ColumnFQ.java | 24 -
.../org/apache/accumulo/core/util/Merge.java | 11 +-
.../accumulo/core/util/MetadataTable.java | 400 ------
.../apache/accumulo/core/util/RootTable.java | 42 -
.../core/util/shell/commands/FlushCommand.java | 2 +-
.../util/shell/commands/GetSplitsCommand.java | 9 +-
.../util/shell/commands/OfflineCommand.java | 2 +-
.../core/util/shell/commands/OnlineCommand.java | 2 +-
.../core/client/impl/TabletLocatorImplTest.java | 25 +-
.../core/metadata/MetadataServicerTest.java | 86 ++
.../accumulo/core/util/MetadataTableTest.java | 109 --
.../org/apache/accumulo/proxy/SimpleTest.java | 11 +-
.../apache/accumulo/server/ServerConstants.java | 14 +-
.../accumulo/server/client/BulkImporter.java | 4 +-
.../accumulo/server/client/HdfsZooInstance.java | 2 +-
.../server/constraints/MetadataConstraints.java | 69 +-
.../server/gc/GarbageCollectWriteAheadLogs.java | 6 +-
.../server/gc/SimpleGarbageCollector.java | 87 +-
.../iterators/MetadataBulkLoadFilter.java | 10 +-
.../apache/accumulo/server/master/Master.java | 35 +-
.../server/master/TabletGroupWatcher.java | 68 +-
.../master/balancer/ChaoticLoadBalancer.java | 2 +-
.../server/master/state/MergeStats.java | 6 +-
.../server/master/state/MetaDataStateStore.java | 17 +-
.../master/state/MetaDataTableScanner.java | 27 +-
.../master/state/RootTabletStateStore.java | 5 +-
.../server/master/state/TServerInstance.java | 10 +-
.../master/state/ZooTabletStateStore.java | 6 +-
.../server/master/tableOps/BulkImport.java | 17 +-
.../server/master/tableOps/CloneTable.java | 8 +-
.../server/master/tableOps/CompactRange.java | 15 +-
.../server/master/tableOps/CreateTable.java | 6 +-
.../server/master/tableOps/DeleteTable.java | 11 +-
.../server/master/tableOps/ExportTable.java | 23 +-
.../server/master/tableOps/ImportTable.java | 17 +-
.../server/master/tableOps/TableRangeOp.java | 5 +
.../accumulo/server/metanalysis/FilterMeta.java | 2 +-
.../accumulo/server/metanalysis/IndexMeta.java | 5 +-
.../server/metanalysis/PrintEvents.java | 16 +-
.../server/monitor/servlets/TablesServlet.java | 4 +-
.../accumulo/server/problems/ProblemReport.java | 6 +-
.../server/problems/ProblemReports.java | 7 +-
.../security/AuditedSecurityOperation.java | 2 +-
.../server/security/SecurityOperation.java | 4 +-
.../server/security/handler/ZKAuthorizor.java | 2 +-
.../server/security/handler/ZKPermHandler.java | 4 +-
.../accumulo/server/tabletserver/Compactor.java | 2 +-
.../server/tabletserver/FileManager.java | 2 +-
.../server/tabletserver/MinorCompactor.java | 2 +-
.../accumulo/server/tabletserver/Tablet.java | 83 +-
.../tabletserver/TabletIteratorEnvironment.java | 2 +-
.../server/tabletserver/TabletServer.java | 53 +-
.../TabletServerResourceManager.java | 4 +-
.../server/util/AddFilesWithMissingEntries.java | 22 +-
.../org/apache/accumulo/server/util/Admin.java | 2 +-
.../server/util/CheckForMetadataProblems.java | 17 +-
.../server/util/FindOfflineTablets.java | 13 +-
.../apache/accumulo/server/util/Initialize.java | 43 +-
.../accumulo/server/util/LocalityCheck.java | 15 +-
.../accumulo/server/util/MetadataTable.java | 1275 -----------------
.../accumulo/server/util/MetadataTableUtil.java | 1313 ++++++++++++++++++
.../server/util/OfflineMetadataScanner.java | 17 +-
.../util/RemoveEntriesForMissingFiles.java | 12 +-
.../accumulo/server/util/TableDiskUsage.java | 15 +-
.../accumulo/server/util/TabletIterator.java | 15 +-
.../server/util/VerifyTabletAssignments.java | 42 +-
.../constraints/MetadataConstraintsTest.java | 88 +-
.../accumulo/server/gc/TestConfirmDeletes.java | 2 +-
.../iterators/MetadataBulkLoadFilterTest.java | 41 +-
.../accumulo/server/master/TestMergeState.java | 20 +-
.../master/state/RootTabletStateStoreTest.java | 2 +-
.../tabletserver/CheckTabletMetadataTest.java | 45 +-
.../apache/accumulo/server/util/CloneTest.java | 79 +-
.../server/util/TabletIteratorTest.java | 11 +-
.../accumulo/test/QueryMetadataTable.java | 5 +-
.../accumulo/test/TestMultiTableIngest.java | 9 +-
.../continuous/ContinuousStatsCollector.java | 5 +-
.../test/functional/FunctionalTest.java | 10 +-
.../test/functional/SplitRecoveryTest.java | 49 +-
.../metadata/MetadataBatchScanTest.java | 12 +-
.../performance/scan/CollectTabletStats.java | 68 +-
.../test/randomwalk/concurrent/AddSplits.java | 2 +-
.../randomwalk/concurrent/CheckBalance.java | 5 +-
.../test/randomwalk/concurrent/Merge.java | 2 +-
.../org/apache/accumulo/test/MetaSplitTest.java | 23 +-
.../test/TestAccumuloSplitRecovery.java | 18 +-
.../org/apache/accumulo/test/VolumeTest.java | 4 +-
.../accumulo/test/functional/CompactionIT.java | 20 +-
.../test/functional/DynamicThreadPoolsIT.java | 4 +-
.../test/functional/FunctionalTestUtils.java | 12 +-
.../test/functional/GarbageCollectorIT.java | 24 +-
.../accumulo/test/functional/MergeMetaIT.java | 17 +-
.../accumulo/test/functional/PermissionsIT.java | 33 +-
.../accumulo/test/functional/ReadWriteIT.java | 68 +-
.../accumulo/test/functional/SplitIT.java | 31 +-
.../accumulo/test/functional/TableIT.java | 16 +-
.../accumulo/test/functional/TimeoutIT.java | 5 +-
125 files changed, 3395 insertions(+), 2925 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
index 087fc88..5b56adb 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ZooKeeperInstance.java
@@ -30,13 +30,13 @@ import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.file.FileUtil;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.OpTimer;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.zookeeper.ZooUtil;
import org.apache.accumulo.fate.zookeeper.ZooCache;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
index ece5eff..3ad4217 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperationsImpl.java
@@ -81,6 +81,9 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
import org.apache.accumulo.core.master.thrift.MasterClientService;
import org.apache.accumulo.core.master.thrift.TableOperation;
+import org.apache.accumulo.core.metadata.MetadataServicer;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
@@ -90,11 +93,9 @@ import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.LocalityGroupUtil;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.NamingThreadFactory;
import org.apache.accumulo.core.util.OpTimer;
import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.StringUtil;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.ThriftUtil;
@@ -436,7 +437,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
private void addSplits(String tableName, SortedSet<Text> partitionKeys, String tableId) throws AccumuloException, AccumuloSecurityException,
TableNotFoundException, AccumuloServerException {
- TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(tableId));
+ TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(tableId));
for (Text split : partitionKeys) {
boolean successful = false;
@@ -543,38 +544,34 @@ public class TableOperationsImpl extends TableOperationsHelper {
ArgumentChecker.notNull(tableName);
String tableId = Tables.getTableId(instance, tableName);
- if (RootTable.ID.equals(tableId))
- return Collections.emptyList();
- SortedSet<KeyExtent> tablets = new TreeSet<KeyExtent>();
- Map<KeyExtent,String> locations = new TreeMap<KeyExtent,String>();
+ TreeMap<KeyExtent,String> tabletLocations = new TreeMap<KeyExtent,String>();
while (true) {
try {
- tablets.clear();
- locations.clear();
+ tabletLocations.clear();
// the following method throws AccumuloException for some conditions that should be retried
- MetadataTable.getEntries(instance, credentials, tableId, true, locations, tablets);
+ MetadataServicer.forTableId(instance, credentials, tableId).getTabletLocations(tabletLocations);
break;
} catch (AccumuloSecurityException ase) {
throw ase;
- } catch (Throwable t) {
+ } catch (Exception e) {
if (!Tables.exists(instance, tableId)) {
throw new TableNotFoundException(tableId, tableName, null);
}
- if (t instanceof RuntimeException && t.getCause() instanceof AccumuloSecurityException) {
- throw (AccumuloSecurityException) t.getCause();
+ if (e instanceof RuntimeException && e.getCause() instanceof AccumuloSecurityException) {
+ throw (AccumuloSecurityException) e.getCause();
}
- log.info(t.getMessage() + " ... retrying ...");
+ log.info(e.getMessage() + " ... retrying ...");
UtilWaitThread.sleep(3000);
}
}
- ArrayList<Text> endRows = new ArrayList<Text>(tablets.size());
+ ArrayList<Text> endRows = new ArrayList<Text>(tabletLocations.size());
- for (KeyExtent ke : tablets)
+ for (KeyExtent ke : tabletLocations.keySet())
if (ke.getEndRow() != null)
endRows.add(ke.getEndRow());
@@ -1039,7 +1036,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
Map<String,Map<KeyExtent,List<Range>>> binnedRanges = new HashMap<String,Map<KeyExtent,List<Range>>>();
String tableId = Tables.getTableId(instance, tableName);
- TabletLocator tl = TabletLocator.getInstance(instance, new Text(tableId));
+ TabletLocator tl = TabletLocator.getLocator(instance, new Text(tableId));
// its possible that the cache could contain complete, but old information about a tables tablets... so clear it
tl.invalidateCache();
while (!tl.binRanges(Collections.singletonList(range), binnedRanges, credentials).isEmpty()) {
@@ -1181,7 +1178,7 @@ public class TableOperationsImpl extends TableOperationsHelper {
@Override
public void clearLocatorCache(String tableName) throws TableNotFoundException {
ArgumentChecker.notNull(tableName);
- TabletLocator tabLocator = TabletLocator.getInstance(instance, new Text(Tables.getTableId(instance, tableName)));
+ TabletLocator tabLocator = TabletLocator.getLocator(instance, new Text(Tables.getTableId(instance, tableName)));
tabLocator.invalidateCache();
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java b/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
deleted file mode 100644
index 58ec913..0000000
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
+++ /dev/null
@@ -1,206 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.accumulo.core.client.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.SortedMap;
-import java.util.SortedSet;
-import java.util.TreeMap;
-import java.util.TreeSet;
-
-import org.apache.accumulo.core.Constants;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
-import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocations;
-import org.apache.accumulo.core.client.impl.TabletLocatorImpl.TabletLocationObtainer;
-import org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.ResultReceiver;
-import org.apache.accumulo.core.data.Column;
-import org.apache.accumulo.core.data.Key;
-import org.apache.accumulo.core.data.KeyExtent;
-import org.apache.accumulo.core.data.PartialKey;
-import org.apache.accumulo.core.data.Range;
-import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.core.data.thrift.IterInfo;
-import org.apache.accumulo.core.iterators.user.WholeRowIterator;
-import org.apache.accumulo.core.security.Authorizations;
-import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.OpTimer;
-import org.apache.accumulo.core.util.Pair;
-import org.apache.accumulo.core.util.TextUtil;
-import org.apache.hadoop.io.Text;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-public class MetadataLocationObtainer implements TabletLocationObtainer {
- private static final Logger log = Logger.getLogger(MetadataLocationObtainer.class);
- private SortedSet<Column> locCols;
- private ArrayList<Column> columns;
- private Instance instance;
-
- MetadataLocationObtainer(Instance instance) {
-
- this.instance = instance;
-
- locCols = new TreeSet<Column>();
- locCols.add(new Column(TextUtil.getBytes(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY), null, null));
- locCols.add(MetadataTable.PREV_ROW_COLUMN.toColumn());
- columns = new ArrayList<Column>(locCols);
- }
-
- @Override
- public TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent, TCredentials credentials)
- throws AccumuloSecurityException, AccumuloException {
-
- try {
- ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
-
- OpTimer opTimer = null;
- if (log.isTraceEnabled())
- opTimer = new OpTimer(log, Level.TRACE).start("Looking up in " + src.tablet_extent.getTableId() + " row=" + TextUtil.truncate(row) + " extent="
- + src.tablet_extent + " tserver=" + src.tablet_location);
-
- Range range = new Range(row, true, stopRow, true);
-
- TreeMap<Key,Value> encodedResults = new TreeMap<Key,Value>();
- TreeMap<Key,Value> results = new TreeMap<Key,Value>();
-
- // Use the whole row iterator so that a partial mutations is not read. The code that extracts locations for tablets does a sanity check to ensure there is
- // only one location. Reading a partial mutation could make it appear there are multiple locations when there are not.
- List<IterInfo> serverSideIteratorList = new ArrayList<IterInfo>();
- serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI"));
- Map<String,Map<String,String>> serverSideIteratorOptions = Collections.emptyMap();
-
- boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols,
- serverSideIteratorList, serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Authorizations.EMPTY, false, instance.getConfiguration());
-
- decodeRows(encodedResults, results);
-
- if (more && results.size() == 1) {
- range = new Range(results.lastKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), true, new Key(stopRow).followingKey(PartialKey.ROW), false);
- encodedResults.clear();
- more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols, serverSideIteratorList,
- serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Authorizations.EMPTY, false, instance.getConfiguration());
-
- decodeRows(encodedResults, results);
- }
-
- if (opTimer != null)
- opTimer.stop("Got " + results.size() + " results from " + src.tablet_extent + " in %DURATION%");
-
- // System.out.println("results "+results.keySet());
-
- Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> metadata = MetadataTable.getMetadataLocationEntries(results);
-
- for (Entry<KeyExtent,Text> entry : metadata.getFirst().entrySet()) {
- list.add(new TabletLocation(entry.getKey(), entry.getValue().toString()));
- }
-
- return new TabletLocations(list, metadata.getSecond());
-
- } catch (AccumuloServerException ase) {
- if (log.isTraceEnabled())
- log.trace(src.tablet_extent.getTableId() + " lookup failed, " + src.tablet_location + " server side exception");
- throw ase;
- } catch (NotServingTabletException e) {
- if (log.isTraceEnabled())
- log.trace(src.tablet_extent.getTableId() + " lookup failed, " + src.tablet_location + " not serving " + src.tablet_extent);
- parent.invalidateCache(src.tablet_extent);
- } catch (AccumuloException e) {
- if (log.isTraceEnabled())
- log.trace(src.tablet_extent.getTableId() + " lookup failed", e);
- parent.invalidateCache(src.tablet_location);
- }
-
- return null;
- }
-
- private void decodeRows(TreeMap<Key,Value> encodedResults, TreeMap<Key,Value> results) throws AccumuloException {
- for (Entry<Key,Value> entry : encodedResults.entrySet()) {
- try {
- results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
- } catch (IOException e) {
- throw new AccumuloException(e);
- }
- }
- }
-
- @Override
- public List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>> tabletsRanges, TabletLocator parent, TCredentials credentials)
- throws AccumuloSecurityException, AccumuloException {
-
- final TreeMap<Key,Value> results = new TreeMap<Key,Value>();
-
- ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
-
- ResultReceiver rr = new ResultReceiver() {
-
- @Override
- public void receive(List<Entry<Key,Value>> entries) {
- for (Entry<Key,Value> entry : entries) {
- try {
- results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- };
-
- ScannerOptions opts = new ScannerOptions();
- opts.fetchedColumns = locCols;
- opts.serverSideIteratorList = new ArrayList<IterInfo>();
- opts.serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI")); // see comment in lookupTablet about why iterator is
- // used
-
- Map<KeyExtent,List<Range>> unscanned = new HashMap<KeyExtent,List<Range>>();
- Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
- try {
- TabletServerBatchReaderIterator.doLookup(tserver, tabletsRanges, failures, unscanned, rr, columns, credentials, opts, Authorizations.EMPTY,
- instance.getConfiguration());
- if (failures.size() > 0) {
- // invalidate extents in parents cache
- if (log.isTraceEnabled())
- log.trace("lookupTablets failed for " + failures.size() + " extents");
- parent.invalidateCache(failures.keySet());
- }
- } catch (IOException e) {
- log.trace("lookupTablets failed server=" + tserver, e);
- parent.invalidateCache(tserver);
- } catch (AccumuloServerException e) {
- log.trace("lookupTablets failed server=" + tserver, e);
- throw e;
- }
-
- SortedMap<KeyExtent,Text> metadata = MetadataTable.getMetadataLocationEntries(results).getFirst();
-
- for (Entry<KeyExtent,Text> entry : metadata.entrySet()) {
- list.add(new TabletLocation(entry.getKey(), entry.getValue().toString()));
- }
-
- return list;
- }
-}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
index 9c53c1d..3dc19cd 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/OfflineScanner.java
@@ -53,6 +53,9 @@ import org.apache.accumulo.core.iterators.system.DeletingIterator;
import org.apache.accumulo.core.iterators.system.MultiIterator;
import org.apache.accumulo.core.iterators.system.VisibilityFilter;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.core.security.CredentialHelper;
@@ -60,7 +63,6 @@ import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.util.ArgumentChecker;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.LocalityGroupUtil;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.commons.lang.NotImplementedException;
@@ -230,7 +232,7 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
if (currentExtent != null && !extent.isPreviousExtent(currentExtent))
throw new AccumuloException(" " + currentExtent + " is not previous extent " + extent);
-
+
String tablesDir = instance.getConfiguration().get(Property.INSTANCE_DFS_DIR) + "/tables";
String[] volumes = instance.getConfiguration().get(Property.INSTANCE_VOLUMES).split(",");
if (volumes.length > 1) {
@@ -271,16 +273,16 @@ class OfflineIterator implements Iterator<Entry<Key,Value>> {
Entry<Key,Value> entry = row.next();
Key key = entry.getKey();
- if (key.getColumnFamily().equals(MetadataTable.DATAFILE_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
relFiles.add(key.getColumnQualifier().toString());
}
- if (key.getColumnFamily().equals(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY)
- || key.getColumnFamily().equals(MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME)) {
location = entry.getValue().toString();
}
- if (MetadataTable.PREV_ROW_COLUMN.hasColumns(key)) {
+ if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
extent = new KeyExtent(key.getRow(), entry.getValue());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/RootTabletLocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/RootTabletLocator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/RootTabletLocator.java
index 72101cc..18b2a27 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/RootTabletLocator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/RootTabletLocator.java
@@ -28,8 +28,8 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
index 179ed08..7641755 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ScannerOptions.java
@@ -46,7 +46,7 @@ public class ScannerOptions implements ScannerBase {
protected SortedSet<Column> fetchedColumns = new TreeSet<Column>();
protected long timeOut = Long.MAX_VALUE;
-
+
private String regexIterName = null;
protected ScannerOptions() {}
@@ -102,7 +102,7 @@ public class ScannerOptions implements ScannerBase {
serverSideIteratorOptions.remove(iteratorName);
}
-
+
/**
* Override any existing options on the given named iterator
*/
@@ -190,7 +190,7 @@ public class ScannerOptions implements ScannerBase {
if (timeOut < 0) {
throw new IllegalArgumentException("TimeOut must be positive : " + timeOut);
}
-
+
if (timeout == 0)
this.timeOut = Long.MAX_VALUE;
else
@@ -201,7 +201,7 @@ public class ScannerOptions implements ScannerBase {
public long getTimeout(TimeUnit timeunit) {
return timeunit.convert(timeOut, TimeUnit.MILLISECONDS);
}
-
+
@Override
public void close() {
// Nothing needs to be closed
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
index 0cc012e..de8e053 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocator.java
@@ -31,10 +31,11 @@ import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.metadata.MetadataLocationObtainer;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.util.ArgumentChecker;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.hadoop.io.Text;
public abstract class TabletLocator {
@@ -91,7 +92,7 @@ public abstract class TabletLocator {
private static HashMap<LocatorKey,TabletLocator> locators = new HashMap<LocatorKey,TabletLocator>();
- public static synchronized TabletLocator getInstance(Instance instance, Text tableId) {
+ public static synchronized TabletLocator getLocator(Instance instance, Text tableId) {
LocatorKey key = new LocatorKey(instance.getInstanceID(), tableId);
TabletLocator tl = locators.get(key);
@@ -101,9 +102,9 @@ public abstract class TabletLocator {
if (tableId.toString().equals(RootTable.ID)) {
tl = new RootTabletLocator(instance);
} else if (tableId.toString().equals(MetadataTable.ID)) {
- tl = new TabletLocatorImpl(new Text(MetadataTable.ID), getInstance(instance, new Text(RootTable.ID)), mlo);
+ tl = new TabletLocatorImpl(new Text(MetadataTable.ID), getLocator(instance, new Text(RootTable.ID)), mlo);
} else {
- tl = new TabletLocatorImpl(tableId, getInstance(instance, new Text(MetadataTable.ID)), mlo);
+ tl = new TabletLocatorImpl(tableId, getLocator(instance, new Text(MetadataTable.ID)), mlo);
}
locators.put(key, tl);
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
index a1e2e8a..df5d66b 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
@@ -399,7 +399,7 @@ public class TabletLocatorImpl extends TabletLocator {
if (ptl != null) {
TabletLocations locations = locationObtainer.lookupTablet(ptl, metadataRow, lastTabletRow, parent, credentials);
- while (locations != null && locations.getLocations().isEmpty() && locations.getLocationless().isEmpty() && !ptl.tablet_extent.isRootTablet()) {
+ while (locations != null && locations.getLocations().isEmpty() && locations.getLocationless().isEmpty()) {
// try the next tablet, the current tablet does not have any tablets that overlap the row
Text er = ptl.tablet_extent.getEndRow();
if (er != null && er.compareTo(lastTabletRow) < 0) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
index 883d1a3..a8ce55e 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchReaderIterator.java
@@ -144,7 +144,7 @@ public class TabletServerBatchReaderIterator implements Iterator<Entry<Key,Value
this.options = new ScannerOptions(scannerOptions);
resultsQueue = new ArrayBlockingQueue<List<Entry<Key,Value>>>(numThreads);
- this.locator = new TimeoutTabletLocator(TabletLocator.getInstance(instance, new Text(table)), timeout);
+ this.locator = new TimeoutTabletLocator(TabletLocator.getLocator(instance, new Text(table)), timeout);
timeoutTrackers = Collections.synchronizedMap(new HashMap<String,TabletServerBatchReaderIterator.TimeoutTracker>());
timedoutServers = Collections.synchronizedSet(new HashSet<String>());
@@ -604,7 +604,7 @@ public class TabletServerBatchReaderIterator implements Iterator<Entry<Key,Value
}
}
- static void doLookup(String server, Map<KeyExtent,List<Range>> requested, Map<KeyExtent,List<Range>> failures, Map<KeyExtent,List<Range>> unscanned,
+ public static void doLookup(String server, Map<KeyExtent,List<Range>> requested, Map<KeyExtent,List<Range>> failures, Map<KeyExtent,List<Range>> unscanned,
ResultReceiver receiver, List<Column> columns, TCredentials credentials, ScannerOptions options, Authorizations authorizations, AccumuloConfiguration conf)
throws IOException, AccumuloSecurityException, AccumuloServerException {
doLookup(server, requested, failures, unscanned, receiver, columns, credentials, options, authorizations, conf, new TimeoutTracker(Long.MAX_VALUE));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchWriter.java b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchWriter.java
index 772e0aa..766cea9 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchWriter.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletServerBatchWriter.java
@@ -55,12 +55,12 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.thrift.TMutation;
import org.apache.accumulo.core.data.thrift.UpdateErrors;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.SimpleThreadPool;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.trace.instrument.Span;
@@ -631,7 +631,7 @@ public class TabletServerBatchWriter {
private TabletLocator getLocator(String tableId) {
TabletLocator ret = locators.get(tableId);
if (ret == null) {
- ret = TabletLocator.getInstance(instance, new Text(tableId));
+ ret = TabletLocator.getLocator(instance, new Text(tableId));
ret = new TimeoutTabletLocator(ret, timeout);
locators.put(tableId, ret);
}
@@ -831,7 +831,7 @@ public class TabletServerBatchWriter {
tables.add(ke.getTableId().toString());
for (String table : tables)
- TabletLocator.getInstance(instance, new Text(table)).invalidateCache(location);
+ TabletLocator.getLocator(instance, new Text(table)).invalidateCache(location);
failedMutations.add(location, tsm);
} finally {
@@ -868,7 +868,7 @@ public class TabletServerBatchWriter {
client.update(tinfo, credentials, entry.getKey().toThrift(), entry.getValue().get(0).toThrift());
} catch (NotServingTabletException e) {
allFailures.addAll(entry.getKey().getTableId().toString(), entry.getValue());
- TabletLocator.getInstance(instance, new Text(entry.getKey().getTableId())).invalidateCache(entry.getKey());
+ TabletLocator.getLocator(instance, new Text(entry.getKey().getTableId())).invalidateCache(entry.getKey());
} catch (ConstraintViolationException e) {
updatedConstraintViolations(Translator.translate(e.violationSummaries, Translator.TCVST));
}
@@ -909,7 +909,7 @@ public class TabletServerBatchWriter {
String table = failedExtent.getTableId().toString();
- TabletLocator.getInstance(instance, new Text(table)).invalidateCache(failedExtent);
+ TabletLocator.getLocator(instance, new Text(table)).invalidateCache(failedExtent);
ArrayList<Mutation> mutations = (ArrayList<Mutation>) tabMuts.get(failedExtent);
allFailures.addAll(table, mutations.subList(numCommitted, mutations.size()));
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java b/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
index 37e4579..584b56c 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
@@ -66,8 +66,6 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.apache.thrift.TApplicationException;
import org.apache.thrift.TException;
-import org.apache.thrift.TServiceClient;
-
public class ThriftScanner {
private static final Logger log = Logger.getLogger(ThriftScanner.class);
@@ -80,7 +78,7 @@ public class ThriftScanner {
}
}
- static boolean getBatchFromServer(TCredentials credentials, Range range, KeyExtent extent, String server, SortedMap<Key,Value> results,
+ public static boolean getBatchFromServer(TCredentials credentials, Range range, KeyExtent extent, String server, SortedMap<Key,Value> results,
SortedSet<Column> fetchedColumns, List<IterInfo> serverSideIteratorList, Map<String,Map<String,String>> serverSideIteratorOptions, int size,
Authorizations authorizations, boolean retry, AccumuloConfiguration conf) throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
if (server == null)
@@ -111,7 +109,7 @@ public class ThriftScanner {
return isr.result.more;
} finally {
- ThriftUtil.returnClient((TServiceClient) client);
+ ThriftUtil.returnClient(client);
}
} catch (TApplicationException tae) {
throw new AccumuloServerException(server, tae);
@@ -122,7 +120,7 @@ public class ThriftScanner {
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (TException e) {
log.debug("Error getting transport to " + server + " : " + e);
- }
+ }
throw new AccumuloException("getBatchFromServer: failed");
}
@@ -213,8 +211,8 @@ public class ThriftScanner {
Span locateSpan = Trace.start("scan:locateTablet");
try {
- loc = TabletLocator.getInstance(instance, scanState.tableId).locateTablet(scanState.startRow, scanState.skipStartRow, false, credentials);
-
+ loc = TabletLocator.getLocator(instance, scanState.tableId).locateTablet(scanState.startRow, scanState.skipStartRow, false, credentials);
+
if (loc == null) {
if (!Tables.exists(instance, scanState.tableId.toString()))
throw new TableDeletedException(scanState.tableId.toString());
@@ -281,7 +279,7 @@ public class ThriftScanner {
log.trace(error);
lastError = error;
- TabletLocator.getInstance(instance, scanState.tableId).invalidateCache(loc.tablet_extent);
+ TabletLocator.getLocator(instance, scanState.tableId).invalidateCache(loc.tablet_extent);
loc = null;
// no need to try the current scan id somewhere else
@@ -327,7 +325,7 @@ public class ThriftScanner {
UtilWaitThread.sleep(100);
} catch (TException e) {
- TabletLocator.getInstance(instance, scanState.tableId).invalidateCache(loc.tablet_location);
+ TabletLocator.getLocator(instance, scanState.tableId).invalidateCache(loc.tablet_location);
error = "Scan failed, thrift error " + e.getClass().getName() + " " + e.getMessage() + " " + loc;
if (!error.equals(lastError))
log.debug(error);
@@ -443,7 +441,7 @@ public class ThriftScanner {
} catch (ThriftSecurityException e) {
throw new AccumuloSecurityException(e.user, e.code, e);
} finally {
- ThriftUtil.returnClient((TServiceClient) client);
+ ThriftUtil.returnClient(client);
Thread.currentThread().setName(old);
}
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
index cec700d..85b1869 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/impl/Writer.java
@@ -84,7 +84,7 @@ public class Writer {
throw new IllegalArgumentException("Can not add empty mutations");
while (true) {
- TabletLocation tabLoc = TabletLocator.getInstance(instance, table).locateTablet(new Text(m.getRow()), false, true, credentials);
+ TabletLocation tabLoc = TabletLocator.getLocator(instance, table).locateTablet(new Text(m.getRow()), false, true, credentials);
if (tabLoc == null) {
log.trace("No tablet location found for row " + new String(m.getRow()));
@@ -97,10 +97,10 @@ public class Writer {
return;
} catch (NotServingTabletException e) {
log.trace("Not serving tablet, server = " + tabLoc.tablet_location);
- TabletLocator.getInstance(instance, table).invalidateCache(tabLoc.tablet_extent);
+ TabletLocator.getLocator(instance, table).invalidateCache(tabLoc.tablet_extent);
} catch (TException e) {
log.error("error sending update to " + tabLoc.tablet_location + ": " + e);
- TabletLocator.getInstance(instance, table).invalidateCache(tabLoc.tablet_extent);
+ TabletLocator.getLocator(instance, table).invalidateCache(tabLoc.tablet_extent);
}
UtilWaitThread.sleep(500);
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
index f6fc744..e5829af 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapred/InputFormatBase.java
@@ -53,10 +53,11 @@ import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.hadoop.io.Text;
@@ -687,10 +688,10 @@ public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- MetadataTable.PREV_ROW_COLUMN.fetch(scanner);
- scanner.fetchColumnFamily(MetadataTable.LAST_LOCATION_COLUMN_FAMILY);
- scanner.fetchColumnFamily(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY);
- scanner.fetchColumnFamily(MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
scanner.setRange(metadataRange);
RowIterator rowIter = new RowIterator(scanner);
@@ -707,16 +708,16 @@ public abstract class InputFormatBase<K,V> implements InputFormat<K,V> {
Entry<Key,Value> entry = row.next();
Key key = entry.getKey();
- if (key.getColumnFamily().equals(MetadataTable.LAST_LOCATION_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(TabletsSection.LastLocationColumnFamily.NAME)) {
last = entry.getValue().toString();
}
- if (key.getColumnFamily().equals(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY)
- || key.getColumnFamily().equals(MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME)) {
location = entry.getValue().toString();
}
- if (MetadataTable.PREV_ROW_COLUMN.hasColumns(key)) {
+ if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
extent = new KeyExtent(key.getRow(), entry.getValue());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
index c280b97..ea40e02 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/InputFormatBase.java
@@ -64,10 +64,11 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.user.VersioningIterator;
import org.apache.accumulo.core.master.state.tables.TableState;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
-import org.apache.accumulo.core.util.MetadataTable;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.UtilWaitThread;
import org.apache.hadoop.conf.Configuration;
@@ -711,10 +712,10 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
Range metadataRange = new Range(new KeyExtent(new Text(tableId), startRow, null).getMetadataEntry(), true, null, false);
Scanner scanner = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
- MetadataTable.PREV_ROW_COLUMN.fetch(scanner);
- scanner.fetchColumnFamily(MetadataTable.LAST_LOCATION_COLUMN_FAMILY);
- scanner.fetchColumnFamily(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY);
- scanner.fetchColumnFamily(MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY);
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
+ scanner.fetchColumnFamily(TabletsSection.LastLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.CurrentLocationColumnFamily.NAME);
+ scanner.fetchColumnFamily(TabletsSection.FutureLocationColumnFamily.NAME);
scanner.setRange(metadataRange);
RowIterator rowIter = new RowIterator(scanner);
@@ -731,16 +732,16 @@ public abstract class InputFormatBase<K,V> extends InputFormat<K,V> {
Entry<Key,Value> entry = row.next();
Key key = entry.getKey();
- if (key.getColumnFamily().equals(MetadataTable.LAST_LOCATION_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(TabletsSection.LastLocationColumnFamily.NAME)) {
last = entry.getValue().toString();
}
- if (key.getColumnFamily().equals(MetadataTable.CURRENT_LOCATION_COLUMN_FAMILY)
- || key.getColumnFamily().equals(MetadataTable.FUTURE_LOCATION_COLUMN_FAMILY)) {
+ if (key.getColumnFamily().equals(TabletsSection.CurrentLocationColumnFamily.NAME)
+ || key.getColumnFamily().equals(TabletsSection.FutureLocationColumnFamily.NAME)) {
location = entry.getValue().toString();
}
- if (MetadataTable.PREV_ROW_COLUMN.hasColumns(key)) {
+ if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.hasColumns(key)) {
extent = new KeyExtent(key.getRow(), entry.getValue());
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
index c582a72..15e045b 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/lib/util/InputConfigurator.java
@@ -480,7 +480,7 @@ public class InputConfigurator extends ConfiguratorBase {
return new MockTabletLocator();
Instance instance = getInstance(implementingClass, conf);
String tableName = getInputTableName(implementingClass, conf);
- return TabletLocator.getInstance(instance, new Text(Tables.getTableId(instance, tableName)));
+ return TabletLocator.getLocator(instance, new Text(Tables.getTableId(instance, tableName)));
}
// InputFormat doesn't have the equivalent of OutputFormat's checkOutputSpecs(JobContext job)
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
index 346e6ee..5ee144d 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockAccumulo.java
@@ -26,11 +26,11 @@ import org.apache.accumulo.core.client.BatchScanner;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.client.security.tokens.PasswordToken;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.SystemPermission;
import org.apache.accumulo.core.security.TablePermission;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.io.Text;
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
index bf7e7f9..2c416c6 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/mock/MockTableOperations.java
@@ -46,6 +46,8 @@ import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.security.ColumnVisibility;
import org.apache.accumulo.start.classloader.vfs.AccumuloVFSClassLoader;
@@ -122,7 +124,7 @@ public class MockTableOperations extends TableOperationsHelper {
throw new TableNotFoundException(tableName, tableName, "");
return acu.getSplits(tableName);
}
-
+
@Override
public Collection<Text> listSplits(String tableName, int maxSplits) throws TableNotFoundException {
return listSplits(tableName);
@@ -297,20 +299,25 @@ public class MockTableOperations extends TableOperationsHelper {
public Map<String,String> tableIdMap() {
Map<String,String> result = new HashMap<String,String>();
for (String table : acu.tables.keySet()) {
- result.put(table, table);
+ if (RootTable.NAME.equals(table))
+ result.put(table, RootTable.ID);
+ else if (MetadataTable.NAME.equals(table))
+ result.put(table, MetadataTable.ID);
+ else
+ result.put(table, table);
}
return result;
}
-
+
@Override
public List<DiskUsage> getDiskUsage(Set<String> tables) throws AccumuloException, AccumuloSecurityException {
-
+
List<DiskUsage> diskUsages = new ArrayList<DiskUsage>();
diskUsages.add(new DiskUsage(new TreeSet<String>(tables), 0l));
-
+
return diskUsages;
}
-
+
@Override
public void merge(String tableName, Text start, Text end) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
if (!exists(tableName))
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
index dc45e91..fc766fe 100644
--- a/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
+++ b/core/src/main/java/org/apache/accumulo/core/data/KeyExtent.java
@@ -40,9 +40,11 @@ import java.util.UUID;
import java.util.WeakHashMap;
import org.apache.accumulo.core.data.thrift.TKeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import org.apache.accumulo.core.util.ByteBufferUtil;
-import org.apache.accumulo.core.util.MetadataTable;
-import org.apache.accumulo.core.util.RootTable;
import org.apache.accumulo.core.util.TextUtil;
import org.apache.hadoop.io.BinaryComparable;
import org.apache.hadoop.io.Text;
@@ -128,18 +130,8 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
return getMetadataEntry(getTableId(), getEndRow());
}
- public static Text getMetadataEntry(Text table, Text row) {
- Text entry = new Text(table);
-
- if (row == null) {
- entry.append(new byte[] {'<'}, 0, 1);
- } else {
- entry.append(new byte[] {';'}, 0, 1);
- entry.append(row.getBytes(), 0, row.getLength());
- }
-
- return entry;
-
+ public static Text getMetadataEntry(Text tableId, Text endRow) {
+ return MetadataSchema.TabletsSection.getRow(tableId, endRow);
}
// constructor for loading extents from metadata rows
@@ -398,7 +390,7 @@ public class KeyExtent implements WritableComparable<KeyExtent> {
public static Mutation getPrevRowUpdateMutation(KeyExtent ke) {
Mutation m = new Mutation(ke.getMetadataEntry());
- MetadataTable.PREV_ROW_COLUMN.put(m, encodePrevEndRow(ke.getPrevEndRow()));
+ TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.put(m, encodePrevEndRow(ke.getPrevEndRow()));
return m;
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
new file mode 100644
index 0000000..237cacc
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.impl.AccumuloServerException;
+import org.apache.accumulo.core.client.impl.ScannerOptions;
+import org.apache.accumulo.core.client.impl.TabletLocator;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocation;
+import org.apache.accumulo.core.client.impl.TabletLocator.TabletLocations;
+import org.apache.accumulo.core.client.impl.TabletLocatorImpl.TabletLocationObtainer;
+import org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator;
+import org.apache.accumulo.core.client.impl.TabletServerBatchReaderIterator.ResultReceiver;
+import org.apache.accumulo.core.client.impl.ThriftScanner;
+import org.apache.accumulo.core.data.Column;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
+import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public class MetadataLocationObtainer implements TabletLocationObtainer {
+ private static final Logger log = Logger.getLogger(MetadataLocationObtainer.class);
+ private SortedSet<Column> locCols;
+ private ArrayList<Column> columns;
+ private Instance instance;
+
+ public MetadataLocationObtainer(Instance instance) {
+
+ this.instance = instance;
+
+ locCols = new TreeSet<Column>();
+ locCols.add(new Column(TextUtil.getBytes(TabletsSection.CurrentLocationColumnFamily.NAME), null, null));
+ locCols.add(TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.toColumn());
+ columns = new ArrayList<Column>(locCols);
+ }
+
+ @Override
+ public TabletLocations lookupTablet(TabletLocation src, Text row, Text stopRow, TabletLocator parent, TCredentials credentials)
+ throws AccumuloSecurityException, AccumuloException {
+
+ try {
+ ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
+
+ OpTimer opTimer = null;
+ if (log.isTraceEnabled())
+ opTimer = new OpTimer(log, Level.TRACE).start("Looking up in " + src.tablet_extent.getTableId() + " row=" + TextUtil.truncate(row) + " extent="
+ + src.tablet_extent + " tserver=" + src.tablet_location);
+
+ Range range = new Range(row, true, stopRow, true);
+
+ TreeMap<Key,Value> encodedResults = new TreeMap<Key,Value>();
+ TreeMap<Key,Value> results = new TreeMap<Key,Value>();
+
+ // Use the whole row iterator so that a partial mutations is not read. The code that extracts locations for tablets does a sanity check to ensure there is
+ // only one location. Reading a partial mutation could make it appear there are multiple locations when there are not.
+ List<IterInfo> serverSideIteratorList = new ArrayList<IterInfo>();
+ serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI"));
+ Map<String,Map<String,String>> serverSideIteratorOptions = Collections.emptyMap();
+
+ boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols,
+ serverSideIteratorList, serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Authorizations.EMPTY, false, instance.getConfiguration());
+
+ decodeRows(encodedResults, results);
+
+ if (more && results.size() == 1) {
+ range = new Range(results.lastKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), true, new Key(stopRow).followingKey(PartialKey.ROW), false);
+ encodedResults.clear();
+ more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols, serverSideIteratorList,
+ serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Authorizations.EMPTY, false, instance.getConfiguration());
+
+ decodeRows(encodedResults, results);
+ }
+
+ if (opTimer != null)
+ opTimer.stop("Got " + results.size() + " results from " + src.tablet_extent + " in %DURATION%");
+
+ // System.out.println("results "+results.keySet());
+
+ Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> metadata = MetadataLocationObtainer.getMetadataLocationEntries(results);
+
+ for (Entry<KeyExtent,Text> entry : metadata.getFirst().entrySet()) {
+ list.add(new TabletLocation(entry.getKey(), entry.getValue().toString()));
+ }
+
+ return new TabletLocations(list, metadata.getSecond());
+
+ } catch (AccumuloServerException ase) {
+ if (log.isTraceEnabled())
+ log.trace(src.tablet_extent.getTableId() + " lookup failed, " + src.tablet_location + " server side exception");
+ throw ase;
+ } catch (NotServingTabletException e) {
+ if (log.isTraceEnabled())
+ log.trace(src.tablet_extent.getTableId() + " lookup failed, " + src.tablet_location + " not serving " + src.tablet_extent);
+ parent.invalidateCache(src.tablet_extent);
+ } catch (AccumuloException e) {
+ if (log.isTraceEnabled())
+ log.trace(src.tablet_extent.getTableId() + " lookup failed", e);
+ parent.invalidateCache(src.tablet_location);
+ }
+
+ return null;
+ }
+
+ private void decodeRows(TreeMap<Key,Value> encodedResults, TreeMap<Key,Value> results) throws AccumuloException {
+ for (Entry<Key,Value> entry : encodedResults.entrySet()) {
+ try {
+ results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
+ } catch (IOException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ }
+
+ @Override
+ public List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>> tabletsRanges, TabletLocator parent, TCredentials credentials)
+ throws AccumuloSecurityException, AccumuloException {
+
+ final TreeMap<Key,Value> results = new TreeMap<Key,Value>();
+
+ ArrayList<TabletLocation> list = new ArrayList<TabletLocation>();
+
+ ResultReceiver rr = new ResultReceiver() {
+
+ @Override
+ public void receive(List<Entry<Key,Value>> entries) {
+ for (Entry<Key,Value> entry : entries) {
+ try {
+ results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ };
+
+ ScannerOptions opts = new ScannerOptions() {
+ ScannerOptions setOpts() {
+ this.fetchedColumns = locCols;
+ this.serverSideIteratorList = new ArrayList<IterInfo>();
+ // see comment in lookupTablet about why iterator is used
+ this.serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI"));
+ return this;
+ }
+ }.setOpts();
+
+ Map<KeyExtent,List<Range>> unscanned = new HashMap<KeyExtent,List<Range>>();
+ Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
+ try {
+ TabletServerBatchReaderIterator.doLookup(tserver, tabletsRanges, failures, unscanned, rr, columns, credentials, opts, Authorizations.EMPTY,
+ instance.getConfiguration());
+ if (failures.size() > 0) {
+ // invalidate extents in parents cache
+ if (log.isTraceEnabled())
+ log.trace("lookupTablets failed for " + failures.size() + " extents");
+ parent.invalidateCache(failures.keySet());
+ }
+ } catch (IOException e) {
+ log.trace("lookupTablets failed server=" + tserver, e);
+ parent.invalidateCache(tserver);
+ } catch (AccumuloServerException e) {
+ log.trace("lookupTablets failed server=" + tserver, e);
+ throw e;
+ }
+
+ SortedMap<KeyExtent,Text> metadata = MetadataLocationObtainer.getMetadataLocationEntries(results).getFirst();
+
+ for (Entry<KeyExtent,Text> entry : metadata.entrySet()) {
+ list.add(new TabletLocation(entry.getKey(), entry.getValue().toString()));
+ }
+
+ return list;
+ }
+
+ public static Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> getMetadataLocationEntries(SortedMap<Key,Value> entries) {
+ Key key;
+ Value val;
+ Text location = null;
+ Value prevRow = null;
+ KeyExtent ke;
+
+ SortedMap<KeyExtent,Text> results = new TreeMap<KeyExtent,Text>();
+ ArrayList<KeyExtent> locationless = new ArrayList<KeyExtent>();
+
+ Text lastRowFromKey = new Text();
+
+ // text obj below is meant to be reused in loop for efficiency
+ Text colf = new Text();
+ Text colq = new Text();
+
+ for (Entry<Key,Value> entry : entries.entrySet()) {
+ key = entry.getKey();
+ val = entry.getValue();
+
+ if (key.compareRow(lastRowFromKey) != 0) {
+ prevRow = null;
+ location = null;
+ key.getRow(lastRowFromKey);
+ }
+
+ colf = key.getColumnFamily(colf);
+ colq = key.getColumnQualifier(colq);
+
+ // interpret the row id as a key extent
+ if (colf.equals(TabletsSection.CurrentLocationColumnFamily.NAME) || colf.equals(TabletsSection.FutureLocationColumnFamily.NAME)) {
+ if (location != null) {
+ throw new IllegalStateException("Tablet has multiple locations : " + lastRowFromKey);
+ }
+ location = new Text(val.toString());
+ } else if (TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.equals(colf, colq)) {
+ prevRow = new Value(val);
+ }
+
+ if (prevRow != null) {
+ ke = new KeyExtent(key.getRow(), prevRow);
+ if (location != null)
+ results.put(ke, location);
+ else
+ locationless.add(ke);
+
+ location = null;
+ prevRow = null;
+ }
+ }
+
+ return new Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>>(results, locationless);
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/MetadataServicer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataServicer.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataServicer.java
new file mode 100644
index 0000000..7473fcf
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataServicer.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.security.CredentialHelper;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+import org.apache.accumulo.core.util.ArgumentChecker;
+
+/**
+ * Provides a consolidated API for handling table metadata
+ */
+public abstract class MetadataServicer {
+
+ public static MetadataServicer forTableName(Instance instance, TCredentials credentials, String tableName) throws AccumuloException,
+ AccumuloSecurityException {
+ ArgumentChecker.notNull(tableName);
+ Connector conn = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials));
+ return forTableId(instance, credentials, conn.tableOperations().tableIdMap().get(tableName));
+ }
+
+ public static MetadataServicer forTableId(Instance instance, TCredentials credentials, String tableId) {
+ ArgumentChecker.notNull(tableId);
+ if (RootTable.ID.equals(tableId))
+ return new ServicerForRootTable(instance, credentials);
+ else if (MetadataTable.ID.equals(tableId))
+ return new ServicerForMetadataTable(instance, credentials);
+ else
+ return new ServicerForUserTables(instance, credentials, tableId);
+ }
+
+ /**
+ *
+ * @return the table id of the table currently being serviced
+ */
+ public abstract String getServicedTableId();
+
+ /**
+ * Populate the provided data structure with the known tablets for the table being serviced
+ *
+ * @param tablets
+ * A mapping of all known tablets to their location (if available, null otherwise)
+ */
+ public abstract void getTabletLocations(SortedMap<KeyExtent,String> tablets) throws AccumuloException, AccumuloSecurityException, TableNotFoundException;
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/MetadataTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataTable.java
new file mode 100644
index 0000000..7276f6c
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataTable.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+
+public class MetadataTable {
+
+ public static final String ID = "!0";
+ public static final String NAME = "!METADATA";
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
new file mode 100644
index 0000000..402c1cc
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/RootTable.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.hadoop.io.Text;
+
+/**
+ *
+ */
+public class RootTable {
+
+ public static final String ID = "!!R";
+ public static final String NAME = "!!ROOT";
+
+ /**
+ * DFS location relative to the Accumulo directory
+ */
+ public static final String ROOT_TABLET_LOCATION = "/root_tablet";
+
+ /**
+ * ZK path relative to the instance directory for information about the root tablet
+ */
+ public static final String ZROOT_TABLET = ROOT_TABLET_LOCATION;
+ public static final String ZROOT_TABLET_LOCATION = ZROOT_TABLET + "/location";
+ public static final String ZROOT_TABLET_FUTURE_LOCATION = ZROOT_TABLET + "/future_location";
+ public static final String ZROOT_TABLET_LAST_LOCATION = ZROOT_TABLET + "/lastlocation";
+ public static final String ZROOT_TABLET_WALOGS = ZROOT_TABLET + "/walogs";
+
+ public static final KeyExtent EXTENT = new KeyExtent(new Text(ID), null, null);
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForMetadataTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForMetadataTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForMetadataTable.java
new file mode 100644
index 0000000..64e49ef
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForMetadataTable.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+
+/**
+ * A metadata servicer for the metadata table (which holds metadata for user tables).<br />
+ * The metadata table's metadata is serviced in the root table.
+ */
+class ServicerForMetadataTable extends TableMetadataServicer {
+
+ public ServicerForMetadataTable(Instance instance, TCredentials credentials) {
+ super(instance, credentials, RootTable.NAME, MetadataTable.ID);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForRootTable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForRootTable.java b/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForRootTable.java
new file mode 100644
index 0000000..4366991
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForRootTable.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import java.util.SortedMap;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.KeyExtent;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+
+/**
+ * A metadata servicer for the root table.<br />
+ * The root table's metadata is serviced in zookeeper.
+ */
+class ServicerForRootTable extends MetadataServicer {
+
+ private Instance instance;
+
+ public ServicerForRootTable(Instance instance, TCredentials credentials) {
+ this.instance = instance;
+ }
+
+ @Override
+ public String getServicedTableId() {
+ return RootTable.ID;
+ }
+
+ @Override
+ public void getTabletLocations(SortedMap<KeyExtent,String> tablets) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
+ tablets.put(RootTable.EXTENT, instance.getRootTabletLocation());
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/446a37a9/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java b/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java
new file mode 100644
index 0000000..7e15bc0
--- /dev/null
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/ServicerForUserTables.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.metadata;
+
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.security.thrift.TCredentials;
+
+/**
+ * A metadata servicer for user tables.<br />
+ * Metadata for user tables are serviced in the metadata table.
+ */
+class ServicerForUserTables extends TableMetadataServicer {
+
+ public ServicerForUserTables(Instance instance, TCredentials credentials, String tableId) {
+ super(instance, credentials, MetadataTable.NAME, tableId);
+ }
+
+}