You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2013/04/06 01:12:58 UTC
svn commit: r1465155 - in /accumulo/branches/1.5:
core/src/main/java/org/apache/accumulo/core/client/impl/
core/src/main/java/org/apache/accumulo/core/util/
core/src/test/java/org/apache/accumulo/core/client/impl/
server/src/main/java/org/apache/accumu...
Author: kturner
Date: Fri Apr 5 23:12:58 2013
New Revision: 1465155
URL: http://svn.apache.org/r1465155
Log:
ACCUMULO-1248 made tablet location cache throw an exception if a tablet has multiple locations
Modified:
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java?rev=1465155&r1=1465154&r2=1465155&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java (original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/MetadataLocationObtainer.java Fri Apr 5 23:12:58 2013
@@ -18,6 +18,7 @@ package org.apache.accumulo.core.client.
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -41,6 +42,8 @@ import org.apache.accumulo.core.data.Key
import org.apache.accumulo.core.data.PartialKey;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.data.thrift.IterInfo;
+import org.apache.accumulo.core.iterators.user.WholeRowIterator;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.util.MetadataTable;
@@ -81,16 +84,27 @@ public class MetadataLocationObtainer im
Range range = new Range(row, true, stopRow, true);
+ TreeMap<Key,Value> encodedResults = new TreeMap<Key,Value>();
TreeMap<Key,Value> results = new TreeMap<Key,Value>();
- // System.out.println(range);
+ // Use the whole row iterator so that a partial mutations is not read. The code that extracts locations for tablets does a sanity check to ensure there is
+ // only one location. Reading a partial mutation could make it appear there are multiple locations when there are not.
+ List<IterInfo> serverSideIteratorList = new ArrayList<IterInfo>();
+ serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI"));
+ Map<String,Map<String,String>> serverSideIteratorOptions = Collections.emptyMap();
- boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, results, locCols, Constants.SCAN_BATCH_SIZE,
- Constants.NO_AUTHS, false, instance.getConfiguration());
+ boolean more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols,
+ serverSideIteratorList, serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Constants.NO_AUTHS, false, instance.getConfiguration());
+
+ decodeRows(encodedResults, results);
+
if (more && results.size() == 1) {
range = new Range(results.lastKey().followingKey(PartialKey.ROW_COLFAM_COLQUAL_COLVIS_TIME), true, new Key(stopRow).followingKey(PartialKey.ROW), false);
- more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, results, locCols, Constants.SCAN_BATCH_SIZE,
- Constants.NO_AUTHS, false, instance.getConfiguration());
+ encodedResults.clear();
+ more = ThriftScanner.getBatchFromServer(credentials, range, src.tablet_extent, src.tablet_location, encodedResults, locCols, serverSideIteratorList,
+ serverSideIteratorOptions, Constants.SCAN_BATCH_SIZE, Constants.NO_AUTHS, false, instance.getConfiguration());
+
+ decodeRows(encodedResults, results);
}
if (opTimer != null)
@@ -122,10 +136,20 @@ public class MetadataLocationObtainer im
return null;
}
+
+ private void decodeRows(TreeMap<Key,Value> encodedResults, TreeMap<Key,Value> results) throws AccumuloException {
+ for (Entry<Key,Value> entry : encodedResults.entrySet()) {
+ try {
+ results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
+ } catch (IOException e) {
+ throw new AccumuloException(e);
+ }
+ }
+ }
@Override
- public List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>> tabletsRanges, TabletLocator parent, TCredentials credentials) throws AccumuloSecurityException,
- AccumuloException {
+ public List<TabletLocation> lookupTablets(String tserver, Map<KeyExtent,List<Range>> tabletsRanges, TabletLocator parent, TCredentials credentials)
+ throws AccumuloSecurityException, AccumuloException {
final TreeMap<Key,Value> results = new TreeMap<Key,Value>();
@@ -136,13 +160,20 @@ public class MetadataLocationObtainer im
@Override
public void receive(List<Entry<Key,Value>> entries) {
for (Entry<Key,Value> entry : entries) {
- results.put(entry.getKey(), entry.getValue());
+ try {
+ results.putAll(WholeRowIterator.decodeRow(entry.getKey(), entry.getValue()));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
}
};
ScannerOptions opts = new ScannerOptions();
opts.fetchedColumns = locCols;
+ opts.serverSideIteratorList = new ArrayList<IterInfo>();
+ opts.serverSideIteratorList.add(new IterInfo(10000, WholeRowIterator.class.getName(), "WRI")); // see comment in lookupTablet about why iterator is
+ // used
Map<KeyExtent,List<Range>> unscanned = new HashMap<KeyExtent,List<Range>>();
Map<KeyExtent,List<Range>> failures = new HashMap<KeyExtent,List<Range>>();
@@ -162,7 +193,7 @@ public class MetadataLocationObtainer im
log.trace("lookupTablets failed server=" + tserver, e);
throw e;
}
-
+
SortedMap<KeyExtent,Text> metadata = MetadataTable.getMetadataLocationEntries(results).getFirst();
for (Entry<KeyExtent,Text> entry : metadata.entrySet()) {
Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java?rev=1465155&r1=1465154&r2=1465155&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java (original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/client/impl/ThriftScanner.java Fri Apr 5 23:12:58 2013
@@ -27,10 +27,6 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
-import org.apache.accumulo.trace.instrument.Span;
-import org.apache.accumulo.trace.instrument.Trace;
-import org.apache.accumulo.trace.instrument.Tracer;
-import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
import org.apache.accumulo.core.client.Instance;
@@ -59,9 +55,12 @@ import org.apache.accumulo.core.tabletse
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
import org.apache.accumulo.core.tabletserver.thrift.TooManyFilesException;
import org.apache.accumulo.core.util.OpTimer;
-import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.ThriftUtil;
import org.apache.accumulo.core.util.UtilWaitThread;
+import org.apache.accumulo.trace.instrument.Span;
+import org.apache.accumulo.trace.instrument.Trace;
+import org.apache.accumulo.trace.instrument.Tracer;
+import org.apache.accumulo.trace.thrift.TInfo;
import org.apache.hadoop.io.Text;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -71,7 +70,6 @@ import org.apache.thrift.TServiceClient;
public class ThriftScanner {
- private static final byte[] EMPTY_BYTES = new byte[0];
private static final Logger log = Logger.getLogger(ThriftScanner.class);
public static final Map<TabletType,Set<String>> serversWaitedForWrites = new EnumMap<TabletType,Set<String>>(TabletType.class);
@@ -82,38 +80,9 @@ public class ThriftScanner {
}
}
- public static boolean getBatchFromServer(TCredentials credentials, Text startRow, KeyExtent extent, String server, SortedMap<Key,Value> results,
- SortedSet<Column> fetchedColumns, boolean skipStartKey, int size, Authorizations authorizations, boolean retry, AccumuloConfiguration conf)
- throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
- Key startKey;
-
- if (fetchedColumns.size() > 0) {
- byte[] cf = fetchedColumns.first().columnFamily;
- byte[] cq = fetchedColumns.first().columnQualifier;
- byte[] cv = fetchedColumns.first().columnVisibility;
-
- startKey = new Key(TextUtil.getBytes(startRow), cf, cq == null ? EMPTY_BYTES : cq, cv == null ? EMPTY_BYTES : cv, Long.MAX_VALUE);
-
- } else
- startKey = new Key(startRow);
-
- if (skipStartKey)
- startKey = startKey.followingKey(PartialKey.ROW);
- else
- startKey.setTimestamp(Long.MAX_VALUE);
-
- return getBatchFromServer(credentials, startKey, (Key) null, extent, server, results, fetchedColumns, size, authorizations, retry, conf);
- }
-
- static boolean getBatchFromServer(TCredentials credentials, Key key, Key endKey, KeyExtent extent, String server, SortedMap<Key,Value> results,
- SortedSet<Column> fetchedColumns, int size, Authorizations authorizations, boolean retry, AccumuloConfiguration conf) throws AccumuloException,
- AccumuloSecurityException, NotServingTabletException {
- return getBatchFromServer(credentials, new Range(key, true, endKey, true), extent, server, results, fetchedColumns, size, authorizations, retry, conf);
- }
-
static boolean getBatchFromServer(TCredentials credentials, Range range, KeyExtent extent, String server, SortedMap<Key,Value> results,
- SortedSet<Column> fetchedColumns, int size, Authorizations authorizations, boolean retry, AccumuloConfiguration conf) throws AccumuloException,
- AccumuloSecurityException, NotServingTabletException {
+ SortedSet<Column> fetchedColumns, List<IterInfo> serverSideIteratorList, Map<String,Map<String,String>> serverSideIteratorOptions, int size,
+ Authorizations authorizations, boolean retry, AccumuloConfiguration conf) throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
if (server == null)
throw new AccumuloException(new IOException());
@@ -121,10 +90,9 @@ public class ThriftScanner {
TInfo tinfo = Tracer.traceInfo();
TabletClientService.Client client = ThriftUtil.getTServerClient(server, conf);
try {
- List<IterInfo> emptyList = Collections.emptyList();
- Map<String,Map<String,String>> emptyMap = Collections.emptyMap();
// not reading whole rows (or stopping on row boundries) so there is no need to enable isolation below
- ScanState scanState = new ScanState(credentials, extent.getTableId(), authorizations, range, fetchedColumns, size, emptyList, emptyMap, false);
+ ScanState scanState = new ScanState(credentials, extent.getTableId(), authorizations, range, fetchedColumns, size, serverSideIteratorList,
+ serverSideIteratorOptions, false);
TabletType ttype = TabletType.type(extent);
boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server);
Modified: accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java?rev=1465155&r1=1465154&r2=1465155&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java (original)
+++ accumulo/branches/1.5/core/src/main/java/org/apache/accumulo/core/util/MetadataTable.java Fri Apr 5 23:12:58 2013
@@ -150,10 +150,14 @@ public class MetadataTable {
colq = key.getColumnQualifier(colq);
// interpret the row id as a key extent
- if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) || colf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY))
+ if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) || colf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY)) {
+ if (location != null) {
+ throw new IllegalStateException("Tablet has multiple locations : " + lastRowFromKey);
+ }
location = new Text(val.toString());
- else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
+ } else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq)) {
prevRow = new Value(val);
+ }
if (prevRow != null) {
ke = new KeyExtent(key.getRow(), prevRow);
Modified: accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java?rev=1465155&r1=1465154&r2=1465155&view=diff
==============================================================================
--- accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java (original)
+++ accumulo/branches/1.5/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java Fri Apr 5 23:12:58 2013
@@ -605,7 +605,7 @@ public class TabletLocatorImplTest exten
}
}
- static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location) {
+ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location, String instance) {
Map<KeyExtent,SortedMap<Key,Value>> tablets = tservers.tservers.get(server);
if (tablets == null) {
tablets = new HashMap<KeyExtent,SortedMap<Key,Value>>();
@@ -622,15 +622,20 @@ public class TabletLocatorImplTest exten
Value per = KeyExtent.encodePrevEndRow(ke.getPrevEndRow());
if (location != null) {
- Key lk = new Key(mr, Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text());
+ if (instance == null)
+ instance = "";
+ Key lk = new Key(mr, Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY, new Text(instance));
tabletData.put(lk, new Value(location.getBytes()));
}
Key pk = new Key(mr, Constants.METADATA_PREV_ROW_COLUMN.getColumnFamily(), Constants.METADATA_PREV_ROW_COLUMN.getColumnQualifier());
tabletData.put(pk, per);
-
}
+ static void setLocation(TServers tservers, String server, KeyExtent tablet, KeyExtent ke, String location) {
+ setLocation(tservers, server, tablet, ke, location, "");
+ }
+
static void deleteServer(TServers tservers, String server) {
tservers.tservers.remove(server);
@@ -1269,4 +1274,25 @@ public class TabletLocatorImplTest exten
locateTabletTest(tab0TabletCache, "a", ke1, "tserver7", credential);
}
+
+ public void testAccumulo1248() throws Exception {
+ TServers tservers = new TServers();
+ TabletLocatorImpl metaCache = createLocators(tservers, "tserver1", "tserver2", "foo");
+
+ KeyExtent ke1 = nke("foo", null, null);
+
+ // set two locations for a tablet, this is not supposed to happen. The metadata cache should throw an exception if it sees this rather than caching one of
+ // the locations.
+ setLocation(tservers, "tserver2", MTE, ke1, "L1", "I1");
+ setLocation(tservers, "tserver2", MTE, ke1, "L2", "I2");
+
+ try {
+ metaCache.locateTablet(new Text("a"), false, false, credential);
+ assertTrue(false);
+ } catch (Exception e) {
+
+ }
+
+
+ }
}
Modified: accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java
URL: http://svn.apache.org/viewvc/accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java?rev=1465155&r1=1465154&r2=1465155&view=diff
==============================================================================
--- accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java (original)
+++ accumulo/branches/1.5/server/src/main/java/org/apache/accumulo/server/util/MetadataTable.java Fri Apr 5 23:12:58 2013
@@ -30,7 +30,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
-import java.util.SortedSet;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
@@ -47,10 +46,7 @@ import org.apache.accumulo.core.client.S
import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.impl.BatchWriterImpl;
import org.apache.accumulo.core.client.impl.ScannerImpl;
-import org.apache.accumulo.core.client.impl.ThriftScanner;
import org.apache.accumulo.core.client.impl.Writer;
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
-import org.apache.accumulo.core.data.Column;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.KeyExtent;
import org.apache.accumulo.core.data.Mutation;
@@ -61,7 +57,6 @@ import org.apache.accumulo.core.file.Fil
import org.apache.accumulo.core.security.CredentialHelper;
import org.apache.accumulo.core.security.thrift.TCredentials;
import org.apache.accumulo.core.tabletserver.thrift.ConstraintViolationException;
-import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.util.CachedConfiguration;
import org.apache.accumulo.core.util.ColumnFQ;
import org.apache.accumulo.core.util.FastFormat;
@@ -324,21 +319,6 @@ public class MetadataTable extends org.a
return results;
}
- public static boolean getBatchFromRootTablet(AccumuloConfiguration conf, TCredentials credentials, Text startRow, SortedMap<Key,Value> results,
- SortedSet<Column> columns, boolean skipStartRow, int size) throws AccumuloSecurityException {
- while (true) {
- try {
- return ThriftScanner.getBatchFromServer(credentials, startRow, Constants.ROOT_TABLET_EXTENT, HdfsZooInstance.getInstance().getRootTabletLocation(),
- results, columns, skipStartRow, size, Constants.NO_AUTHS, true, conf);
- } catch (NotServingTabletException e) {
- UtilWaitThread.sleep(100);
- } catch (AccumuloException e) {
- UtilWaitThread.sleep(100);
- }
- }
-
- }
-
public static boolean recordRootTabletLocation(String address) {
IZooReaderWriter zoo = ZooReaderWriter.getInstance();
for (int i = 0; i < SAVE_ROOT_TABLET_RETRIES; i++) {