You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2013/08/28 21:59:10 UTC
svn commit: r1518346 [1/3] - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/
hbase-client/src/test/java/org...
Author: stack
Date: Wed Aug 28 19:59:09 2013
New Revision: 1518346
URL: http://svn.apache.org/r1518346
Log:
HBASE-9230 Fix the server so it can take a pure pb request param and return a pure pb result
Added:
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSideNoCodec.java
Modified:
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java
hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/RPCProtos.java
hbase/trunk/hbase-protocol/src/main/protobuf/Client.proto
hbase/trunk/hbase-protocol/src/main/protobuf/RPC.proto
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcCallContext.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServerInterface.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/TestCheckTestClasses.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/catalog/TestMetaReaderEditorNoCluster.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverScannerOpenHook.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/ipc/TestIPC.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
hbase/trunk/src/main/docbkx/rpc.xml
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java Wed Aug 28 19:59:09 2013
@@ -48,17 +48,16 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Chore;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.DoNotRetryIOException;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.Stoppable;
+import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
@@ -73,26 +72,6 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ClientService;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceRequest;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.CoprocessorServiceResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyNamespaceResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyNamespaceRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateNamespaceResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateNamespaceRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteNamespaceResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteNamespaceRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.GetNamespaceDescriptorResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.GetNamespaceDescriptorRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos
- .ListNamespaceDescriptorsResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos
- .ListNamespaceDescriptorsRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos
- .ListTableDescriptorsByNamespaceResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos
- .ListTableDescriptorsByNamespaceRequest;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos
- .ListTableNamesByNamespaceResponse;
-import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos
- .ListTableNamesByNamespaceRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AddColumnResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.AssignRegionRequest;
@@ -101,10 +80,14 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.BalanceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateNamespaceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CreateTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteNamespaceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteSnapshotResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableRequest;
@@ -117,17 +100,27 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.GetNamespaceDescriptorRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.GetNamespaceDescriptorResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsRestoreSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsRestoreSnapshotDoneResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsSnapshotDoneResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListNamespaceDescriptorsRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListNamespaceDescriptorsResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListSnapshotResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListTableDescriptorsByNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListTableDescriptorsByNamespaceResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListTableNamesByNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ListTableNamesByNamespaceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.MasterAdminService;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyColumnResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyNamespaceRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyNamespaceResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.ModifyTableResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.MoveRegionRequest;
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiServerCallable.java Wed Aug 28 19:59:09 2013
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScannable;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HRegionLocation;
@@ -42,12 +43,14 @@ import com.google.protobuf.ServiceExcept
*/
class MultiServerCallable<R> extends RegionServerCallable<MultiResponse> {
private final MultiAction<R> multi;
+ private final boolean cellBlock;
MultiServerCallable(final HConnection connection, final TableName tableName,
final HRegionLocation location, final MultiAction<R> multi) {
super(connection, tableName, null);
this.multi = multi;
setLocation(location);
+ this.cellBlock = isCellBlock();
}
MultiAction<R> getMulti() {
@@ -67,16 +70,21 @@ class MultiServerCallable<R> extends Reg
// Row Mutations are a set of Puts and/or Deletes all to be applied atomically
// on the one row. We do these a row at a time.
if (row instanceof RowMutations) {
+ RowMutations rms = (RowMutations)row;
+ List<CellScannable> cells = null;
+ MultiRequest multiRequest;
try {
- RowMutations rms = (RowMutations)row;
- // Stick all Cells for all RowMutations in here into 'cells'. Populated when we call
- // buildNoDataMultiRequest in the below.
- List<CellScannable> cells = new ArrayList<CellScannable>(rms.getMutations().size());
- // Build a multi request absent its Cell payload (this is the 'nodata' in the below).
- MultiRequest multiRequest =
- RequestConverter.buildNoDataMultiRequest(regionName, rms, cells);
- // Carry the cells over the proxy/pb Service interface using the payload carrying
- // rpc controller.
+ if (this.cellBlock) {
+ // Stick all Cells for all RowMutations in here into 'cells'. Populated when we call
+ // buildNoDataMultiRequest in the below.
+ cells = new ArrayList<CellScannable>(rms.getMutations().size());
+ // Build a multi request absent its Cell payload (this is the 'nodata' in the below).
+ multiRequest = RequestConverter.buildNoDataMultiRequest(regionName, rms, cells);
+ } else {
+ multiRequest = RequestConverter.buildMultiRequest(regionName, rms);
+ }
+ // Carry the cells if any over the proxy/pb Service interface using the payload
+ // carrying rpc controller.
getStub().multi(new PayloadCarryingRpcController(cells), multiRequest);
// This multi call does not return results.
response.add(regionName, action.getOriginalIndex(), Result.EMPTY_RESULT);
@@ -91,14 +99,17 @@ class MultiServerCallable<R> extends Reg
if (actions.size() > rowMutations) {
Exception ex = null;
List<Object> results = null;
- // Stick all Cells for the multiRequest in here into 'cells'. Gets filled in when we
- // call buildNoDataMultiRequest
- List<CellScannable> cells = new ArrayList<CellScannable>(actions.size() - rowMutations);
+ List<CellScannable> cells = null;
+ MultiRequest multiRequest;
try {
- // The call to buildNoDataMultiRequest will skip RowMutations. They have
- // already been handled above.
- MultiRequest multiRequest =
- RequestConverter.buildNoDataMultiRequest(regionName, actions, cells);
+ if (isCellBlock()) {
+ // Send data in cellblocks. The call to buildNoDataMultiRequest will skip RowMutations.
+ // They have already been handled above.
+ cells = new ArrayList<CellScannable>(actions.size() - rowMutations);
+ multiRequest = RequestConverter.buildNoDataMultiRequest(regionName, actions, cells);
+ } else {
+ multiRequest = RequestConverter.buildMultiRequest(regionName, actions);
+ }
// Controller optionally carries cell data over the proxy/service boundary and also
// optionally ferries cell response data back out again.
PayloadCarryingRpcController controller = new PayloadCarryingRpcController(cells);
@@ -116,9 +127,25 @@ class MultiServerCallable<R> extends Reg
return response;
}
+
+ /**
+ * @return True if we should send data in cellblocks. This is an expensive call. Cache the
+ * result if you can rather than call each time.
+ */
+ private boolean isCellBlock() {
+ // This is not exact -- the configuration could have changed on us after connection was set up
+ // but it will do for now.
+ HConnection connection = getConnection();
+ if (connection == null) return true; // Default is to do cellblocks.
+ Configuration configuration = connection.getConfiguration();
+ if (configuration == null) return true;
+ String codec = configuration.get("hbase.client.rpc.codec", "");
+ return codec != null && codec.length() > 0;
+ }
+
@Override
public void prepare(boolean reload) throws IOException {
// Use the location we were given in the constructor rather than go look it up.
setStub(getConnection().getClient(getLocation().getServerName()));
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/IPCUtil.java Wed Aug 28 19:59:09 2013
@@ -31,6 +31,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.codec.Codec;
import org.apache.hadoop.hbase.io.ByteBufferOutputStream;
import org.apache.hadoop.hbase.io.HeapSize;
@@ -71,6 +72,11 @@ class IPCUtil {
}
/**
+ * Thrown if a cellscanner but no codec to encode it with.
+ */
+ public static class CellScannerButNoCodecException extends HBaseIOException {};
+
+ /**
* Puts CellScanner Cells into a cell block using passed in <code>codec</code> and/or
* <code>compressor</code>.
* @param codec
@@ -86,6 +92,7 @@ class IPCUtil {
final CellScanner cellScanner)
throws IOException {
if (cellScanner == null) return null;
+ if (codec == null) throw new CellScannerButNoCodecException();
int bufferSize = this.cellBlockBuildingInitialBufferSize;
if (cellScanner instanceof HeapSize) {
long longSize = ((HeapSize)cellScanner).heapSize();
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/PayloadCarryingRpcController.java Wed Aug 28 19:59:09 2013
@@ -55,7 +55,7 @@ public class PayloadCarryingRpcControlle
}
public PayloadCarryingRpcController(final List<CellScannable> cellIterables) {
- this.cellScanner = CellUtil.createCellScanner(cellIterables);
+ this.cellScanner = cellIterables == null? null: CellUtil.createCellScanner(cellIterables);
}
/**
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/ipc/RpcClient.java Wed Aug 28 19:59:09 2013
@@ -89,6 +89,7 @@ import org.apache.hadoop.security.token.
import org.cloudera.htrace.Span;
import org.cloudera.htrace.Trace;
+import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.BlockingRpcChannel;
import com.google.protobuf.Descriptors.MethodDescriptor;
import com.google.protobuf.Message;
@@ -426,7 +427,9 @@ public class RpcClient {
if ((userInfoPB = getUserInfo(ticket)) != null) {
builder.setUserInfo(userInfoPB);
}
- builder.setCellBlockCodecClass(this.codec.getClass().getCanonicalName());
+ if (this.codec != null) {
+ builder.setCellBlockCodecClass(this.codec.getClass().getCanonicalName());
+ }
if (this.compressor != null) {
builder.setCellBlockCompressorClass(this.compressor.getClass().getCanonicalName());
}
@@ -1249,7 +1252,7 @@ public class RpcClient {
this.pingInterval = getPingInterval(conf);
this.ipcUtil = new IPCUtil(conf);
this.conf = conf;
- this.codec = getCodec(conf);
+ this.codec = getCodec();
this.compressor = getCompressor(conf);
this.socketFactory = factory;
this.clusterId = clusterId != null ? clusterId : HConstants.CLUSTER_ID_DEFAULT;
@@ -1291,18 +1294,28 @@ public class RpcClient {
/**
* Encapsulate the ugly casting and RuntimeException conversion in private method.
- * @param conf
* @return Codec to use on this client.
*/
- private static Codec getCodec(final Configuration conf) {
- String className = conf.get("hbase.client.rpc.codec", KeyValueCodec.class.getCanonicalName());
+ Codec getCodec() {
+ // For NO CODEC, "hbase.client.rpc.codec" must be the empty string AND
+ // "hbase.client.default.rpc.codec" -- because default is to do cell block encoding.
+ String className = conf.get("hbase.client.rpc.codec", getDefaultCodec(this.conf));
+ if (className == null || className.length() == 0) return null;
try {
- return (Codec)Class.forName(className).newInstance();
+ return (Codec)Class.forName(className).newInstance();
} catch (Exception e) {
throw new RuntimeException("Failed getting codec " + className, e);
}
}
+ @VisibleForTesting
+ public static String getDefaultCodec(final Configuration c) {
+ // If "hbase.client.default.rpc.codec" is empty string -- you can't set it to null because
+ // Configuration will complain -- then no default codec (and we'll pb everything). Else
+ // default is KeyValueCodec
+ return c.get("hbase.client.default.rpc.codec", KeyValueCodec.class.getCanonicalName());
+ }
+
/**
* Encapsulate the ugly casting and RuntimeException conversion in private method.
* @param conf
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ResponseConverter.java Wed Aug 28 19:59:09 2013
@@ -40,7 +40,6 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ActionResult;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ResultCellMeta;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.ScanResponse;
import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameBytesPair;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.CatalogScanResponse;
@@ -277,36 +276,44 @@ public final class ResponseConverter {
*/
public static Result[] getResults(CellScanner cellScanner, ScanResponse response)
throws IOException {
- if (response == null || cellScanner == null) return null;
- ResultCellMeta resultCellMeta = response.getResultCellMeta();
- if (resultCellMeta == null) return null;
- int noOfResults = resultCellMeta.getCellsLengthCount();
+ if (response == null) return null;
+ // If cellscanner, then the number of Results to return is the count of elements in the
+ // cellsPerResult list. Otherwise, it is how many results are embedded inside the response.
+ int noOfResults = cellScanner != null?
+ response.getCellsPerResultCount(): response.getResultsCount();
Result[] results = new Result[noOfResults];
for (int i = 0; i < noOfResults; i++) {
- int noOfCells = resultCellMeta.getCellsLength(i);
- List<Cell> cells = new ArrayList<Cell>(noOfCells);
- for (int j = 0; j < noOfCells; j++) {
- try {
- if (cellScanner.advance() == false) {
- // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
+ if (cellScanner != null) {
+ // Cells are out in cellblocks. Group them up again as Results. How many to read at a
+ // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
+ int noOfCells = response.getCellsPerResult(i);
+ List<Cell> cells = new ArrayList<Cell>(noOfCells);
+ for (int j = 0; j < noOfCells; j++) {
+ try {
+ if (cellScanner.advance() == false) {
+ // We are not able to retrieve the exact number of cells which ResultCellMeta says us.
+ // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
+ // same scanner will result in OutOfOrderScannerNextException
+ String msg = "Results sent from server=" + noOfResults + ". But only got " + i
+ + " results completely at client. Resetting the scanner to scan again.";
+ LOG.error(msg);
+ throw new DoNotRetryIOException(msg);
+ }
+ } catch (IOException ioe) {
+ // We are getting IOE while retrieving the cells for Results.
// We have to scan for the same results again. Throwing DNRIOE as a client retry on the
// same scanner will result in OutOfOrderScannerNextException
- String msg = "Results sent from server=" + noOfResults + ". But only got " + i
- + " results completely at client. Resetting the scanner to scan again.";
- LOG.error(msg);
- throw new DoNotRetryIOException(msg);
- }
- } catch (IOException ioe) {
- // We are getting IOE while retrieving the cells for Results.
- // We have to scan for the same results again. Throwing DNRIOE as a client retry on the
- // same scanner will result in OutOfOrderScannerNextException
- LOG.error("Exception while reading cells from result."
+ LOG.error("Exception while reading cells from result."
+ "Resetting the scanner to scan again.", ioe);
- throw new DoNotRetryIOException("Resetting the scanner.", ioe);
+ throw new DoNotRetryIOException("Resetting the scanner.", ioe);
+ }
+ cells.add(cellScanner.current());
}
- cells.add(cellScanner.current());
+ results[i] = new Result(cells);
+ } else {
+ // Result is pure pb.
+ results[i] = ProtobufUtil.toResult(response.getResults(i));
}
- results[i] = new Result(cells);
}
return results;
}
Modified: hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java (original)
+++ hbase/trunk/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestClientNoCluster.java Wed Aug 28 19:59:09 2013
@@ -26,6 +26,7 @@ import java.util.concurrent.ExecutorServ
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
@@ -40,6 +41,7 @@ import org.apache.hadoop.hbase.util.Byte
import org.junit.Before;
import org.junit.Test;
import org.junit.Ignore;
+import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
import com.google.protobuf.RpcController;
@@ -49,6 +51,7 @@ import com.google.protobuf.ServiceExcept
* Test client behavior w/o setting up a cluster.
* Mock up cluster emissions.
*/
+@Category(SmallTests.class)
public class TestClientNoCluster {
private static final Log LOG = LogFactory.getLog(TestClientNoCluster.class);
private Configuration conf;
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellScanner.java Wed Aug 28 19:59:09 2013
@@ -40,8 +40,8 @@ import org.apache.hadoop.hbase.Cell;
* Typical usage:
*
* <pre>
- * while (scanner.next()) {
- * Cell cell = scanner.get();
+ * while (scanner.advance()) {
+ * Cell cell = scanner.current();
* // do something
* }
* </pre>
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java?rev=1518346&r1=1518345&r2=1518346&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java Wed Aug 28 19:59:09 2013
@@ -172,14 +172,17 @@ public final class CellUtil {
* @return CellScanner interface over <code>cellIterable</code>
*/
public static CellScanner createCellScanner(final Iterable<Cell> cellIterable) {
+ if (cellIterable == null) return null;
return createCellScanner(cellIterable.iterator());
}
/**
* @param cells
- * @return CellScanner interface over <code>cellIterable</code>
+ * @return CellScanner interface over <code>cellIterable</code> or null if <code>cells</code> is
+ * null
*/
public static CellScanner createCellScanner(final Iterator<Cell> cells) {
+ if (cells == null) return null;
return new CellScanner() {
private final Iterator<Cell> iterator = cells;
private Cell current = null;