You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ch...@apache.org on 2017/08/17 10:32:20 UTC
hbase git commit: HBASE-17617 Backport HBASE-16731 (Inconsistent
results from the Get/Scan if we use the empty FilterList) to branch-1
Repository: hbase
Updated Branches:
refs/heads/branch-1 6a1a97422 -> deeda60c6
HBASE-17617 Backport HBASE-16731 (Inconsistent results from the Get/Scan if we use the empty FilterList) to branch-1
Signed-off-by: Chia-Ping Tsai <ch...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/deeda60c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/deeda60c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/deeda60c
Branch: refs/heads/branch-1
Commit: deeda60c69f564e3a54d85380deace5aebfd6d94
Parents: 6a1a974
Author: Pankaj Kumar <pa...@huawei.com>
Authored: Thu Aug 17 15:58:17 2017 +0800
Committer: Chia-Ping Tsai <ch...@gmail.com>
Committed: Thu Aug 17 18:31:57 2017 +0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Get.java | 5 +
.../org/apache/hadoop/hbase/client/Query.java | 35 ++++
.../org/apache/hadoop/hbase/client/Scan.java | 36 +---
.../hadoop/hbase/protobuf/ProtobufUtil.java | 9 +-
.../apache/hadoop/hbase/client/TestScan.java | 32 ++++
.../hbase/protobuf/generated/ClientProtos.java | 185 +++++++++++++++----
hbase-protocol/src/main/protobuf/Client.proto | 1 +
.../hadoop/hbase/regionserver/HRegion.java | 3 +
.../hadoop/hbase/client/TestFromClientSide.java | 44 +++++
.../hadoop/hbase/protobuf/TestProtobufUtil.java | 1 +
10 files changed, 285 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java
index 3286d57..f8c34bd 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java
@@ -110,6 +110,7 @@ public class Get extends Query
this.storeOffset = get.getRowOffsetPerColumnFamily();
this.tr = get.getTimeRange();
this.checkExistenceOnly = get.isCheckExistenceOnly();
+ this.loadColumnFamiliesOnDemand = get.getLoadColumnFamiliesOnDemandValue();
this.closestRowBefore = get.isClosestRowBefore();
Map<byte[], NavigableSet<byte[]>> fams = get.getFamilyMap();
for (Map.Entry<byte[],NavigableSet<byte[]>> entry : fams.entrySet()) {
@@ -242,6 +243,10 @@ public class Get extends Query
return this;
}
+ public Get setLoadColumnFamiliesOnDemand(boolean value) {
+ return (Get) super.setLoadColumnFamiliesOnDemand(value);
+ }
+
/**
* Set the maximum number of values to return per row per Column Family
* @param limit the maximum number of values returned / row / CF
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
index 99d5a6a..7d3b628 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
@@ -42,6 +42,7 @@ public abstract class Query extends OperationWithAttributes {
protected int targetReplicaId = -1;
protected Consistency consistency = Consistency.STRONG;
protected Map<byte[], TimeRange> colFamTimeRangeMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
+ protected Boolean loadColumnFamiliesOnDemand = null;
/**
* @return Filter
@@ -177,6 +178,40 @@ public abstract class Query extends OperationWithAttributes {
IsolationLevel.fromBytes(attr);
}
+ /**
+ * Set the value indicating whether loading CFs on demand should be allowed (cluster
+ * default is false). On-demand CF loading doesn't load column families until necessary, e.g.
+ * if you filter on one column, the other column family data will be loaded only for the rows
+ * that are included in result, not all rows like in normal case.
+ * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true,
+ * this can deliver huge perf gains when there's a cf with lots of data; however, it can
+ * also lead to some inconsistent results, as follows:
+ * - if someone does a concurrent update to both column families in question you may get a row
+ * that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } }
+ * someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan
+ * filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 },
+ * { video => "my dog" } }.
+ * - if there's a concurrent split and you have more than 2 column families, some rows may be
+ * missing some column families.
+ */
+ public Query setLoadColumnFamiliesOnDemand(boolean value) {
+ this.loadColumnFamiliesOnDemand = value;
+ return this;
+ }
+
+ /**
+ * Get the raw loadColumnFamiliesOnDemand setting; if it's not set, can be null.
+ */
+ public Boolean getLoadColumnFamiliesOnDemandValue() {
+ return this.loadColumnFamiliesOnDemand;
+ }
+
+ /**
+ * Get the logical value indicating whether on-demand CF loading should be allowed.
+ */
+ public boolean doLoadColumnFamiliesOnDemand() {
+ return (this.loadColumnFamiliesOnDemand != null) && this.loadColumnFamiliesOnDemand;
+ }
/**
* Get versions of columns only within the specified timestamp range,
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
index 1fbf23e..1340602 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
@@ -148,7 +148,6 @@ public class Scan extends Query {
private TimeRange tr = new TimeRange();
private Map<byte [], NavigableSet<byte []>> familyMap =
new TreeMap<byte [], NavigableSet<byte []>>(Bytes.BYTES_COMPARATOR);
- private Boolean loadColumnFamiliesOnDemand = null;
/**
* Set it true for small scan to get better performance
@@ -300,6 +299,7 @@ public class Scan extends Query {
this.getScan = true;
this.consistency = get.getConsistency();
this.setIsolationLevel(get.getIsolationLevel());
+ this.loadColumnFamiliesOnDemand = get.getLoadColumnFamiliesOnDemandValue();
for (Map.Entry<String, byte[]> attr : get.getAttributesMap().entrySet()) {
setAttribute(attr.getKey(), attr.getValue());
}
@@ -871,40 +871,8 @@ public class Scan extends Query {
return allowPartialResults;
}
- /**
- * Set the value indicating whether loading CFs on demand should be allowed (cluster
- * default is false). On-demand CF loading doesn't load column families until necessary, e.g.
- * if you filter on one column, the other column family data will be loaded only for the rows
- * that are included in result, not all rows like in normal case.
- * With column-specific filters, like SingleColumnValueFilter w/filterIfMissing == true,
- * this can deliver huge perf gains when there's a cf with lots of data; however, it can
- * also lead to some inconsistent results, as follows:
- * - if someone does a concurrent update to both column families in question you may get a row
- * that never existed, e.g. for { rowKey = 5, { cat_videos => 1 }, { video => "my cat" } }
- * someone puts rowKey 5 with { cat_videos => 0 }, { video => "my dog" }, concurrent scan
- * filtering on "cat_videos == 1" can get { rowKey = 5, { cat_videos => 1 },
- * { video => "my dog" } }.
- * - if there's a concurrent split and you have more than 2 column families, some rows may be
- * missing some column families.
- */
public Scan setLoadColumnFamiliesOnDemand(boolean value) {
- this.loadColumnFamiliesOnDemand = value;
- return this;
- }
-
- /**
- * Get the raw loadColumnFamiliesOnDemand setting; if it's not set, can be null.
- */
- public Boolean getLoadColumnFamiliesOnDemandValue() {
- return this.loadColumnFamiliesOnDemand;
- }
-
- /**
- * Get the logical value indicating whether on-demand CF loading should be allowed.
- */
- public boolean doLoadColumnFamiliesOnDemand() {
- return (this.loadColumnFamiliesOnDemand != null)
- && this.loadColumnFamiliesOnDemand.booleanValue();
+ return (Scan) super.setLoadColumnFamiliesOnDemand(value);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
index fa44cdd..f0715da 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
@@ -532,6 +532,9 @@ public final class ProtobufUtil {
if (proto.hasConsistency()) {
get.setConsistency(toConsistency(proto.getConsistency()));
}
+ if (proto.hasLoadColumnFamiliesOnDemand()) {
+ get.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
+ }
return get;
}
@@ -980,7 +983,7 @@ public final class ProtobufUtil {
}
Boolean loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue();
if (loadColumnFamiliesOnDemand != null) {
- scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand.booleanValue());
+ scanBuilder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand);
}
scanBuilder.setMaxVersions(scan.getMaxVersions());
for (Entry<byte[], TimeRange> cftr : scan.getColumnFamilyTimeRange().entrySet()) {
@@ -1264,6 +1267,10 @@ public final class ProtobufUtil {
if (get.getConsistency() != null && get.getConsistency() != Consistency.STRONG) {
builder.setConsistency(toConsistency(get.getConsistency()));
}
+ Boolean loadColumnFamiliesOnDemand = get.getLoadColumnFamiliesOnDemandValue();
+ if (loadColumnFamiliesOnDemand != null) {
+ builder.setLoadColumnFamiliesOnDemand(loadColumnFamiliesOnDemand);
+ }
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
index 5f771cd..77d624d 100644
--- a/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
+++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestScan.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.hbase.client;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
@@ -26,6 +28,7 @@ import java.util.Arrays;
import java.util.Set;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.security.visibility.Authorizations;
@@ -57,6 +60,35 @@ public class TestScan {
}
@Test
+ public void testGetToScan() throws IOException {
+ Get get = new Get(Bytes.toBytes(1));
+ get.setCacheBlocks(true).setConsistency(Consistency.TIMELINE).setFilter(new FilterList())
+ .setId("get").setIsolationLevel(IsolationLevel.READ_COMMITTED)
+ .setLoadColumnFamiliesOnDemand(false).setMaxResultsPerColumnFamily(1000)
+ .setMaxVersions(9999).setRowOffsetPerColumnFamily(5).setTimeRange(0, 13)
+ .setAttribute("att_v0", Bytes.toBytes("att_v0"))
+ .setColumnFamilyTimeRange(Bytes.toBytes("cf"), 0, 123);
+ Scan scan = new Scan(get);
+ assertEquals(get.getCacheBlocks(), scan.getCacheBlocks());
+ assertEquals(get.getConsistency(), scan.getConsistency());
+ assertEquals(get.getFilter(), scan.getFilter());
+ assertEquals(get.getId(), scan.getId());
+ assertEquals(get.getIsolationLevel(), scan.getIsolationLevel());
+ assertEquals(get.getLoadColumnFamiliesOnDemandValue(),
+ scan.getLoadColumnFamiliesOnDemandValue());
+ assertEquals(get.getMaxResultsPerColumnFamily(), scan.getMaxResultsPerColumnFamily());
+ assertEquals(get.getMaxVersions(), scan.getMaxVersions());
+ assertEquals(get.getRowOffsetPerColumnFamily(), scan.getRowOffsetPerColumnFamily());
+ assertEquals(get.getTimeRange().getMin(), scan.getTimeRange().getMin());
+ assertEquals(get.getTimeRange().getMax(), scan.getTimeRange().getMax());
+ assertTrue(Bytes.equals(get.getAttribute("att_v0"), scan.getAttribute("att_v0")));
+ assertEquals(get.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMin(),
+ scan.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMin());
+ assertEquals(get.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMax(),
+ scan.getColumnFamilyTimeRange().get(Bytes.toBytes("cf")).getMax());
+ }
+
+ @Test
public void testScanAttributes() {
Scan scan = new Scan();
Assert.assertTrue(scan.getAttributesMap().isEmpty());
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
index 4bd3f5f..06ebc65 100644
--- a/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
+++ b/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/ClientProtos.java
@@ -1980,6 +1980,24 @@ public final class ClientProtos {
*/
org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilyTimeRangeOrBuilder getCfTimeRangeOrBuilder(
int index);
+
+ // optional bool load_column_families_on_demand = 14;
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ boolean hasLoadColumnFamiliesOnDemand();
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ boolean getLoadColumnFamiliesOnDemand();
}
/**
* Protobuf type {@code hbase.pb.Get}
@@ -2136,6 +2154,11 @@ public final class ClientProtos {
cfTimeRange_.add(input.readMessage(org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ColumnFamilyTimeRange.PARSER, extensionRegistry));
break;
}
+ case 112: {
+ bitField0_ |= 0x00000400;
+ loadColumnFamiliesOnDemand_ = input.readBool();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -2485,6 +2508,30 @@ public final class ClientProtos {
return cfTimeRange_.get(index);
}
+ // optional bool load_column_families_on_demand = 14;
+ public static final int LOAD_COLUMN_FAMILIES_ON_DEMAND_FIELD_NUMBER = 14;
+ private boolean loadColumnFamiliesOnDemand_;
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ public boolean hasLoadColumnFamiliesOnDemand() {
+ return ((bitField0_ & 0x00000400) == 0x00000400);
+ }
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ public boolean getLoadColumnFamiliesOnDemand() {
+ return loadColumnFamiliesOnDemand_;
+ }
+
private void initFields() {
row_ = com.google.protobuf.ByteString.EMPTY;
column_ = java.util.Collections.emptyList();
@@ -2499,6 +2546,7 @@ public final class ClientProtos {
closestRowBefore_ = false;
consistency_ = org.apache.hadoop.hbase.protobuf.generated.ClientProtos.Consistency.STRONG;
cfTimeRange_ = java.util.Collections.emptyList();
+ loadColumnFamiliesOnDemand_ = false;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -2579,6 +2627,9 @@ public final class ClientProtos {
for (int i = 0; i < cfTimeRange_.size(); i++) {
output.writeMessage(13, cfTimeRange_.get(i));
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ output.writeBool(14, loadColumnFamiliesOnDemand_);
+ }
getUnknownFields().writeTo(output);
}
@@ -2640,6 +2691,10 @@ public final class ClientProtos {
size += com.google.protobuf.CodedOutputStream
.computeMessageSize(13, cfTimeRange_.get(i));
}
+ if (((bitField0_ & 0x00000400) == 0x00000400)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeBoolSize(14, loadColumnFamiliesOnDemand_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -2719,6 +2774,11 @@ public final class ClientProtos {
}
result = result && getCfTimeRangeList()
.equals(other.getCfTimeRangeList());
+ result = result && (hasLoadColumnFamiliesOnDemand() == other.hasLoadColumnFamiliesOnDemand());
+ if (hasLoadColumnFamiliesOnDemand()) {
+ result = result && (getLoadColumnFamiliesOnDemand()
+ == other.getLoadColumnFamiliesOnDemand());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -2784,6 +2844,10 @@ public final class ClientProtos {
hash = (37 * hash) + CF_TIME_RANGE_FIELD_NUMBER;
hash = (53 * hash) + getCfTimeRangeList().hashCode();
}
+ if (hasLoadColumnFamiliesOnDemand()) {
+ hash = (37 * hash) + LOAD_COLUMN_FAMILIES_ON_DEMAND_FIELD_NUMBER;
+ hash = (53 * hash) + hashBoolean(getLoadColumnFamiliesOnDemand());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -2952,6 +3016,8 @@ public final class ClientProtos {
} else {
cfTimeRangeBuilder_.clear();
}
+ loadColumnFamiliesOnDemand_ = false;
+ bitField0_ = (bitField0_ & ~0x00002000);
return this;
}
@@ -3055,6 +3121,10 @@ public final class ClientProtos {
} else {
result.cfTimeRange_ = cfTimeRangeBuilder_.build();
}
+ if (((from_bitField0_ & 0x00002000) == 0x00002000)) {
+ to_bitField0_ |= 0x00000400;
+ }
+ result.loadColumnFamiliesOnDemand_ = loadColumnFamiliesOnDemand_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -3179,6 +3249,9 @@ public final class ClientProtos {
}
}
}
+ if (other.hasLoadColumnFamiliesOnDemand()) {
+ setLoadColumnFamiliesOnDemand(other.getLoadColumnFamiliesOnDemand());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -4498,6 +4571,55 @@ public final class ClientProtos {
return cfTimeRangeBuilder_;
}
+ // optional bool load_column_families_on_demand = 14;
+ private boolean loadColumnFamiliesOnDemand_ ;
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ public boolean hasLoadColumnFamiliesOnDemand() {
+ return ((bitField0_ & 0x00002000) == 0x00002000);
+ }
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ public boolean getLoadColumnFamiliesOnDemand() {
+ return loadColumnFamiliesOnDemand_;
+ }
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ public Builder setLoadColumnFamiliesOnDemand(boolean value) {
+ bitField0_ |= 0x00002000;
+ loadColumnFamiliesOnDemand_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional bool load_column_families_on_demand = 14;</code>
+ *
+ * <pre>
+ * DO NOT add defaults to load_column_families_on_demand.
+ * </pre>
+ */
+ public Builder clearLoadColumnFamiliesOnDemand() {
+ bitField0_ = (bitField0_ & ~0x00002000);
+ loadColumnFamiliesOnDemand_ = false;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:hbase.pb.Get)
}
@@ -37168,7 +37290,7 @@ public final class ClientProtos {
"o\032\017MapReduce.proto\"\037\n\016Authorizations\022\r\n\005" +
"label\030\001 \003(\t\"$\n\016CellVisibility\022\022\n\nexpress" +
"ion\030\001 \002(\t\"+\n\006Column\022\016\n\006family\030\001 \002(\014\022\021\n\tq" +
- "ualifier\030\002 \003(\014\"\271\003\n\003Get\022\013\n\003row\030\001 \002(\014\022 \n\006c" +
+ "ualifier\030\002 \003(\014\"\341\003\n\003Get\022\013\n\003row\030\001 \002(\014\022 \n\006c" +
"olumn\030\002 \003(\0132\020.hbase.pb.Column\022*\n\tattribu" +
"te\030\003 \003(\0132\027.hbase.pb.NameBytesPair\022 \n\006fil" +
"ter\030\004 \001(\0132\020.hbase.pb.Filter\022\'\n\ntime_rang" +
@@ -37179,10 +37301,11 @@ public final class ClientProtos {
"losest_row_before\030\013 \001(\010:\005false\0222\n\013consis" +
"tency\030\014 \001(\0162\025.hbase.pb.Consistency:\006STRO" +
"NG\0226\n\rcf_time_range\030\r \003(\0132\037.hbase.pb.Col" +
- "umnFamilyTimeRange\"\203\001\n\006Result\022\034\n\004cell\030\001 " +
+ "umnFamilyTimeRange\022&\n\036load_column_famili" +
+ "es_on_demand\030\016 \001(\010\"\203\001\n\006Result\022\034\n\004cell\030\001 " +
"\003(\0132\016.hbase.pb.Cell\022\035\n\025associated_cell_c" +
- "ount\030\002 \001(\005\022\016\n\006exists\030\003 \001(\010\022\024\n\005stale\030\004 \001(" +
- "\010:\005false\022\026\n\007partial\030\005 \001(\010:\005false\"S\n\nGetR",
+ "ount\030\002 \001(\005\022\016\n\006exists\030\003 \001(\010\022\024\n\005stale\030\004 \001(",
+ "\010:\005false\022\026\n\007partial\030\005 \001(\010:\005false\"S\n\nGetR" +
"equest\022)\n\006region\030\001 \002(\0132\031.hbase.pb.Region" +
"Specifier\022\032\n\003get\030\002 \002(\0132\r.hbase.pb.Get\"/\n" +
"\013GetResponse\022 \n\006result\030\001 \001(\0132\020.hbase.pb." +
@@ -37191,8 +37314,8 @@ public final class ClientProtos {
"type\030\004 \002(\0162\025.hbase.pb.CompareType\022(\n\ncom" +
"parator\030\005 \002(\0132\024.hbase.pb.Comparator\"\364\006\n\r" +
"MutationProto\022\013\n\003row\030\001 \001(\014\0229\n\013mutate_typ" +
- "e\030\002 \001(\0162$.hbase.pb.MutationProto.Mutatio" +
- "nType\0229\n\014column_value\030\003 \003(\0132#.hbase.pb.M",
+ "e\030\002 \001(\0162$.hbase.pb.MutationProto.Mutatio",
+ "nType\0229\n\014column_value\030\003 \003(\0132#.hbase.pb.M" +
"utationProto.ColumnValue\022\021\n\ttimestamp\030\004 " +
"\001(\004\022*\n\tattribute\030\005 \003(\0132\027.hbase.pb.NameBy" +
"tesPair\022C\n\ndurability\030\006 \001(\0162\".hbase.pb.M" +
@@ -37201,8 +37324,8 @@ public final class ClientProtos {
"\025associated_cell_count\030\010 \001(\005\022\r\n\005nonce\030\t " +
"\001(\004\032\371\001\n\013ColumnValue\022\016\n\006family\030\001 \002(\014\022K\n\017q" +
"ualifier_value\030\002 \003(\01322.hbase.pb.Mutation" +
- "Proto.ColumnValue.QualifierValue\032\214\001\n\016Qua" +
- "lifierValue\022\021\n\tqualifier\030\001 \001(\014\022\r\n\005value\030",
+ "Proto.ColumnValue.QualifierValue\032\214\001\n\016Qua",
+ "lifierValue\022\021\n\tqualifier\030\001 \001(\014\022\r\n\005value\030" +
"\002 \001(\014\022\021\n\ttimestamp\030\003 \001(\004\0227\n\013delete_type\030" +
"\004 \001(\0162\".hbase.pb.MutationProto.DeleteTyp" +
"e\022\014\n\004tags\030\005 \001(\014\"W\n\nDurability\022\017\n\013USE_DEF" +
@@ -37211,8 +37334,8 @@ public final class ClientProtos {
"\022\n\n\006APPEND\020\000\022\r\n\tINCREMENT\020\001\022\007\n\003PUT\020\002\022\n\n\006" +
"DELETE\020\003\"p\n\nDeleteType\022\026\n\022DELETE_ONE_VER" +
"SION\020\000\022\034\n\030DELETE_MULTIPLE_VERSIONS\020\001\022\021\n\r" +
- "DELETE_FAMILY\020\002\022\031\n\025DELETE_FAMILY_VERSION" +
- "\020\003\"\242\001\n\rMutateRequest\022)\n\006region\030\001 \002(\0132\031.h",
+ "DELETE_FAMILY\020\002\022\031\n\025DELETE_FAMILY_VERSION",
+ "\020\003\"\242\001\n\rMutateRequest\022)\n\006region\030\001 \002(\0132\031.h" +
"base.pb.RegionSpecifier\022)\n\010mutation\030\002 \002(" +
"\0132\027.hbase.pb.MutationProto\022&\n\tcondition\030" +
"\003 \001(\0132\023.hbase.pb.Condition\022\023\n\013nonce_grou" +
@@ -37221,8 +37344,8 @@ public final class ClientProtos {
"\006\n\004Scan\022 \n\006column\030\001 \003(\0132\020.hbase.pb.Colum" +
"n\022*\n\tattribute\030\002 \003(\0132\027.hbase.pb.NameByte" +
"sPair\022\021\n\tstart_row\030\003 \001(\014\022\020\n\010stop_row\030\004 \001" +
- "(\014\022 \n\006filter\030\005 \001(\0132\020.hbase.pb.Filter\022\'\n\n" +
- "time_range\030\006 \001(\0132\023.hbase.pb.TimeRange\022\027\n",
+ "(\014\022 \n\006filter\030\005 \001(\0132\020.hbase.pb.Filter\022\'\n\n",
+ "time_range\030\006 \001(\0132\023.hbase.pb.TimeRange\022\027\n" +
"\014max_versions\030\007 \001(\r:\0011\022\032\n\014cache_blocks\030\010" +
" \001(\010:\004true\022\022\n\nbatch_size\030\t \001(\r\022\027\n\017max_re" +
"sult_size\030\n \001(\004\022\023\n\013store_limit\030\013 \001(\r\022\024\n\014" +
@@ -37231,8 +37354,8 @@ public final class ClientProtos {
"ersed\030\017 \001(\010:\005false\0222\n\013consistency\030\020 \001(\0162" +
"\025.hbase.pb.Consistency:\006STRONG\022\017\n\007cachin" +
"g\030\021 \001(\r\022\035\n\025allow_partial_results\030\022 \001(\010\0226" +
- "\n\rcf_time_range\030\023 \003(\0132\037.hbase.pb.ColumnF" +
- "amilyTimeRange\022\032\n\017mvcc_read_point\030\024 \001(\004:",
+ "\n\rcf_time_range\030\023 \003(\0132\037.hbase.pb.ColumnF",
+ "amilyTimeRange\022\032\n\017mvcc_read_point\030\024 \001(\004:" +
"\0010\022\037\n\021include_start_row\030\025 \001(\010:\004true\022\037\n\020i" +
"nclude_stop_row\030\026 \001(\010:\005false\0222\n\010readType" +
"\030\027 \001(\0162\027.hbase.pb.Scan.ReadType:\007DEFAULT" +
@@ -37241,8 +37364,8 @@ public final class ClientProtos {
"D\020\002\"\300\002\n\013ScanRequest\022)\n\006region\030\001 \001(\0132\031.hb" +
"ase.pb.RegionSpecifier\022\034\n\004scan\030\002 \001(\0132\016.h" +
"base.pb.Scan\022\022\n\nscanner_id\030\003 \001(\004\022\026\n\016numb" +
- "er_of_rows\030\004 \001(\r\022\025\n\rclose_scanner\030\005 \001(\010\022" +
- "\025\n\rnext_call_seq\030\006 \001(\004\022\037\n\027client_handles",
+ "er_of_rows\030\004 \001(\r\022\025\n\rclose_scanner\030\005 \001(\010\022",
+ "\025\n\rnext_call_seq\030\006 \001(\004\022\037\n\027client_handles" +
"_partials\030\007 \001(\010\022!\n\031client_handles_heartb" +
"eats\030\010 \001(\010\022\032\n\022track_scan_metrics\030\t \001(\010\022\024" +
"\n\005renew\030\n \001(\010:\005false\022\030\n\rlimit_of_rows\030\013 " +
@@ -37251,8 +37374,8 @@ public final class ClientProtos {
"ner_id\030\002 \001(\004\022\024\n\014more_results\030\003 \001(\010\022\013\n\003tt" +
"l\030\004 \001(\r\022!\n\007results\030\005 \003(\0132\020.hbase.pb.Resu" +
"lt\022\r\n\005stale\030\006 \001(\010\022\037\n\027partial_flag_per_re" +
- "sult\030\007 \003(\010\022\036\n\026more_results_in_region\030\010 \001" +
- "(\010\022\031\n\021heartbeat_message\030\t \001(\010\022+\n\014scan_me",
+ "sult\030\007 \003(\010\022\036\n\026more_results_in_region\030\010 \001",
+ "(\010\022\031\n\021heartbeat_message\030\t \001(\010\022+\n\014scan_me" +
"trics\030\n \001(\0132\025.hbase.pb.ScanMetrics\022\032\n\017mv" +
"cc_read_point\030\013 \001(\004:\0010\022 \n\006cursor\030\014 \001(\0132\020" +
".hbase.pb.Cursor\"\305\001\n\024BulkLoadHFileReques" +
@@ -37261,8 +37384,8 @@ public final class ClientProtos {
"kLoadHFileRequest.FamilyPath\022\026\n\016assign_s" +
"eq_num\030\003 \001(\010\032*\n\nFamilyPath\022\016\n\006family\030\001 \002" +
"(\014\022\014\n\004path\030\002 \002(\t\"\'\n\025BulkLoadHFileRespons" +
- "e\022\016\n\006loaded\030\001 \002(\010\"a\n\026CoprocessorServiceC" +
- "all\022\013\n\003row\030\001 \002(\014\022\024\n\014service_name\030\002 \002(\t\022\023",
+ "e\022\016\n\006loaded\030\001 \002(\010\"a\n\026CoprocessorServiceC",
+ "all\022\013\n\003row\030\001 \002(\014\022\024\n\014service_name\030\002 \002(\t\022\023" +
"\n\013method_name\030\003 \002(\t\022\017\n\007request\030\004 \002(\014\"B\n\030" +
"CoprocessorServiceResult\022&\n\005value\030\001 \001(\0132" +
"\027.hbase.pb.NameBytesPair\"v\n\031CoprocessorS" +
@@ -37271,8 +37394,8 @@ public final class ClientProtos {
"b.CoprocessorServiceCall\"o\n\032CoprocessorS" +
"erviceResponse\022)\n\006region\030\001 \002(\0132\031.hbase.p" +
"b.RegionSpecifier\022&\n\005value\030\002 \002(\0132\027.hbase" +
- ".pb.NameBytesPair\"\226\001\n\006Action\022\r\n\005index\030\001 " +
- "\001(\r\022)\n\010mutation\030\002 \001(\0132\027.hbase.pb.Mutatio",
+ ".pb.NameBytesPair\"\226\001\n\006Action\022\r\n\005index\030\001 ",
+ "\001(\r\022)\n\010mutation\030\002 \001(\0132\027.hbase.pb.Mutatio" +
"nProto\022\032\n\003get\030\003 \001(\0132\r.hbase.pb.Get\0226\n\014se" +
"rvice_call\030\004 \001(\0132 .hbase.pb.CoprocessorS" +
"erviceCall\"k\n\014RegionAction\022)\n\006region\030\001 \002" +
@@ -37281,8 +37404,8 @@ public final class ClientProtos {
"c\n\017RegionLoadStats\022\027\n\014memstoreLoad\030\001 \001(\005" +
":\0010\022\030\n\rheapOccupancy\030\002 \001(\005:\0010\022\035\n\022compact" +
"ionPressure\030\003 \001(\005:\0010\"j\n\024MultiRegionLoadS" +
- "tats\022)\n\006region\030\001 \003(\0132\031.hbase.pb.RegionSp" +
- "ecifier\022\'\n\004stat\030\002 \003(\0132\031.hbase.pb.RegionL",
+ "tats\022)\n\006region\030\001 \003(\0132\031.hbase.pb.RegionSp",
+ "ecifier\022\'\n\004stat\030\002 \003(\0132\031.hbase.pb.RegionL" +
"oadStats\"\336\001\n\021ResultOrException\022\r\n\005index\030" +
"\001 \001(\r\022 \n\006result\030\002 \001(\0132\020.hbase.pb.Result\022" +
"*\n\texception\030\003 \001(\0132\027.hbase.pb.NameBytesP" +
@@ -37291,8 +37414,8 @@ public final class ClientProtos {
"(\0132\031.hbase.pb.RegionLoadStatsB\002\030\001\"x\n\022Reg" +
"ionActionResult\0226\n\021resultOrException\030\001 \003" +
"(\0132\033.hbase.pb.ResultOrException\022*\n\texcep" +
- "tion\030\002 \001(\0132\027.hbase.pb.NameBytesPair\"x\n\014M" +
- "ultiRequest\022,\n\014regionAction\030\001 \003(\0132\026.hbas",
+ "tion\030\002 \001(\0132\027.hbase.pb.NameBytesPair\"x\n\014M",
+ "ultiRequest\022,\n\014regionAction\030\001 \003(\0132\026.hbas" +
"e.pb.RegionAction\022\022\n\nnonceGroup\030\002 \001(\004\022&\n" +
"\tcondition\030\003 \001(\0132\023.hbase.pb.Condition\"\226\001" +
"\n\rMultiResponse\0228\n\022regionActionResult\030\001 " +
@@ -37301,8 +37424,8 @@ public final class ClientProtos {
".hbase.pb.MultiRegionLoadStats*\'\n\013Consis" +
"tency\022\n\n\006STRONG\020\000\022\014\n\010TIMELINE\020\0012\203\004\n\rClie" +
"ntService\0222\n\003Get\022\024.hbase.pb.GetRequest\032\025" +
- ".hbase.pb.GetResponse\022;\n\006Mutate\022\027.hbase." +
- "pb.MutateRequest\032\030.hbase.pb.MutateRespon",
+ ".hbase.pb.GetResponse\022;\n\006Mutate\022\027.hbase.",
+ "pb.MutateRequest\032\030.hbase.pb.MutateRespon" +
"se\0225\n\004Scan\022\025.hbase.pb.ScanRequest\032\026.hbas" +
"e.pb.ScanResponse\022P\n\rBulkLoadHFile\022\036.hba" +
"se.pb.BulkLoadHFileRequest\032\037.hbase.pb.Bu" +
@@ -37311,8 +37434,8 @@ public final class ClientProtos {
"pb.CoprocessorServiceResponse\022d\n\027ExecReg" +
"ionServerService\022#.hbase.pb.CoprocessorS" +
"erviceRequest\032$.hbase.pb.CoprocessorServ" +
- "iceResponse\0228\n\005Multi\022\026.hbase.pb.MultiReq" +
- "uest\032\027.hbase.pb.MultiResponseBB\n*org.apa",
+ "iceResponse\0228\n\005Multi\022\026.hbase.pb.MultiReq",
+ "uest\032\027.hbase.pb.MultiResponseBB\n*org.apa" +
"che.hadoop.hbase.protobuf.generatedB\014Cli" +
"entProtosH\001\210\001\001\240\001\001"
};
@@ -37344,7 +37467,7 @@ public final class ClientProtos {
internal_static_hbase_pb_Get_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_hbase_pb_Get_descriptor,
- new java.lang.String[] { "Row", "Column", "Attribute", "Filter", "TimeRange", "MaxVersions", "CacheBlocks", "StoreLimit", "StoreOffset", "ExistenceOnly", "ClosestRowBefore", "Consistency", "CfTimeRange", });
+ new java.lang.String[] { "Row", "Column", "Attribute", "Filter", "TimeRange", "MaxVersions", "CacheBlocks", "StoreLimit", "StoreOffset", "ExistenceOnly", "ClosestRowBefore", "Consistency", "CfTimeRange", "LoadColumnFamiliesOnDemand", });
internal_static_hbase_pb_Result_descriptor =
getDescriptor().getMessageTypes().get(4);
internal_static_hbase_pb_Result_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-protocol/src/main/protobuf/Client.proto
----------------------------------------------------------------------
diff --git a/hbase-protocol/src/main/protobuf/Client.proto b/hbase-protocol/src/main/protobuf/Client.proto
index 9a68031..83f4e25 100644
--- a/hbase-protocol/src/main/protobuf/Client.proto
+++ b/hbase-protocol/src/main/protobuf/Client.proto
@@ -88,6 +88,7 @@ message Get {
optional Consistency consistency = 12 [default = STRONG];
repeated ColumnFamilyTimeRange cf_time_range = 13;
+ optional bool load_column_families_on_demand = 14; /* DO NOT add defaults to load_column_families_on_demand. */
}
message Result {
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
index 71c7892..de2461b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
@@ -7270,6 +7270,9 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi
scan = new Scan(get);
}
+ if (scan.getLoadColumnFamiliesOnDemandValue() == null) {
+ scan.setLoadColumnFamiliesOnDemand(isLoadingCfsOnDemandDefault());
+ }
RegionScanner scanner = null;
try {
scanner = getScanner(scan, null, nonceGroup, nonce);
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
index e21e51a..de6ecab 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
@@ -5730,6 +5730,50 @@ public class TestFromClientSide {
}
@Test
+ public void testEmptyFilterList() throws Exception {
+ // Test Initialization.
+ TableName TABLE = TableName.valueOf("testEmptyFilterList");
+ Table table = TEST_UTIL.createTable(TABLE, FAMILY);
+
+ // Insert one row each region
+ Put put = new Put(Bytes.toBytes("row"));
+ put.addColumn(FAMILY, QUALIFIER, VALUE);
+ table.put(put);
+
+ List<Result> scanResults = new LinkedList<>();
+ Scan scan = new Scan();
+ scan.setFilter(new FilterList());
+ try (ResultScanner scanner = table.getScanner(scan)) {
+ for (Result r : scanner) {
+ scanResults.add(r);
+ }
+ }
+
+ Get g = new Get(Bytes.toBytes("row"));
+ g.setFilter(new FilterList());
+ Result getResult = table.get(g);
+ if (scanResults.isEmpty()) {
+ assertTrue(getResult.isEmpty());
+ } else if (scanResults.size() == 1) {
+ Result scanResult = scanResults.get(0);
+ assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
+ for (int i = 0; i != scanResult.rawCells().length; ++i) {
+ Cell scanCell = scanResult.rawCells()[i];
+ Cell getCell = getResult.rawCells()[i];
+ assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), CellUtil.cloneRow(getCell)));
+ assertEquals(0,
+ Bytes.compareTo(CellUtil.cloneFamily(scanCell), CellUtil.cloneFamily(getCell)));
+ assertEquals(0,
+ Bytes.compareTo(CellUtil.cloneQualifier(scanCell), CellUtil.cloneQualifier(getCell)));
+ assertEquals(0,
+ Bytes.compareTo(CellUtil.cloneValue(scanCell), CellUtil.cloneValue(getCell)));
+ }
+ } else {
+ fail("The result retrieved from SCAN and Get should be same");
+ }
+ }
+
+ @Test
public void testSmallScan() throws Exception {
// Test Initialization.
TableName TABLE = TableName.valueOf("testSmallScan");
http://git-wip-us.apache.org/repos/asf/hbase/blob/deeda60c/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
index ddbbb74..7cff7fd 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java
@@ -82,6 +82,7 @@ public class TestProtobufUtil {
columnBuilder.clear();
columnBuilder.setFamily(ByteString.copyFromUtf8("f2"));
getBuilder.addColumn(columnBuilder.build());
+ getBuilder.setLoadColumnFamiliesOnDemand(true);
ClientProtos.Get proto = getBuilder.build();
// default fields