You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by yc...@apache.org on 2018/01/09 16:20:35 UTC
hive git commit: HIVE-11609: Capability to add a filter to hbase scan
via composite key doesn't work (Barna Zsombor Klara, Swarnim Kulkarni,
reviewed by Aihua Xue and Yongzhi Chen)
Repository: hive
Updated Branches:
refs/heads/master 8412748a7 -> c253955e1
HIVE-11609: Capability to add a filter to hbase scan via composite key doesn't work (Barna Zsombor Klara, Swarnim Kulkarni, reviewed by Aihua Xue and Yongzhi Chen)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c253955e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c253955e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c253955e
Branch: refs/heads/master
Commit: c253955e1a786ab93e569de6520f021f6359d1fe
Parents: 8412748
Author: Yongzhi Chen <yc...@apache.org>
Authored: Tue Jan 9 11:08:25 2018 -0500
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Tue Jan 9 11:08:25 2018 -0500
----------------------------------------------------------------------
.../hadoop/hive/hbase/HBaseScanRange.java | 28 ++-
.../hive/hbase/HiveHBaseInputFormatUtil.java | 186 +++++++++++++++++++
.../hive/hbase/HiveHBaseTableInputFormat.java | 155 +---------------
.../hive/hbase/SampleHBaseKeyFactory3.java | 20 +-
.../results/positive/hbase_custom_key3.q.out | 1 -
.../hadoop/hive/ql/plan/TableScanDesc.java | 2 +-
6 files changed, 218 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c253955e/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseScanRange.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseScanRange.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseScanRange.java
index 8b64321..7b64a03 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseScanRange.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseScanRange.java
@@ -18,17 +18,17 @@
package org.apache.hadoop.hive.hbase;
+import java.io.Serializable;
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.io.BytesWritable;
-import java.io.Serializable;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.List;
-
public class HBaseScanRange implements Serializable {
private byte[] startRow;
@@ -59,12 +59,20 @@ public class HBaseScanRange implements Serializable {
}
public void setup(Scan scan, Configuration conf) throws Exception {
- if (startRow != null) {
- scan.setStartRow(startRow);
- }
- if (stopRow != null) {
- scan.setStopRow(stopRow);
+ setup(scan, conf, false);
+ }
+
+ public void setup(Scan scan, Configuration conf, boolean filterOnly) throws Exception {
+ if (!filterOnly) {
+ // Set the start and stop rows only if asked to
+ if (startRow != null) {
+ scan.setStartRow(startRow);
+ }
+ if (stopRow != null) {
+ scan.setStopRow(stopRow);
+ }
}
+
if (filterDescs.isEmpty()) {
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c253955e/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
index d42b7ec..e60fd7d 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseInputFormatUtil.java
@@ -24,14 +24,38 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
+import org.apache.hadoop.hive.ql.exec.ExprNodeConstantEvaluator;
+import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.TableScanDesc;
+import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
/**
@@ -39,6 +63,8 @@ import org.apache.hadoop.mapred.JobConf;
*/
class HiveHBaseInputFormatUtil {
+ private static final Log LOG = LogFactory.getLog(HiveHBaseInputFormatUtil.class);
+
/**
* Parse {@code jobConf} to create a {@link Scan} instance.
*/
@@ -123,6 +149,13 @@ class HiveHBaseInputFormatUtil {
if (scanBatch != null) {
scan.setBatch(Integer.parseInt(scanBatch));
}
+
+ String filterObjectSerialized = jobConf.get(TableScanDesc.FILTER_OBJECT_CONF_STR);
+
+ if (filterObjectSerialized != null) {
+ setupScanRange(scan, filterObjectSerialized, jobConf, true);
+ }
+
return scan;
}
@@ -164,4 +197,157 @@ class HiveHBaseInputFormatUtil {
}
return result;
}
+
+ static void setupScanRange(Scan scan, String filterObjectSerialized, JobConf jobConf,
+ boolean filterOnly) throws IOException {
+ HBaseScanRange range =
+ SerializationUtilities.deserializeObject(filterObjectSerialized,
+ HBaseScanRange.class);
+ try {
+ range.setup(scan, jobConf, filterOnly);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ }
+
+ static void setupKeyRange(Scan scan, List<IndexSearchCondition> conditions, boolean isBinary)
+ throws IOException {
+ // Convert the search condition into a restriction on the HBase scan
+ byte[] startRow = HConstants.EMPTY_START_ROW, stopRow = HConstants.EMPTY_END_ROW;
+ for (IndexSearchCondition sc : conditions) {
+
+ ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc());
+ PrimitiveObjectInspector objInspector;
+ Object writable;
+
+ try {
+ objInspector = (PrimitiveObjectInspector) eval.initialize(null);
+ writable = eval.evaluate(null);
+ } catch (ClassCastException cce) {
+ throw new IOException("Currently only primitve types are supported. Found: "
+ + sc.getConstantDesc().getTypeString());
+ } catch (HiveException e) {
+ throw new IOException(e);
+ }
+
+ byte[] constantVal = getConstantVal(writable, objInspector, isBinary);
+ String comparisonOp = sc.getComparisonOp();
+
+ if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)) {
+ startRow = constantVal;
+ stopRow = getNextBA(constantVal);
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan".equals(comparisonOp)) {
+ stopRow = constantVal;
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan"
+ .equals(comparisonOp)) {
+ startRow = constantVal;
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"
+ .equals(comparisonOp)) {
+ startRow = getNextBA(constantVal);
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan"
+ .equals(comparisonOp)) {
+ stopRow = getNextBA(constantVal);
+ } else {
+ throw new IOException(comparisonOp + " is not a supported comparison operator");
+ }
+ }
+ scan.setStartRow(startRow);
+ scan.setStopRow(stopRow);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(Bytes.toStringBinary(startRow) + " ~ " + Bytes.toStringBinary(stopRow));
+ }
+ }
+
+ static void setupTimeRange(Scan scan, List<IndexSearchCondition> conditions) throws IOException {
+ long start = 0;
+ long end = Long.MAX_VALUE;
+ for (IndexSearchCondition sc : conditions) {
+ long timestamp = getTimestampVal(sc);
+ String comparisonOp = sc.getComparisonOp();
+ if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)) {
+ start = timestamp;
+ end = timestamp + 1;
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan".equals(comparisonOp)) {
+ end = timestamp;
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan"
+ .equals(comparisonOp)) {
+ start = timestamp;
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"
+ .equals(comparisonOp)) {
+ start = timestamp + 1;
+ } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan"
+ .equals(comparisonOp)) {
+ end = timestamp + 1;
+ } else {
+ throw new IOException(comparisonOp + " is not a supported comparison operator");
+ }
+ }
+ scan.setTimeRange(start, end);
+ }
+
+ static long getTimestampVal(IndexSearchCondition sc) throws IOException {
+ long timestamp;
+ try {
+ ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc());
+ ObjectInspector inspector = eval.initialize(null);
+ Object value = eval.evaluate(null);
+ if (inspector instanceof LongObjectInspector) {
+ timestamp = ((LongObjectInspector) inspector).get(value);
+ } else {
+ PrimitiveObjectInspector primitive = (PrimitiveObjectInspector) inspector;
+ timestamp = PrimitiveObjectInspectorUtils.getTimestamp(value, primitive).getTime();
+ }
+ } catch (HiveException e) {
+ throw new IOException(e);
+ }
+ return timestamp;
+ }
+
+ static byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi, boolean isKeyBinary)
+ throws IOException {
+
+ if (!isKeyBinary) {
+ // Key is stored in text format. Get bytes representation of constant also of
+ // text format.
+ byte[] startRow;
+ ByteStream.Output serializeStream = new ByteStream.Output();
+ LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null);
+ startRow = new byte[serializeStream.getLength()];
+ System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength());
+ return startRow;
+ }
+
+ PrimitiveCategory pc = poi.getPrimitiveCategory();
+ switch (poi.getPrimitiveCategory()) {
+ case INT:
+ return Bytes.toBytes(((IntWritable) writable).get());
+ case BOOLEAN:
+ return Bytes.toBytes(((BooleanWritable) writable).get());
+ case LONG:
+ return Bytes.toBytes(((LongWritable) writable).get());
+ case FLOAT:
+ return Bytes.toBytes(((FloatWritable) writable).get());
+ case DOUBLE:
+ return Bytes.toBytes(((DoubleWritable) writable).get());
+ case SHORT:
+ return Bytes.toBytes(((ShortWritable) writable).get());
+ case STRING:
+ return Bytes.toBytes(((Text) writable).toString());
+ case BYTE:
+ return Bytes.toBytes(((ByteWritable) writable).get());
+
+ default:
+ throw new IOException("Type not supported " + pc);
+ }
+ }
+
+ static byte[] getNextBA(byte[] current) {
+ // startRow is inclusive while stopRow is exclusive,
+ // this util method returns very next bytearray which will occur after the current one
+ // by padding current one with a trailing 0 byte.
+ byte[] next = new byte[current.length + 1];
+ System.arraycopy(current, 0, next, 0, current.length);
+ return next;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c253955e/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
index b6a7f2d..9c5ed0b 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HiveHBaseTableInputFormat.java
@@ -203,13 +203,7 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
Scan scan = new Scan();
String filterObjectSerialized = jobConf.get(TableScanDesc.FILTER_OBJECT_CONF_STR);
if (filterObjectSerialized != null) {
- HBaseScanRange range = SerializationUtilities.deserializeObject(filterObjectSerialized,
- HBaseScanRange.class);
- try {
- range.setup(scan, jobConf);
- } catch (Exception e) {
- throw new IOException(e);
- }
+ HiveHBaseInputFormatUtil.setupScanRange(scan, filterObjectSerialized, jobConf, false);
return scan;
}
@@ -251,158 +245,15 @@ public class HiveHBaseTableInputFormat extends TableInputFormatBase
Map<String, List<IndexSearchCondition>> split = HiveHBaseInputFormatUtil.decompose(conditions);
List<IndexSearchCondition> keyConditions = split.get(keyColName);
if (keyConditions != null && !keyConditions.isEmpty()) {
- setupKeyRange(scan, keyConditions, isKeyBinary);
+ HiveHBaseInputFormatUtil.setupKeyRange(scan, keyConditions, isKeyBinary);
}
List<IndexSearchCondition> tsConditions = split.get(tsColName);
if (tsConditions != null && !tsConditions.isEmpty()) {
- setupTimeRange(scan, tsConditions);
+ HiveHBaseInputFormatUtil.setupTimeRange(scan, tsConditions);
}
return scan;
}
- private void setupKeyRange(Scan scan, List<IndexSearchCondition> conditions, boolean isBinary)
- throws IOException {
- // Convert the search condition into a restriction on the HBase scan
- byte [] startRow = HConstants.EMPTY_START_ROW, stopRow = HConstants.EMPTY_END_ROW;
- for (IndexSearchCondition sc : conditions) {
-
- ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc());
- PrimitiveObjectInspector objInspector;
- Object writable;
-
- try {
- objInspector = (PrimitiveObjectInspector)eval.initialize(null);
- writable = eval.evaluate(null);
- } catch (ClassCastException cce) {
- throw new IOException("Currently only primitive types are supported. Found: " +
- sc.getConstantDesc().getTypeString());
- } catch (HiveException e) {
- throw new IOException(e);
- }
-
- byte[] constantVal = getConstantVal(writable, objInspector, isBinary);
- String comparisonOp = sc.getComparisonOp();
-
- if("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)){
- startRow = constantVal;
- stopRow = getNextBA(constantVal);
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan".equals(comparisonOp)){
- stopRow = constantVal;
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan"
- .equals(comparisonOp)) {
- startRow = constantVal;
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"
- .equals(comparisonOp)){
- startRow = getNextBA(constantVal);
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan"
- .equals(comparisonOp)){
- stopRow = getNextBA(constantVal);
- } else {
- throw new IOException(comparisonOp + " is not a supported comparison operator");
- }
- }
- scan.setStartRow(startRow);
- scan.setStopRow(stopRow);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(Bytes.toStringBinary(startRow) + " ~ " + Bytes.toStringBinary(stopRow));
- }
- }
-
- private void setupTimeRange(Scan scan, List<IndexSearchCondition> conditions)
- throws IOException {
- long start = 0;
- long end = Long.MAX_VALUE;
- for (IndexSearchCondition sc : conditions) {
- long timestamp = getTimestampVal(sc);
- String comparisonOp = sc.getComparisonOp();
- if("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqual".equals(comparisonOp)){
- start = timestamp;
- end = timestamp + 1;
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPLessThan".equals(comparisonOp)){
- end = timestamp;
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrGreaterThan"
- .equals(comparisonOp)) {
- start = timestamp;
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPGreaterThan"
- .equals(comparisonOp)){
- start = timestamp + 1;
- } else if ("org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPEqualOrLessThan"
- .equals(comparisonOp)){
- end = timestamp + 1;
- } else {
- throw new IOException(comparisonOp + " is not a supported comparison operator");
- }
- }
- scan.setTimeRange(start, end);
- }
-
- private long getTimestampVal(IndexSearchCondition sc) throws IOException {
- long timestamp;
- try {
- ExprNodeConstantEvaluator eval = new ExprNodeConstantEvaluator(sc.getConstantDesc());
- ObjectInspector inspector = eval.initialize(null);
- Object value = eval.evaluate(null);
- if (inspector instanceof LongObjectInspector) {
- timestamp = ((LongObjectInspector)inspector).get(value);
- } else {
- PrimitiveObjectInspector primitive = (PrimitiveObjectInspector) inspector;
- timestamp = PrimitiveObjectInspectorUtils.getTimestamp(value, primitive).getTime();
- }
- } catch (HiveException e) {
- throw new IOException(e);
- }
- return timestamp;
- }
-
- private byte[] getConstantVal(Object writable, PrimitiveObjectInspector poi,
- boolean isKeyBinary) throws IOException{
-
- if (!isKeyBinary){
- // Key is stored in text format. Get bytes representation of constant also of
- // text format.
- byte[] startRow;
- ByteStream.Output serializeStream = new ByteStream.Output();
- LazyUtils.writePrimitiveUTF8(serializeStream, writable, poi, false, (byte) 0, null);
- startRow = new byte[serializeStream.getLength()];
- System.arraycopy(serializeStream.getData(), 0, startRow, 0, serializeStream.getLength());
- return startRow;
- }
-
- PrimitiveCategory pc = poi.getPrimitiveCategory();
- switch (poi.getPrimitiveCategory()) {
- case INT:
- return Bytes.toBytes(((IntWritable)writable).get());
- case BOOLEAN:
- return Bytes.toBytes(((BooleanWritable)writable).get());
- case LONG:
- return Bytes.toBytes(((LongWritable)writable).get());
- case FLOAT:
- return Bytes.toBytes(((FloatWritable)writable).get());
- case DOUBLE:
- return Bytes.toBytes(((DoubleWritable)writable).get());
- case SHORT:
- return Bytes.toBytes(((ShortWritable)writable).get());
- case STRING:
- return Bytes.toBytes(((Text)writable).toString());
- case BYTE:
- return Bytes.toBytes(((ByteWritable)writable).get());
-
- default:
- throw new IOException("Type not supported " + pc);
- }
- }
-
-
- private byte[] getNextBA(byte[] current){
- // startRow is inclusive while stopRow is exclusive,
- // this util method returns very next bytearray which will occur after the current one
- // by padding current one with a trailing 0 byte.
- byte[] next = new byte[current.length + 1];
- System.arraycopy(current, 0, next, 0, current.length);
- return next;
- }
-
/**
* Instantiates a new predicate analyzer suitable for
* determining how to push a filter down into the HBase scan,
http://git-wip-us.apache.org/repos/asf/hive/blob/c253955e/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java
index 712725f..dd86ddd 100644
--- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java
+++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/SampleHBaseKeyFactory3.java
@@ -28,6 +28,8 @@ import java.util.Map;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hive.hbase.ColumnMappings.ColumnMapping;
import org.apache.hadoop.hive.ql.index.IndexSearchCondition;
@@ -73,7 +75,7 @@ class SampleHBasePredicateDecomposer extends AbstractHBaseKeyPredicateDecomposer
}
fieldCond.add(condition);
}
- Filter filter = null;
+ List<Filter> filters = new ArrayList<Filter>();
HBaseScanRange range = new HBaseScanRange();
StructTypeInfo type = (StructTypeInfo) keyMapping.columnType;
@@ -91,23 +93,21 @@ class SampleHBasePredicateDecomposer extends AbstractHBaseKeyPredicateDecomposer
byte[] valueAsBytes = toBinary(constantVal, FIXED_LENGTH, false, false);
- if (comparisonOp.endsWith("UDFOPEqual")) {
- filter = new RowFilter(CompareOp.EQUAL, new BinaryComparator(valueAsBytes));
- } else if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) {
- filter = new RowFilter(CompareOp.GREATER_OR_EQUAL, new BinaryComparator(valueAsBytes));
+ if (comparisonOp.endsWith("UDFOPEqualOrGreaterThan")) {
+ filters.add(new RowFilter(CompareOp.GREATER_OR_EQUAL, new BinaryComparator(valueAsBytes)));
} else if (comparisonOp.endsWith("UDFOPGreaterThan")) {
- filter = new RowFilter(CompareOp.GREATER, new BinaryComparator(valueAsBytes));
+ filters.add(new RowFilter(CompareOp.GREATER, new BinaryComparator(valueAsBytes)));
} else if (comparisonOp.endsWith("UDFOPEqualOrLessThan")) {
- filter = new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator(valueAsBytes));
+ filters.add(new RowFilter(CompareOp.LESS_OR_EQUAL, new BinaryComparator(valueAsBytes)));
} else if (comparisonOp.endsWith("UDFOPLessThan")) {
- filter = new RowFilter(CompareOp.LESS, new BinaryComparator(valueAsBytes));
+ filters.add(new RowFilter(CompareOp.LESS, new BinaryComparator(valueAsBytes)));
} else {
throw new IOException(comparisonOp + " is not a supported comparison operator");
}
}
}
- if (filter != null) {
- range.addFilter(filter);
+ if (!filters.isEmpty()) {
+ range.addFilter(new FilterList(Operator.MUST_PASS_ALL, filters));
}
return range;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/c253955e/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out b/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out
index 60721cb..1b042fd 100644
--- a/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_custom_key3.q.out
@@ -65,7 +65,6 @@ STAGE PLANS:
Processor Tree:
TableScan
alias: hbase_ck_5
- filterExpr: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean)
Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: NONE
Filter Operator
predicate: ((key.col1 = '238') and (key.col2 = '1238')) (type: boolean)
http://git-wip-us.apache.org/repos/asf/hive/blob/c253955e/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
index 6e15234..ba75a14 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
@@ -74,7 +74,7 @@ public class TableScanDesc extends AbstractOperatorDesc implements IStatsGatherD
private String tmpStatsDir;
private ExprNodeGenericFuncDesc filterExpr;
- private transient Serializable filterObject;
+ private Serializable filterObject;
private String serializedFilterExpr;
private String serializedFilterObject;