You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2015/10/16 20:22:19 UTC
[4/5] hbase git commit: HBASE-14406 The dataframe datasource filter
is wrong, and will result in data loss or unexpected behavior (Ted Malaska)
http://git-wip-us.apache.org/repos/asf/hbase/blob/dae1775a/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java
----------------------------------------------------------------------
diff --git a/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java b/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java
index d0f1091..c3fd25c 100644
--- a/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java
+++ b/hbase-spark/src/main/java/org/apache/hadoop/hbase/spark/SparkSQLPushDownFilter.java
@@ -24,8 +24,9 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.exceptions.DeserializationException;
import org.apache.hadoop.hbase.filter.FilterBase;
-import org.apache.hadoop.hbase.protobuf.generated.FilterProtos;
+import org.apache.hadoop.hbase.spark.protobuf.generated.FilterProtos;
import org.apache.hadoop.hbase.util.ByteStringer;
+import org.apache.hadoop.hbase.util.Bytes;
import scala.collection.mutable.MutableList;
import java.io.IOException;
@@ -38,51 +39,133 @@ import java.util.Map;
* by SparkSQL so that we have make the filters at the region server level
* and avoid sending the data back to the client to be filtered.
*/
-public class SparkSQLPushDownFilter extends FilterBase {
+public class SparkSQLPushDownFilter extends FilterBase{
protected static final Log log = LogFactory.getLog(SparkSQLPushDownFilter.class);
- HashMap<ColumnFamilyQualifierMapKeyWrapper, ColumnFilter> columnFamilyQualifierFilterMap;
+ //The following values are populated with protobuffer
+ DynamicLogicExpression dynamicLogicExpression;
+ byte[][] valueFromQueryArray;
+ HashMap<ByteArrayComparable, HashMap<ByteArrayComparable, String>>
+ currentCellToColumnIndexMap;
+
+ //The following values are transient
+ HashMap<String, ByteArrayComparable> columnToCurrentRowValueMap = null;
+
+ static final byte[] rowKeyFamily = new byte[0];
+ static final byte[] rowKeyQualifier = Bytes.toBytes("key");
+
+ public SparkSQLPushDownFilter(DynamicLogicExpression dynamicLogicExpression,
+ byte[][] valueFromQueryArray,
+ HashMap<ByteArrayComparable,
+ HashMap<ByteArrayComparable, String>>
+ currentCellToColumnIndexMap) {
+ this.dynamicLogicExpression = dynamicLogicExpression;
+ this.valueFromQueryArray = valueFromQueryArray;
+ this.currentCellToColumnIndexMap = currentCellToColumnIndexMap;
+ }
+
+ public SparkSQLPushDownFilter(DynamicLogicExpression dynamicLogicExpression,
+ byte[][] valueFromQueryArray,
+ MutableList<SchemaQualifierDefinition> columnDefinitions) {
+ this.dynamicLogicExpression = dynamicLogicExpression;
+ this.valueFromQueryArray = valueFromQueryArray;
+
+ //generate family qualifier to index mapping
+ this.currentCellToColumnIndexMap =
+ new HashMap<>();
+
+ for (int i = 0; i < columnDefinitions.size(); i++) {
+ SchemaQualifierDefinition definition = columnDefinitions.get(i).get();
+
+ ByteArrayComparable familyByteComparable =
+ new ByteArrayComparable(definition.columnFamilyBytes(),
+ 0, definition.columnFamilyBytes().length);
- public SparkSQLPushDownFilter(HashMap<ColumnFamilyQualifierMapKeyWrapper,
- ColumnFilter> columnFamilyQualifierFilterMap) {
- this.columnFamilyQualifierFilterMap = columnFamilyQualifierFilterMap;
+ HashMap<ByteArrayComparable, String> qualifierIndexMap =
+ currentCellToColumnIndexMap.get(familyByteComparable);
+
+ if (qualifierIndexMap == null) {
+ qualifierIndexMap = new HashMap<>();
+ currentCellToColumnIndexMap.put(familyByteComparable, qualifierIndexMap);
+ }
+ ByteArrayComparable qualifierByteComparable =
+ new ByteArrayComparable(definition.qualifierBytes(), 0,
+ definition.qualifierBytes().length);
+
+ qualifierIndexMap.put(qualifierByteComparable, definition.columnName());
+ }
}
- /**
- * This method will find the related filter logic for the given
- * column family and qualifier then execute it. It will also
- * not clone the in coming cell to avoid extra object creation
- *
- * @param c The Cell to be validated
- * @return ReturnCode object to determine if skipping is required
- * @throws IOException
- */
@Override
public ReturnCode filterKeyValue(Cell c) throws IOException {
- //Get filter if one exist
- ColumnFilter filter =
- columnFamilyQualifierFilterMap.get(new ColumnFamilyQualifierMapKeyWrapper(
- c.getFamilyArray(),
+ //If the map RowValueMap is empty then we need to populate
+ // the row key
+ if (columnToCurrentRowValueMap == null) {
+ columnToCurrentRowValueMap = new HashMap<>();
+ HashMap<ByteArrayComparable, String> qualifierColumnMap =
+ currentCellToColumnIndexMap.get(
+ new ByteArrayComparable(rowKeyFamily, 0, rowKeyFamily.length));
+
+ if (qualifierColumnMap != null) {
+ String rowKeyColumnName =
+ qualifierColumnMap.get(
+ new ByteArrayComparable(rowKeyQualifier, 0,
+ rowKeyQualifier.length));
+ //Make sure that the rowKey is part of the where clause
+ if (rowKeyColumnName != null) {
+ columnToCurrentRowValueMap.put(rowKeyColumnName,
+ new ByteArrayComparable(c.getRowArray(),
+ c.getRowOffset(), c.getRowLength()));
+ }
+ }
+ }
+
+ //Always populate the column value into the RowValueMap
+ ByteArrayComparable currentFamilyByteComparable =
+ new ByteArrayComparable(c.getFamilyArray(),
c.getFamilyOffset(),
- c.getFamilyLength(),
- c.getQualifierArray(),
- c.getQualifierOffset(),
- c.getQualifierLength()));
-
- if (filter == null) {
- //If no filter then just include values
- return ReturnCode.INCLUDE;
- } else {
- //If there is a filter then run validation
- if (filter.validate(c.getValueArray(), c.getValueOffset(), c.getValueLength())) {
- return ReturnCode.INCLUDE;
+ c.getFamilyLength());
+
+ HashMap<ByteArrayComparable, String> qualifierColumnMap =
+ currentCellToColumnIndexMap.get(
+ currentFamilyByteComparable);
+
+ if (qualifierColumnMap != null) {
+
+ String columnName =
+ qualifierColumnMap.get(
+ new ByteArrayComparable(c.getQualifierArray(),
+ c.getQualifierOffset(),
+ c.getQualifierLength()));
+
+ if (columnName != null) {
+ columnToCurrentRowValueMap.put(columnName,
+ new ByteArrayComparable(c.getValueArray(),
+ c.getValueOffset(), c.getValueLength()));
}
}
- //If validation fails then skip whole row
- return ReturnCode.NEXT_ROW;
+
+ return ReturnCode.INCLUDE;
+ }
+
+
+ @Override
+ public boolean filterRow() throws IOException {
+
+ try {
+ boolean result =
+ dynamicLogicExpression.execute(columnToCurrentRowValueMap,
+ valueFromQueryArray);
+ columnToCurrentRowValueMap = null;
+ return !result;
+ } catch (Throwable e) {
+ log.error("Error running dynamic logic on row", e);
+ }
+ return false;
}
+
/**
* @param pbBytes A pb serialized instance
* @return An instance of SparkSQLPushDownFilter
@@ -92,47 +175,55 @@ public class SparkSQLPushDownFilter extends FilterBase {
public static SparkSQLPushDownFilter parseFrom(final byte[] pbBytes)
throws DeserializationException {
- HashMap<ColumnFamilyQualifierMapKeyWrapper,
- ColumnFilter> columnFamilyQualifierFilterMap = new HashMap<>();
-
FilterProtos.SQLPredicatePushDownFilter proto;
try {
proto = FilterProtos.SQLPredicatePushDownFilter.parseFrom(pbBytes);
} catch (InvalidProtocolBufferException e) {
throw new DeserializationException(e);
}
- final List<FilterProtos.SQLPredicatePushDownColumnFilter> columnFilterListList =
- proto.getColumnFilterListList();
- for (FilterProtos.SQLPredicatePushDownColumnFilter columnFilter: columnFilterListList) {
+ //Load DynamicLogicExpression
+ DynamicLogicExpression dynamicLogicExpression =
+ DynamicLogicExpressionBuilder.build(proto.getDynamicLogicExpression());
- byte[] columnFamily = columnFilter.getColumnFamily().toByteArray();
- byte[] qualifier = columnFilter.getQualifier().toByteArray();
- final ColumnFamilyQualifierMapKeyWrapper columnFamilyQualifierMapKeyWrapper =
- new ColumnFamilyQualifierMapKeyWrapper(columnFamily, 0, columnFamily.length,
- qualifier, 0, qualifier.length);
+ //Load valuesFromQuery
+ final List<ByteString> valueFromQueryArrayList = proto.getValueFromQueryArrayList();
+ byte[][] valueFromQueryArray = new byte[valueFromQueryArrayList.size()][];
+ for (int i = 0; i < valueFromQueryArrayList.size(); i++) {
+ valueFromQueryArray[i] = valueFromQueryArrayList.get(i).toByteArray();
+ }
- final MutableList<byte[]> points = new MutableList<>();
- final MutableList<ScanRange> scanRanges = new MutableList<>();
+ //Load mapping from HBase family/qualifier to Spark SQL columnName
+ HashMap<ByteArrayComparable, HashMap<ByteArrayComparable, String>>
+ currentCellToColumnIndexMap = new HashMap<>();
- for (ByteString byteString: columnFilter.getGetPointListList()) {
- points.$plus$eq(byteString.toByteArray());
- }
+ for (FilterProtos.SQLPredicatePushDownCellToColumnMapping
+ sqlPredicatePushDownCellToColumnMapping :
+ proto.getCellToColumnMappingList()) {
- for (FilterProtos.RowRange rowRange: columnFilter.getRangeListList()) {
- ScanRange scanRange = new ScanRange(rowRange.getStopRow().toByteArray(),
- rowRange.getStopRowInclusive(),
- rowRange.getStartRow().toByteArray(),
- rowRange.getStartRowInclusive());
- scanRanges.$plus$eq(scanRange);
+ byte[] familyArray =
+ sqlPredicatePushDownCellToColumnMapping.getColumnFamily().toByteArray();
+ ByteArrayComparable familyByteComparable =
+ new ByteArrayComparable(familyArray, 0, familyArray.length);
+ HashMap<ByteArrayComparable, String> qualifierMap =
+ currentCellToColumnIndexMap.get(familyByteComparable);
+
+ if (qualifierMap == null) {
+ qualifierMap = new HashMap<>();
+ currentCellToColumnIndexMap.put(familyByteComparable, qualifierMap);
}
+ byte[] qualifierArray =
+ sqlPredicatePushDownCellToColumnMapping.getQualifier().toByteArray();
- final ColumnFilter columnFilterObj = new ColumnFilter(null, null, points, scanRanges);
+ ByteArrayComparable qualifierByteComparable =
+ new ByteArrayComparable(qualifierArray, 0 ,qualifierArray.length);
- columnFamilyQualifierFilterMap.put(columnFamilyQualifierMapKeyWrapper, columnFilterObj);
+ qualifierMap.put(qualifierByteComparable,
+ sqlPredicatePushDownCellToColumnMapping.getColumnName());
}
- return new SparkSQLPushDownFilter(columnFamilyQualifierFilterMap);
+ return new SparkSQLPushDownFilter(dynamicLogicExpression,
+ valueFromQueryArray, currentCellToColumnIndexMap);
}
/**
@@ -143,44 +234,27 @@ public class SparkSQLPushDownFilter extends FilterBase {
FilterProtos.SQLPredicatePushDownFilter.Builder builder =
FilterProtos.SQLPredicatePushDownFilter.newBuilder();
- FilterProtos.SQLPredicatePushDownColumnFilter.Builder columnBuilder =
- FilterProtos.SQLPredicatePushDownColumnFilter.newBuilder();
-
- FilterProtos.RowRange.Builder rowRangeBuilder = FilterProtos.RowRange.newBuilder();
-
- for (Map.Entry<ColumnFamilyQualifierMapKeyWrapper, ColumnFilter> entry :
- columnFamilyQualifierFilterMap.entrySet()) {
-
- columnBuilder.setColumnFamily(
- ByteStringer.wrap(entry.getKey().cloneColumnFamily()));
- columnBuilder.setQualifier(
- ByteStringer.wrap(entry.getKey().cloneQualifier()));
-
- final MutableList<byte[]> points = entry.getValue().points();
+ FilterProtos.SQLPredicatePushDownCellToColumnMapping.Builder columnMappingBuilder =
+ FilterProtos.SQLPredicatePushDownCellToColumnMapping.newBuilder();
- int pointLength = points.length();
- for (int i = 0; i < pointLength; i++) {
- byte[] point = points.get(i).get();
- columnBuilder.addGetPointList(ByteStringer.wrap(point));
-
- }
+ builder.setDynamicLogicExpression(dynamicLogicExpression.toExpressionString());
+ for (byte[] valueFromQuery: valueFromQueryArray) {
+ builder.addValueFromQueryArray(ByteStringer.wrap(valueFromQuery));
+ }
- final MutableList<ScanRange> ranges = entry.getValue().ranges();
- int rangeLength = ranges.length();
- for (int i = 0; i < rangeLength; i++) {
- ScanRange scanRange = ranges.get(i).get();
- rowRangeBuilder.clear();
- rowRangeBuilder.setStartRow(ByteStringer.wrap(scanRange.lowerBound()));
- rowRangeBuilder.setStopRow(ByteStringer.wrap(scanRange.upperBound()));
- rowRangeBuilder.setStartRowInclusive(scanRange.isLowerBoundEqualTo());
- rowRangeBuilder.setStopRowInclusive(scanRange.isUpperBoundEqualTo());
-
- columnBuilder.addRangeList(rowRangeBuilder.build());
+ for (Map.Entry<ByteArrayComparable, HashMap<ByteArrayComparable, String>>
+ familyEntry : currentCellToColumnIndexMap.entrySet()) {
+ for (Map.Entry<ByteArrayComparable, String> qualifierEntry :
+ familyEntry.getValue().entrySet()) {
+ columnMappingBuilder.setColumnFamily(
+ ByteStringer.wrap(familyEntry.getKey().bytes()));
+ columnMappingBuilder.setQualifier(
+ ByteStringer.wrap(qualifierEntry.getKey().bytes()));
+ columnMappingBuilder.setColumnName(qualifierEntry.getValue());
+ builder.addCellToColumnMapping(columnMappingBuilder.build());
}
-
- builder.addColumnFilterList(columnBuilder.build());
- columnBuilder.clear();
}
+
return builder.build().toByteArray();
}
}