You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ab...@apache.org on 2016/09/08 04:39:31 UTC
[1/7] incubator-impala git commit: IMPALA-3481: Use Kudu ScanToken
API for scan ranges
Repository: incubator-impala
Updated Branches:
refs/heads/master 94ef5b19b -> faebfebdf
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/main/java/com/cloudera/impala/planner/KuduScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/KuduScanNode.java b/fe/src/main/java/com/cloudera/impala/planner/KuduScanNode.java
index c1330b2..4f654a9 100644
--- a/fe/src/main/java/com/cloudera/impala/planner/KuduScanNode.java
+++ b/fe/src/main/java/com/cloudera/impala/planner/KuduScanNode.java
@@ -17,33 +17,37 @@
package com.cloudera.impala.planner;
-import java.math.BigDecimal;
+import java.io.IOException;
import java.util.List;
import java.util.ListIterator;
import java.util.Set;
-import com.cloudera.impala.analysis.BinaryPredicate;
-import com.cloudera.impala.analysis.BinaryPredicate.Operator;
-import com.cloudera.impala.analysis.Expr;
-import com.cloudera.impala.analysis.NumericLiteral;
-import com.cloudera.impala.analysis.SlotRef;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.ImpalaRuntimeException;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import org.kududb.client.KuduClient;
-import org.kududb.client.LocatedTablet;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduClient.KuduClientBuilder;
+import org.apache.kudu.client.KuduPredicate;
+import org.apache.kudu.client.KuduPredicate.ComparisonOp;
+import org.apache.kudu.client.KuduScanToken;
+import org.apache.kudu.client.KuduScanToken.KuduScanTokenBuilder;
+import org.apache.kudu.client.LocatedTablet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.cloudera.impala.analysis.Analyzer;
+import com.cloudera.impala.analysis.BinaryPredicate;
+import com.cloudera.impala.analysis.BoolLiteral;
+import com.cloudera.impala.analysis.Expr;
+import com.cloudera.impala.analysis.LiteralExpr;
+import com.cloudera.impala.analysis.NullLiteral;
+import com.cloudera.impala.analysis.NumericLiteral;
+import com.cloudera.impala.analysis.SlotDescriptor;
+import com.cloudera.impala.analysis.SlotRef;
+import com.cloudera.impala.analysis.StringLiteral;
import com.cloudera.impala.analysis.TupleDescriptor;
import com.cloudera.impala.catalog.KuduTable;
-import com.cloudera.impala.common.InternalException;
+import com.cloudera.impala.common.ImpalaRuntimeException;
import com.cloudera.impala.thrift.TExplainLevel;
-import com.cloudera.impala.thrift.TKuduKeyRange;
import com.cloudera.impala.thrift.TKuduScanNode;
import com.cloudera.impala.thrift.TNetworkAddress;
import com.cloudera.impala.thrift.TPlanNode;
@@ -51,20 +55,28 @@ import com.cloudera.impala.thrift.TPlanNodeType;
import com.cloudera.impala.thrift.TScanRange;
import com.cloudera.impala.thrift.TScanRangeLocation;
import com.cloudera.impala.thrift.TScanRangeLocations;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import static org.kududb.client.KuduClient.*;
-
/**
* Scan of a single Kudu table.
- * Extracts predicates that can be pushed down to Kudu and sends them to the backend
- * as part of a TKuduScanNode.
- * Currently only binary predicates (<=, >=, ==) that have a constant expression on one
- * side and a slot ref on the other can be evaluated by Kudu.
+ *
+ * Extracts predicates that can be pushed down to Kudu. Currently only binary predicates
+ * that have a constant expression on one side and a slot ref on the other can be
+ * evaluated by Kudu.
+ *
+ * Uses the Kudu ScanToken API to generate a set of Kudu "scan tokens" which are used for
+ * scheduling and initializing the scanners. Scan tokens are opaque objects that represent
+ * a scan for some Kudu data on a tablet (currently one token represents one tablet), and
+ * it contains the tablet locations and all information needed to produce a Kudu scanner,
+ * including the projected columns and predicates that are pushed down.
+ *
+ * After KUDU-1065 is resolved, Kudu will also prune the tablets that don't need to be
+ * scanned, and only the tokens for those tablets will be returned.
*/
public class KuduScanNode extends ScanNode {
-
private final static Logger LOG = LoggerFactory.getLogger(KuduScanNode.class);
private final KuduTable kuduTable_;
@@ -73,89 +85,123 @@ public class KuduScanNode extends ScanNode {
// From analyzer.getHostIndex().getIndex(address)
private final Set<Integer> hostIndexSet_ = Sets.newHashSet();
- // List of conjuncts that can be pushed down to Kudu
- // TODO use the portion of this list that pertains to keys to do partition pruning.
+ // List of conjuncts that can be pushed down to Kudu, after they have been normalized
+ // by BinaryPredicate.normalizeSlotRefComparison(). Used for computing stats and
+ // explain strings.
private final List<Expr> kuduConjuncts_ = Lists.newArrayList();
+ // Exprs in kuduConjuncts_ converted to KuduPredicates.
+ private final List<KuduPredicate> kuduPredicates_ = Lists.newArrayList();
+
public KuduScanNode(PlanNodeId id, TupleDescriptor desc) {
super(id, desc, "SCAN KUDU");
kuduTable_ = (KuduTable) desc_.getTable();
}
@Override
- public void init(Analyzer analyzer) throws InternalException {
+ public void init(Analyzer analyzer) throws ImpalaRuntimeException {
assignConjuncts(analyzer);
analyzer.createEquivConjuncts(tupleIds_.get(0), conjuncts_);
conjuncts_ = orderConjunctsByCost(conjuncts_);
- // Extract predicates that can be evaluated by Kudu.
- try {
- kuduConjuncts_.addAll(extractKuduConjuncts(analyzer));
- // Mark these slots as materialized, otherwise the toThrift() of SlotRefs
- // referencing them will fail. These slots will never be filled with data though as
- // Kudu won't return these columns.
- // TODO KUDU-935 Don't require that slots be materialized in order to serialize
- // SlotRefs.
- analyzer.materializeSlots(kuduConjuncts_);
- } catch (AnalysisException e) {
- throw new InternalException("Error while extracting Kudu conjuncts.", e);
+ try (KuduClient client =
+ new KuduClientBuilder(kuduTable_.getKuduMasterAddresses()).build()) {
+ org.apache.kudu.client.KuduTable rpcTable =
+ client.openTable(kuduTable_.getKuduTableName());
+ validateSchema(rpcTable);
+
+ // Extract predicates that can be evaluated by Kudu.
+ extractKuduConjuncts(analyzer, client, rpcTable);
+
+ // Materialize the slots of the remaining conjuncts (i.e. those not pushed to Kudu)
+ analyzer.materializeSlots(conjuncts_);
+
+ // Creates Kudu scan tokens and sets the scan range locations.
+ computeScanRangeLocations(analyzer, client, rpcTable);
+ } catch (Exception e) {
+ throw new ImpalaRuntimeException("Unable to initialize the Kudu scan node", e);
}
- computeScanRangeLocations(analyzer);
- analyzer.materializeSlots(conjuncts_);
computeMemLayout(analyzer);
computeStats(analyzer);
}
/**
- * Compute the scan range locations for the given table. Does not look at predicates.
- * To get the locations, we look at the table and load its tablets, for each tablet
- * we get the key-range and for each tablet we get the replicated hosts as well.
+ * Validate the columns Impala expects are actually in the Kudu table.
+ */
+ private void validateSchema(org.apache.kudu.client.KuduTable rpcTable)
+ throws ImpalaRuntimeException {
+ Schema tableSchema = rpcTable.getSchema();
+ for (SlotDescriptor desc: getTupleDesc().getSlots()) {
+ String colName = desc.getColumn().getName();
+ try {
+ tableSchema.getColumn(colName);
+ } catch (Exception e) {
+ throw new ImpalaRuntimeException("Column '" + colName + "' not found in kudu " +
+ "table " + rpcTable.getName());
+ }
+ }
+ }
+
+ /**
+ * Compute the scan range locations for the given table using the scan tokens.
*/
- private void computeScanRangeLocations(Analyzer analyzer) {
+ private void computeScanRangeLocations(Analyzer analyzer,
+ KuduClient client, org.apache.kudu.client.KuduTable rpcTable)
+ throws ImpalaRuntimeException {
scanRanges_ = Lists.newArrayList();
- try (KuduClient client = new KuduClientBuilder(
- kuduTable_.getKuduMasterAddresses()).build()) {
- org.kududb.client.KuduTable rpcTable =
- client.openTable(kuduTable_.getKuduTableName());
- List<LocatedTablet> tabletLocations =
- rpcTable.getTabletsLocations(KuduTable.KUDU_RPC_TIMEOUT_MS);
-
- for (LocatedTablet tablet : tabletLocations) {
- List<TScanRangeLocation> locations = Lists.newArrayList();
- if (tablet.getReplicas().isEmpty()) {
- throw new ImpalaRuntimeException(String.format(
- "At least one tablet does not have any replicas. Tablet ID: %s",
- new String(tablet.getTabletId(), Charsets.UTF_8)));
- }
- for (LocatedTablet.Replica replica : tablet.getReplicas()) {
- TNetworkAddress address = new TNetworkAddress(replica.getRpcHost(),
- replica.getRpcPort());
- // Use the network address to look up the host in the global list
- Integer hostIndex = analyzer.getHostIndex().getIndex(address);
- locations.add(new TScanRangeLocation(hostIndex));
- hostIndexSet_.add(hostIndex);
- }
- TScanRangeLocations locs = new TScanRangeLocations();
+ List<KuduScanToken> scanTokens = createScanTokens(client, rpcTable);
+ for (KuduScanToken token: scanTokens) {
+ LocatedTablet tablet = token.getTablet();
+ List<TScanRangeLocation> locations = Lists.newArrayList();
+ if (tablet.getReplicas().isEmpty()) {
+ throw new ImpalaRuntimeException(String.format(
+ "At least one tablet does not have any replicas. Tablet ID: %s",
+ new String(tablet.getTabletId(), Charsets.UTF_8)));
+ }
- // Now set the scan range of this tablet
- TKuduKeyRange keyRange = new TKuduKeyRange();
- keyRange.setRange_start_key(tablet.getPartition().getPartitionKeyStart());
- keyRange.setRange_stop_key(tablet.getPartition().getPartitionKeyEnd());
- TScanRange scanRange = new TScanRange();
- scanRange.setKudu_key_range(keyRange);
+ for (LocatedTablet.Replica replica: tablet.getReplicas()) {
+ TNetworkAddress address =
+ new TNetworkAddress(replica.getRpcHost(), replica.getRpcPort());
+ // Use the network address to look up the host in the global list
+ Integer hostIndex = analyzer.getHostIndex().getIndex(address);
+ locations.add(new TScanRangeLocation(hostIndex));
+ hostIndexSet_.add(hostIndex);
+ }
- // Set the scan range for this set of locations
- locs.setScan_range(scanRange);
- locs.locations = locations;
- scanRanges_.add(locs);
+ TScanRange scanRange = new TScanRange();
+ try {
+ scanRange.setKudu_scan_token(token.serialize());
+ } catch (IOException e) {
+ throw new ImpalaRuntimeException("Unable to serialize Kudu scan token=" +
+ token.toString(), e);
}
- } catch (Exception e) {
- throw new RuntimeException("Loading Kudu Table failed", e);
+
+ TScanRangeLocations locs = new TScanRangeLocations();
+ locs.setScan_range(scanRange);
+ locs.locations = locations;
+ scanRanges_.add(locs);
}
}
+ /**
+ * Returns KuduScanTokens for this scan given the projected columns and predicates that
+ * will be pushed to Kudu.
+ */
+ private List<KuduScanToken> createScanTokens(KuduClient client,
+ org.apache.kudu.client.KuduTable rpcTable) {
+ List<String> projectedCols = Lists.newArrayList();
+ for (SlotDescriptor desc: getTupleDesc().getSlots()) {
+ if (desc.isMaterialized()) projectedCols.add(desc.getColumn().getName());
+ }
+
+ KuduScanTokenBuilder tokenBuilder = client.newScanTokenBuilder(rpcTable);
+ tokenBuilder.setProjectedColumnNames(projectedCols);
+ for (KuduPredicate predicate: kuduPredicates_) tokenBuilder.addPredicate(predicate);
+ return tokenBuilder.build();
+ }
+
@Override
protected double computeSelectivity() {
List<Expr> allConjuncts = Lists.newArrayList(
@@ -208,121 +254,105 @@ public class KuduScanNode extends ScanNode {
protected void toThrift(TPlanNode node) {
node.node_type = TPlanNodeType.KUDU_SCAN_NODE;
node.kudu_scan_node = new TKuduScanNode(desc_.getId().asInt());
-
- // Thriftify the pushable predicates and set them on the scan node.
- for (Expr predicate : kuduConjuncts_) {
- node.kudu_scan_node.addToKudu_conjuncts(predicate.treeToThrift());
- }
}
/**
* Extracts predicates from conjuncts_ that can be pushed down to Kudu. Currently only
- * binary predicates (<=, >=, ==) that have a constant expression on one side and a slot
- * ref on the other can be evaluated by Kudu. Only looks at comparisons of constants
- * (i.e., the bounds of the result can be evaluated with Expr::GetValue(NULL)).
+ * binary predicates that have a constant expression on one side and a slot ref on the
+ * other can be evaluated by Kudu. Only looks at comparisons of constants (i.e., the
+ * bounds of the result can be evaluated with Expr::GetValue(NULL)). If a conjunct can
+ * be converted into this form, the normalized expr is added to kuduConjuncts_, a
+ * KuduPredicate is added to kuduPredicates_, and the original expr from conjuncts_ is
+ * removed.
*/
- private List<Expr> extractKuduConjuncts(Analyzer analyzer)
- throws InternalException, AnalysisException {
- ImmutableList.Builder<Expr> pushableConjunctsBuilder = ImmutableList.builder();
- ListIterator<Expr> i = conjuncts_.listIterator();
- while (i.hasNext()) {
- Expr e = i.next();
- if (!(e instanceof BinaryPredicate)) continue;
- BinaryPredicate comparisonPred = (BinaryPredicate) e;
- // TODO KUDU-931 look into handling implicit/explicit casts on the SlotRef.
- comparisonPred = BinaryPredicate.normalizeSlotRefComparison(comparisonPred,
- analyzer);
- if (comparisonPred == null) continue;
-
- // Needs to have a literal on the right.
- if (!comparisonPred.getChild(1).isLiteral()) continue;
-
- comparisonPred = normalizeIntLiteralComparison(comparisonPred, analyzer);
-
- Operator op = comparisonPred.getOp();
- switch (comparisonPred.getOp()) {
- case NE: continue;
- case GT: continue; // TODO Exclusive predicates are not supported in Kudu yet.
- case LT: continue; // TODO Exclusive predicates are not supported in Kudu yet.
- case GE: // Fallthrough intended.
- case LE: // Fallthrough intended.
- case EQ: {
- i.remove();
- pushableConjunctsBuilder.add(comparisonPred);
- break;
- }
- default:
- Preconditions.checkState(false, "Unexpected BinaryPredicate type: "
- + op.getName());
- }
+ private void extractKuduConjuncts(Analyzer analyzer,
+ KuduClient client, org.apache.kudu.client.KuduTable rpcTable) {
+ ListIterator<Expr> it = conjuncts_.listIterator();
+ while (it.hasNext()) {
+ if (tryConvertKuduPredicate(analyzer, rpcTable, it.next())) it.remove();
}
- return pushableConjunctsBuilder.build();
}
/**
- * Hack to be able to push to Kudu Int GT/LT conjuncts by incrementing or decrementing
- * the int literal and changing the operator to GE/LE.
- * Expects the predicate to have been previously normalized.
- * Returns the same BinaryPredicate if the transformation was not needed or possible or
- * a new, analyzed predicate if it was.
- * TODO Remove this when KUDU-1148 (inclusive predicate support in Kudu) gets done
+ * If 'expr' can be converted to a KuduPredicate, returns true and updates
+ * kuduPredicates_ and kuduConjuncts_.
*/
- private static BinaryPredicate normalizeIntLiteralComparison(
- BinaryPredicate comparisonPred, Analyzer analyzer) {
- Expr constantExpr = comparisonPred.getChild(1);
- if (!(constantExpr instanceof NumericLiteral)) return comparisonPred;
- NumericLiteral numLiteral = (NumericLiteral) constantExpr;
- long intValue = numLiteral.getLongValue();
- if (comparisonPred.getOp() == Operator.GT) {
- // Make sure we don't overflow the type, in which case the type would change and
- // the slot would get an implicit cast meaning we wouldn't push it anyway.
- switch (constantExpr.getType().getPrimitiveType()) {
- case TINYINT:
- if (intValue >= Byte.MAX_VALUE) return comparisonPred;
- break;
- case SMALLINT:
- if (intValue >= Short.MAX_VALUE) return comparisonPred;
- break;
- case INT:
- if (intValue >= Integer.MAX_VALUE) return comparisonPred;
- break;
- case BIGINT:
- if (intValue >= Long.MAX_VALUE) return comparisonPred;
- break;
- default: return comparisonPred;
+ private boolean tryConvertKuduPredicate(Analyzer analyzer,
+ org.apache.kudu.client.KuduTable table, Expr expr) {
+ if (!(expr instanceof BinaryPredicate)) return false;
+ BinaryPredicate predicate = (BinaryPredicate) expr;
+
+ // TODO KUDU-931 look into handling implicit/explicit casts on the SlotRef.
+ predicate = BinaryPredicate.normalizeSlotRefComparison(predicate, analyzer);
+ if (predicate == null) return false;
+ ComparisonOp op = getKuduOperator(((BinaryPredicate)predicate).getOp());
+ if (op == null) return false;
+
+ SlotRef ref = (SlotRef) predicate.getChild(0);
+ LiteralExpr literal = (LiteralExpr) predicate.getChild(1);
+
+ // Cannot push prediates with null literal values (KUDU-1595).
+ if (literal instanceof NullLiteral) return false;
+
+ String colName = ref.getDesc().getColumn().getName();
+ ColumnSchema column = table.getSchema().getColumn(colName);
+ KuduPredicate kuduPredicate = null;
+ switch (literal.getType().getPrimitiveType()) {
+ case BOOLEAN: {
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ ((BoolLiteral)literal).getValue());
+ break;
}
- BigDecimal newValue = BigDecimal.valueOf(intValue + 1);
- NumericLiteral newLiteral = new NumericLiteral(newValue);
- comparisonPred = new BinaryPredicate(Operator.GE, comparisonPred.getChild(0),
- newLiteral);
- comparisonPred.analyzeNoThrow(analyzer);
- return comparisonPred;
- }
- if (comparisonPred.getOp() == Operator.LT) {
- // Make sure we don't underflow the type, in which case the type would change and
- // the slot would get an implicit cast meaning we wouldn't push it anyway.
- switch (constantExpr.getType().getPrimitiveType()) {
- case TINYINT:
- if (intValue <= Byte.MIN_VALUE) return comparisonPred;
- break;
- case SMALLINT:
- if (intValue <= Short.MIN_VALUE) return comparisonPred;
- break;
- case INT:
- if (intValue <= Integer.MIN_VALUE) return comparisonPred;
- break;
- case BIGINT:
- if (intValue <= Long.MIN_VALUE) return comparisonPred;
- break;
- default: return comparisonPred;
+ case TINYINT:
+ case SMALLINT:
+ case INT: {
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ ((NumericLiteral)literal).getIntValue());
+ break;
+ }
+ case BIGINT: {
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ ((NumericLiteral)literal).getLongValue());
+ break;
+ }
+ case FLOAT: {
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ (float)((NumericLiteral)literal).getDoubleValue());
+ break;
+ }
+ case DOUBLE: {
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ ((NumericLiteral)literal).getDoubleValue());
+ break;
}
- BigDecimal newValue = BigDecimal.valueOf(intValue - 1);
- NumericLiteral newLiteral = new NumericLiteral(newValue);
- comparisonPred = new BinaryPredicate(Operator.LE, comparisonPred.getChild(0),
- newLiteral);
- comparisonPred.analyzeNoThrow(analyzer);
- return comparisonPred;
+ case STRING:
+ case VARCHAR:
+ case CHAR: {
+ kuduPredicate = KuduPredicate.newComparisonPredicate(column, op,
+ ((StringLiteral)literal).getStringValue());
+ break;
+ }
+ default: break;
+ }
+ if (kuduPredicate == null) return false;
+
+ kuduConjuncts_.add(predicate);
+ kuduPredicates_.add(kuduPredicate);
+ return true;
+ }
+
+ /**
+ * Returns a Kudu comparison operator for the BinaryPredicate operator, or null if
+ * the operation is not supported by Kudu.
+ */
+ private static KuduPredicate.ComparisonOp getKuduOperator(BinaryPredicate.Operator op) {
+ switch (op) {
+ case GT: return ComparisonOp.GREATER;
+ case LT: return ComparisonOp.LESS;
+ case GE: return ComparisonOp.GREATER_EQUAL;
+ case LE: return ComparisonOp.LESS_EQUAL;
+ case EQ: return ComparisonOp.EQUAL;
+ default: return null;
}
- return comparisonPred;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/main/java/com/cloudera/impala/util/KuduUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/util/KuduUtil.java b/fe/src/main/java/com/cloudera/impala/util/KuduUtil.java
index 3a97c05..4f627d8 100644
--- a/fe/src/main/java/com/cloudera/impala/util/KuduUtil.java
+++ b/fe/src/main/java/com/cloudera/impala/util/KuduUtil.java
@@ -37,11 +37,11 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.kududb.ColumnSchema;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.KuduTable;
-import org.kududb.client.PartialRow;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.KuduTable;
+import org.apache.kudu.client.PartialRow;
import static com.cloudera.impala.catalog.Type.parseColumnType;
import static java.lang.String.format;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java b/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
deleted file mode 100644
index fe15737..0000000
--- a/fe/src/test/java/com/cloudera/impala/planner/KuduPlannerTest.java
+++ /dev/null
@@ -1,52 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import com.cloudera.impala.testutil.TestUtils;
-import com.cloudera.impala.thrift.TExplainLevel;
-import com.cloudera.impala.thrift.TQueryOptions;
-import org.junit.Before;
-import org.junit.Test;
-
-/**
- * This class contains all plan generation related tests with Kudu as a storage backend.
- */
-public class KuduPlannerTest extends PlannerTestBase {
-
- @Before
- public void checkKuduSupport() { TestUtils.assumeKuduIsSupported(); }
-
- @Test
- public void testKudu() { runPlannerTestFile("kudu"); }
-
- @Test
- public void testUpdate() { runPlannerTestFile("kudu-update"); }
-
- @Test
- public void testDelete() { runPlannerTestFile("kudu-delete"); }
-
- @Test
- public void testSelectivity() {
- TQueryOptions options = defaultQueryOptions();
- options.setExplain_level(TExplainLevel.VERBOSE);
- runPlannerTestFile("kudu-selectivity", options);
- }
-
- @Test
- public void testTpch() { runPlannerTestFile("tpch-kudu"); }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/test/java/com/cloudera/impala/planner/PlannerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/planner/PlannerTest.java b/fe/src/test/java/com/cloudera/impala/planner/PlannerTest.java
index 7472da0..a09f218 100644
--- a/fe/src/test/java/com/cloudera/impala/planner/PlannerTest.java
+++ b/fe/src/test/java/com/cloudera/impala/planner/PlannerTest.java
@@ -20,6 +20,7 @@ package com.cloudera.impala.planner;
import org.junit.Test;
import com.cloudera.impala.catalog.Db;
+import com.cloudera.impala.thrift.TExplainLevel;
import com.cloudera.impala.thrift.TQueryOptions;
import com.cloudera.impala.thrift.TRuntimeFilterMode;
@@ -244,4 +245,23 @@ public class PlannerTest extends PlannerTestBase {
public void testConjunctOrdering() {
runPlannerTestFile("conjunct-ordering");
}
+
+ @Test
+ public void testKudu() { runPlannerTestFile("kudu"); }
+
+ @Test
+ public void testKuduUpdate() { runPlannerTestFile("kudu-update"); }
+
+ @Test
+ public void testKuduDelete() { runPlannerTestFile("kudu-delete"); }
+
+ @Test
+ public void testKuduSelectivity() {
+ TQueryOptions options = defaultQueryOptions();
+ options.setExplain_level(TExplainLevel.VERBOSE);
+ runPlannerTestFile("kudu-selectivity", options);
+ }
+
+ @Test
+ public void testKuduTpch() { runPlannerTestFile("tpch-kudu"); }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/test/java/com/cloudera/impala/planner/PlannerTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/planner/PlannerTestBase.java b/fe/src/test/java/com/cloudera/impala/planner/PlannerTestBase.java
index 328d151..6db1293 100644
--- a/fe/src/test/java/com/cloudera/impala/planner/PlannerTestBase.java
+++ b/fe/src/test/java/com/cloudera/impala/planner/PlannerTestBase.java
@@ -33,6 +33,8 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.fs.Path;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.KuduScanToken;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.slf4j.Logger;
@@ -58,7 +60,6 @@ import com.cloudera.impala.thrift.THdfsPartition;
import com.cloudera.impala.thrift.THdfsPartitionLocation;
import com.cloudera.impala.thrift.THdfsScanNode;
import com.cloudera.impala.thrift.THdfsTable;
-import com.cloudera.impala.thrift.TKuduKeyRange;
import com.cloudera.impala.thrift.TLineageGraph;
import com.cloudera.impala.thrift.TNetworkAddress;
import com.cloudera.impala.thrift.TPlanFragment;
@@ -82,6 +83,7 @@ public class PlannerTestBase extends FrontendTestBase {
private final static boolean GENERATE_OUTPUT_FILE = true;
private final String testDir_ = "functional-planner/queries/PlannerTest";
private final String outDir_ = "/tmp/PlannerTest/";
+ private static KuduClient kuduClient_;
// Map from plan ID (TPlanNodeId) to the plan node with that ID.
private final Map<Integer, TPlanNode> planMap_ = Maps.newHashMap();
@@ -102,11 +104,20 @@ public class PlannerTestBase extends FrontendTestBase {
updateReq.setHostnames(Sets.newHashSet("localhost"));
updateReq.setNum_nodes(3);
MembershipSnapshot.update(updateReq);
+
+ if (RuntimeEnv.INSTANCE.isKuduSupported()) {
+ kuduClient_ = new KuduClient.KuduClientBuilder("127.0.0.1:7051").build();
+ }
}
@AfterClass
- public static void cleanUp() {
+ public static void cleanUp() throws Exception {
RuntimeEnv.INSTANCE.reset();
+
+ if (kuduClient_ != null) {
+ kuduClient_.close();
+ kuduClient_ = null;
+ }
}
/**
@@ -274,20 +285,16 @@ public class PlannerTestBase extends FrontendTestBase {
}
}
- if (locations.scan_range.isSetKudu_key_range()) {
- TKuduKeyRange kr = locations.scan_range.getKudu_key_range();
- Integer hostIdx = locations.locations.get(0).host_idx;
- TNetworkAddress networkAddress = execRequest.getHost_list().get(hostIdx);
- result.append("KUDU KEYRANGE ");
- // TODO Enable the lines below once we have better testing for
- // non-local key-ranges
- //result.append("host=" + networkAddress.hostname + ":" +
- // networkAddress.port + " ");
- result.append(Arrays.toString(kr.getRange_start_key()));
- result.append(":");
- result.append(Arrays.toString(kr.getRange_stop_key()));
+ if (locations.scan_range.isSetKudu_scan_token()) {
+ Preconditions.checkNotNull(kuduClient_,
+ "Test should not be invoked on platforms that do not support Kudu.");
+ try {
+ result.append(KuduScanToken.stringifySerializedToken(
+ locations.scan_range.kudu_scan_token.array(), kuduClient_));
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to parse Kudu scan token", e);
+ }
}
-
result.append("\n");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/test/java/com/cloudera/impala/util/KuduUtilTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/com/cloudera/impala/util/KuduUtilTest.java b/fe/src/test/java/com/cloudera/impala/util/KuduUtilTest.java
index 9b2c884..ef5c817 100644
--- a/fe/src/test/java/com/cloudera/impala/util/KuduUtilTest.java
+++ b/fe/src/test/java/com/cloudera/impala/util/KuduUtilTest.java
@@ -22,11 +22,11 @@ import java.util.List;
import com.cloudera.impala.common.ImpalaRuntimeException;
import com.google.common.collect.ImmutableList;
import org.junit.Test;
-import org.kududb.ColumnSchema;
-import org.kududb.ColumnSchema.ColumnSchemaBuilder;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.PartialRow;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.PartialRow;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test b/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
index ead71ef..73acf7f 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/kudu-selectivity.test
@@ -45,7 +45,7 @@ select * from functional_kudu.zipcode_incomes where id > '1' and zip > '2'
---- PLAN
F00:PLAN FRAGMENT [UNPARTITIONED]
00:SCAN KUDU [functional_kudu.zipcode_incomes]
- predicates: zip > '2', id > '1'
+ kudu predicates: zip > '2', id > '1'
hosts=3 per-host-mem=unavailable
tuple-ids=0 row-size=124B cardinality=3317
---- DISTRIBUTEDPLAN
@@ -57,7 +57,7 @@ F01:PLAN FRAGMENT [UNPARTITIONED]
F00:PLAN FRAGMENT [RANDOM]
DATASTREAM SINK [FRAGMENT=F01, EXCHANGE=01, UNPARTITIONED]
00:SCAN KUDU [functional_kudu.zipcode_incomes]
- predicates: zip > '2', id > '1'
+ kudu predicates: zip > '2', id > '1'
hosts=3 per-host-mem=0B
tuple-ids=0 row-size=124B cardinality=3317
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/testdata/workloads/functional-planner/queries/PlannerTest/kudu-update.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/kudu-update.test b/testdata/workloads/functional-planner/queries/PlannerTest/kudu-update.test
index 0c21b36..80ba800 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/kudu-update.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/kudu-update.test
@@ -4,13 +4,13 @@ UPDATE KUDU [functional_kudu.testtbl]
| check keys exist: false
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: zip >= 94550
+ kudu predicates: zip > 94549
---- DISTRIBUTEDPLAN
UPDATE KUDU [functional_kudu.testtbl]
| check keys exist: false
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: zip >= 94550
+ kudu predicates: zip > 94549
====
# Predicate on key
update functional_kudu.testtbl set name = 'peter' where zip > 94549 and id = 5
@@ -19,13 +19,13 @@ UPDATE KUDU [functional_kudu.testtbl]
| check keys exist: false
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: zip >= 94550, id = 5
+ kudu predicates: zip > 94549, id = 5
---- DISTRIBUTEDPLAN
UPDATE KUDU [functional_kudu.testtbl]
| check keys exist: false
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: zip >= 94550, id = 5
+ kudu predicates: zip > 94549, id = 5
====
# Mixing predicate and value assignment
update functional_kudu.testtbl set zip = 94546 where zip > 94549
@@ -34,13 +34,13 @@ UPDATE KUDU [functional_kudu.testtbl]
| check keys exist: false
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: zip >= 94550
+ kudu predicates: zip > 94549
---- DISTRIBUTEDPLAN
UPDATE KUDU [functional_kudu.testtbl]
| check keys exist: false
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: zip >= 94550
+ kudu predicates: zip > 94549
====
update a
set a.name = b.name
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test b/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
index 35abe35..2c8a415 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
@@ -3,9 +3,9 @@ select * from functional_kudu.testtbl
00:SCAN KUDU [functional_kudu.testtbl]
---- SCANRANGELOCATIONS
NODE 0:
- KUDU KEYRANGE [0, 0, 0, 1]:[0, 0, 0, 2]
- KUDU KEYRANGE [0, 0, 0, 2]:[]
- KUDU KEYRANGE []:[0, 0, 0, 1]
+ ScanToken{table=testtbl, hash-partition-buckets: [0], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [1], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [2], range-partition: [<start>, <end>)}
---- DISTRIBUTEDPLAN
01:EXCHANGE [UNPARTITIONED]
|
@@ -17,15 +17,20 @@ select * from functional_kudu.testtbl where name = '10'
kudu predicates: name = '10'
---- SCANRANGELOCATIONS
NODE 0:
- KUDU KEYRANGE [0, 0, 0, 1]:[0, 0, 0, 2]
- KUDU KEYRANGE [0, 0, 0, 2]:[]
- KUDU KEYRANGE []:[0, 0, 0, 1]
+ ScanToken{table=testtbl, hash-partition-buckets: [0], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [1], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [2], range-partition: [<start>, <end>)}
---- DISTRIBUTEDPLAN
01:EXCHANGE [UNPARTITIONED]
|
00:SCAN KUDU [functional_kudu.testtbl]
kudu predicates: name = '10'
====
+select * from functional_kudu.testtbl where name = NULL
+---- PLAN
+00:SCAN KUDU [functional_kudu.testtbl]
+ predicates: name = NULL
+====
insert into functional_kudu.testtbl(id) values (10)
---- PLAN
INSERT INTO KUDU [functional_kudu.testtbl]
@@ -91,17 +96,17 @@ where id >= 10 and zip <= 5 and 20 >= id and 'foo' = name and zip >= 0 and 30 >=
and zip > 1 and zip < 50
---- PLAN
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: id >= 10, zip <= 5, id <= 20, zip >= 0, zip <= 30, zip >= 2, zip <= 49, name = 'foo'
+ kudu predicates: id >= 10, zip <= 5, id <= 20, zip >= 0, zip <= 30, zip > 1, zip < 50, name = 'foo'
---- SCANRANGELOCATIONS
NODE 0:
- KUDU KEYRANGE [0, 0, 0, 1]:[0, 0, 0, 2]
- KUDU KEYRANGE [0, 0, 0, 2]:[]
- KUDU KEYRANGE []:[0, 0, 0, 1]
+ ScanToken{table=testtbl, hash-partition-buckets: [0], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [1], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [2], range-partition: [<start>, <end>)}
---- DISTRIBUTEDPLAN
01:EXCHANGE [UNPARTITIONED]
|
00:SCAN KUDU [functional_kudu.testtbl]
- kudu predicates: id >= 10, zip <= 5, id <= 20, zip >= 0, zip <= 30, zip >= 2, zip <= 49, name = 'foo'
+ kudu predicates: id >= 10, zip <= 5, id <= 20, zip >= 0, zip <= 30, zip > 1, zip < 50, name = 'foo'
====
# Test constant folding.
select * from functional_kudu.testtbl
@@ -109,18 +114,18 @@ where id < 10 + 30 and cast(sin(id) as boolean) = true and 20 * 3 >= id and 10
---- PLAN
00:SCAN KUDU [functional_kudu.testtbl]
predicates: CAST(sin(id) AS BOOLEAN) = TRUE
- kudu predicates: id <= 39, id <= 60, id <= 102
+ kudu predicates: id < 40, id <= 60, id < 103
---- SCANRANGELOCATIONS
NODE 0:
- KUDU KEYRANGE [0, 0, 0, 1]:[0, 0, 0, 2]
- KUDU KEYRANGE [0, 0, 0, 2]:[]
- KUDU KEYRANGE []:[0, 0, 0, 1]
+ ScanToken{table=testtbl, hash-partition-buckets: [0], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [1], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [2], range-partition: [<start>, <end>)}
---- DISTRIBUTEDPLAN
01:EXCHANGE [UNPARTITIONED]
|
00:SCAN KUDU [functional_kudu.testtbl]
predicates: CAST(sin(id) AS BOOLEAN) = TRUE
- kudu predicates: id <= 39, id <= 60, id <= 102
+ kudu predicates: id < 40, id <= 60, id < 103
====
# Some predicates can be pushed down but others can't (predicate on an non-const value).
select * from functional_kudu.testtbl
@@ -131,9 +136,9 @@ where cast(sin(id) as boolean) = true and name = 'a'
kudu predicates: name = 'a'
---- SCANRANGELOCATIONS
NODE 0:
- KUDU KEYRANGE [0, 0, 0, 1]:[0, 0, 0, 2]
- KUDU KEYRANGE [0, 0, 0, 2]:[]
- KUDU KEYRANGE []:[0, 0, 0, 1]
+ ScanToken{table=testtbl, hash-partition-buckets: [0], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [1], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [2], range-partition: [<start>, <end>)}
---- DISTRIBUTEDPLAN
01:EXCHANGE [UNPARTITIONED]
|
@@ -150,9 +155,9 @@ where cast(sin(id) as boolean) = true and name is null
predicates: name IS NULL, CAST(sin(id) AS BOOLEAN) = TRUE
---- SCANRANGELOCATIONS
NODE 0:
- KUDU KEYRANGE [0, 0, 0, 1]:[0, 0, 0, 2]
- KUDU KEYRANGE [0, 0, 0, 2]:[]
- KUDU KEYRANGE []:[0, 0, 0, 1]
+ ScanToken{table=testtbl, hash-partition-buckets: [0], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [1], range-partition: [<start>, <end>)}
+ ScanToken{table=testtbl, hash-partition-buckets: [2], range-partition: [<start>, <end>)}
---- DISTRIBUTEDPLAN
01:EXCHANGE [UNPARTITIONED]
|
@@ -178,5 +183,5 @@ where t.c <= cast('1995-01-01 00:00:00' as timestamp) order by c
|
00:SCAN KUDU [tpch_kudu.orders]
predicates: CAST(o_orderdate AS TIMESTAMP) <= CAST('1995-01-01 00:00:00' AS TIMESTAMP)
- kudu predicates: o_orderkey <= 9
+ kudu predicates: o_orderkey < 10
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
index 2dc3619..5659b4b 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpch-kudu.test
@@ -172,10 +172,10 @@ limit 10
| hash predicates: l_orderkey = o_orderkey
|
|--01:SCAN KUDU [tpch_kudu.orders]
-| predicates: o_orderdate < '1995-03-15'
+| kudu predicates: o_orderdate < '1995-03-15'
|
02:SCAN KUDU [tpch_kudu.lineitem]
- predicates: l_shipdate > '1995-03-15'
+ kudu predicates: l_shipdate > '1995-03-15'
====
# Q4 - Order Priority Checking Query
select
@@ -211,8 +211,7 @@ order by
| hash predicates: l_orderkey = o_orderkey
|
|--00:SCAN KUDU [tpch_kudu.orders]
-| predicates: o_orderdate < '1993-10-01'
-| kudu predicates: o_orderdate >= '1993-07-01'
+| kudu predicates: o_orderdate < '1993-10-01', o_orderdate >= '1993-07-01'
|
01:SCAN KUDU [tpch_kudu.lineitem]
predicates: l_commitdate < l_receiptdate
@@ -275,8 +274,7 @@ order by
| hash predicates: l_orderkey = o_orderkey
|
|--01:SCAN KUDU [tpch_kudu.orders]
-| predicates: o_orderdate < '1995-01-01'
-| kudu predicates: o_orderdate >= '1994-01-01'
+| kudu predicates: o_orderdate >= '1994-01-01', o_orderdate < '1995-01-01'
|
02:SCAN KUDU [tpch_kudu.lineitem]
====
@@ -295,8 +293,7 @@ where
| output: sum(l_extendedprice * l_discount)
|
00:SCAN KUDU [tpch_kudu.lineitem]
- predicates: l_quantity < 24, l_shipdate < '1995-01-01'
- kudu predicates: l_discount >= 0.05, l_discount <= 0.07, l_shipdate >= '1994-01-01'
+ kudu predicates: l_discount >= 0.05, l_discount <= 0.07, l_quantity < 24, l_shipdate >= '1994-01-01', l_shipdate < '1995-01-01'
====
# Q7 - Volume Shipping Query
select
@@ -582,8 +579,7 @@ limit 20
| | hash predicates: l_orderkey = o_orderkey
| |
| |--01:SCAN KUDU [tpch_kudu.orders]
-| | predicates: o_orderdate < '1994-01-01'
-| | kudu predicates: o_orderdate >= '1993-10-01'
+| | kudu predicates: o_orderdate >= '1993-10-01', o_orderdate < '1994-01-01'
| |
| 02:SCAN KUDU [tpch_kudu.lineitem]
| kudu predicates: l_returnflag = 'R'
@@ -705,8 +701,8 @@ order by
| hash predicates: o_orderkey = l_orderkey
|
|--01:SCAN KUDU [tpch_kudu.lineitem]
-| predicates: l_shipmode IN ('MAIL', 'SHIP'), l_commitdate < l_receiptdate, l_shipdate < l_commitdate, l_receiptdate < '1995-01-01'
-| kudu predicates: l_receiptdate >= '1994-01-01'
+| predicates: l_shipmode IN ('MAIL', 'SHIP'), l_commitdate < l_receiptdate, l_shipdate < l_commitdate
+| kudu predicates: l_receiptdate >= '1994-01-01', l_receiptdate < '1995-01-01'
|
00:SCAN KUDU [tpch_kudu.orders]
====
@@ -775,8 +771,7 @@ where
|--01:SCAN KUDU [tpch_kudu.part]
|
00:SCAN KUDU [tpch_kudu.lineitem]
- predicates: l_shipdate < '1995-10-01'
- kudu predicates: l_shipdate >= '1995-09-01'
+ kudu predicates: l_shipdate >= '1995-09-01', l_shipdate < '1995-10-01'
====
# Q15 - Top Supplier Query
with revenue_view as (
@@ -824,8 +819,7 @@ order by
| | group by: l_suppkey
| |
| 03:SCAN KUDU [tpch_kudu.lineitem]
-| predicates: l_shipdate < '1996-04-01'
-| kudu predicates: l_shipdate >= '1996-01-01'
+| kudu predicates: l_shipdate >= '1996-01-01', l_shipdate < '1996-04-01'
|
06:HASH JOIN [INNER JOIN]
| hash predicates: s_suppkey = l_suppkey
@@ -835,8 +829,7 @@ order by
| | group by: l_suppkey
| |
| 01:SCAN KUDU [tpch_kudu.lineitem]
-| predicates: l_shipdate < '1996-04-01'
-| kudu predicates: l_shipdate >= '1996-01-01'
+| kudu predicates: l_shipdate >= '1996-01-01', l_shipdate < '1996-04-01'
|
00:SCAN KUDU [tpch_kudu.supplier]
====
@@ -1118,8 +1111,7 @@ order by
| group by: l_partkey, l_suppkey
|
04:SCAN KUDU [tpch_kudu.lineitem]
- predicates: l_shipdate < '1995-01-01'
- kudu predicates: l_shipdate >= '1994-01-01'
+ kudu predicates: l_shipdate >= '1994-01-01', l_shipdate < '1995-01-01'
====
# Q21 - Suppliers Who Kept Orders Waiting Query
select
@@ -1256,7 +1248,8 @@ order by
| | | output: avg(c_acctbal)
| | |
| | 01:SCAN KUDU [tpch_kudu.customer]
-| | predicates: c_acctbal > 0.00, substr(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')
+| | predicates: substr(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')
+| | kudu predicates: c_acctbal > 0.00
| |
| 00:SCAN KUDU [tpch_kudu.customer]
| predicates: substr(c_phone, 1, 2) IN ('13', '31', '23', '29', '30', '18', '17')
[4/7] incubator-impala git commit: IMPALA-3491: Use unique database
fixture in test_partitioning.py
Posted by ab...@apache.org.
IMPALA-3491: Use unique database fixture in test_partitioning.py
Testing: Ran the test locally in a loop on exhaustive. Did a private
debug/exhaustive/hdfs test run.
Change-Id: Ib1b33d9977a98894288662a711805e9a54329ec8
Reviewed-on: http://gerrit.cloudera.org:8080/4316
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8c37bf35
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8c37bf35
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8c37bf35
Branch: refs/heads/master
Commit: 8c37bf35438aa827f092d99d6cff83ce1ed83fbb
Parents: f0ffbca
Author: Alex Behm <al...@cloudera.com>
Authored: Fri Sep 2 08:44:25 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Sep 8 04:31:27 2016 +0000
----------------------------------------------------------------------
.../queries/QueryTest/partition-col-types.test | 32 +++++------
tests/query_test/test_partitioning.py | 57 +++++++-------------
2 files changed, 34 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8c37bf35/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test b/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
index 3d58671..e364ce4 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-col-types.test
@@ -12,7 +12,7 @@ PARTITIONED BY (
float_col FLOAT,
double_col DOUBLE,
string_col STRING
-) LOCATION '/test-warehouse/all_insert_partition_col_types'
+) LOCATION '/test-warehouse/$DATABASE.db/all_insert_partition_col_types'
====
---- QUERY
DESCRIBE all_insert_partition_col_types
@@ -83,7 +83,7 @@ string_col in ('1', '2', '3')
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=3/11 files=3 size=6B'
====
---- QUERY
@@ -93,7 +93,7 @@ WHERE tinyint_col < 7
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=7/11 files=7 size=14B'
====
---- QUERY
@@ -103,7 +103,7 @@ WHERE smallint_col < 6
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=6/11 files=6 size=12B'
====
---- QUERY
@@ -113,7 +113,7 @@ WHERE int_col < 5
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=5/11 files=5 size=10B'
====
---- QUERY
@@ -123,7 +123,7 @@ WHERE bigint_col < 40
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=4/11 files=4 size=8B'
====
---- QUERY
@@ -133,7 +133,7 @@ WHERE string_col in ('1', '2', '3')
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=3/11 files=3 size=6B'
====
---- QUERY
@@ -143,7 +143,7 @@ WHERE double_col = 1.1
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=10/11 files=10 size=20B'
====
---- QUERY
@@ -153,7 +153,7 @@ WHERE float_col = 2
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_insert_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_insert_partition_col_types]'
' partitions=1/11 files=1 size=3B'
====
---- QUERY
@@ -193,13 +193,13 @@ STRING, STRING, STRING
ALTER TABLE all_partition_col_types
ADD PARTITION (bool_col=FALSE, tinyint_col=1, smallint_col=1, int_col=1,
bigint_col=10, float_col=0, double_col=1.1, string_col='1')
-LOCATION '/test-warehouse/all_insert_partition_col_types/tinyint_col=1/smallint_col=1/int_col=1/bigint_col=10/float_col=0/double_col=1.1/string_col=1/'
+LOCATION '/test-warehouse/$DATABASE.db/all_insert_partition_col_types/tinyint_col=1/smallint_col=1/int_col=1/bigint_col=10/float_col=0/double_col=1.1/string_col=1/'
====
---- QUERY
ALTER TABLE all_partition_col_types
ADD PARTITION (bool_col=TRUE, tinyint_col=2, smallint_col=2, int_col=2,
bigint_col=20, float_col=0, double_col=1.1, string_col='2')
-LOCATION '/test-warehouse/all_insert_partition_col_types/tinyint_col=2/smallint_col=2/int_col=2/bigint_col=20/float_col=0/double_col=1.1/string_col=2/'
+LOCATION '/test-warehouse/$DATABASE.db/all_insert_partition_col_types/tinyint_col=2/smallint_col=2/int_col=2/bigint_col=20/float_col=0/double_col=1.1/string_col=2/'
====
---- QUERY
EXPLAIN
@@ -208,7 +208,7 @@ WHERE bool_col=false
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.all_partition_col_types]'
+'00:SCAN HDFS [$DATABASE.all_partition_col_types]'
' partitions=1/2 files=1 size=2B'
====
---- QUERY
@@ -237,7 +237,7 @@ WHERE decimal_col = 4.34
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.test_dec_partition]'
+'00:SCAN HDFS [$DATABASE.test_dec_partition]'
' partitions=1/1 files=1 size=9B'
====
---- QUERY
@@ -247,7 +247,7 @@ WHERE decimal_col = 04.340
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.test_dec_partition]'
+'00:SCAN HDFS [$DATABASE.test_dec_partition]'
' partitions=1/1 files=1 size=9B'
====
---- QUERY
@@ -257,7 +257,7 @@ WHERE decimal_col = 4.35
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.test_dec_partition]'
+'00:SCAN HDFS [$DATABASE.test_dec_partition]'
' partitions=0/1 files=0 size=0B'
====
---- QUERY
@@ -280,6 +280,6 @@ WHERE decimal_col = 8.68
---- RESULTS: VERIFY_IS_SUBSET
'01:EXCHANGE [UNPARTITIONED]'
'|'
-'00:SCAN HDFS [hdfs_partitioning.test_dec_partition]'
+'00:SCAN HDFS [$DATABASE.test_dec_partition]'
' partitions=1/3 files=1 size=18B'
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8c37bf35/tests/query_test/test_partitioning.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_partitioning.py b/tests/query_test/test_partitioning.py
index 4369a75..08dcbdd 100644
--- a/tests/query_test/test_partitioning.py
+++ b/tests/query_test/test_partitioning.py
@@ -16,7 +16,6 @@
# under the License.
import pytest
-from subprocess import call
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -25,8 +24,6 @@ from tests.common.test_dimensions import create_single_exec_option_dimension
# Tests to validate HDFS partitioning.
class TestPartitioning(ImpalaTestSuite):
- TEST_DBS = ['hdfs_partitioning', 'bool_partitions']
-
@classmethod
def get_workload(self):
return 'functional-query'
@@ -41,71 +38,53 @@ class TestPartitioning(ImpalaTestSuite):
v.get_value('table_format').file_format == 'text' and\
v.get_value('table_format').compression_codec == 'none')
- @classmethod
- def setup_class(cls):
- super(TestPartitioning, cls).setup_class()
- map(cls.cleanup_db, cls.TEST_DBS)
- cls.filesystem_client.delete_file_dir("test-warehouse/all_insert_partition_col_types/",\
- recursive=True)
-
- @classmethod
- def teardown_class(cls):
- map(cls.cleanup_db, cls.TEST_DBS)
- super(TestPartitioning, cls).teardown_class()
-
@SkipIfLocal.root_path
- @pytest.mark.execute_serially
- def test_partition_col_types(self, vector):
- self.execute_query("create database hdfs_partitioning");
+ def test_partition_col_types(self, vector, unique_database):
self.run_test_case('QueryTest/partition-col-types', vector,
- use_db='hdfs_partitioning')
+ use_db=unique_database)
# Missing Coverage: Impala deals with boolean partitions created by Hive on a non-hdfs
# filesystem.
@SkipIfS3.hive
@SkipIfIsilon.hive
@SkipIfLocal.hive
- @pytest.mark.execute_serially
- def test_boolean_partitions(self, vector):
+ def test_boolean_partitions(self, vector, unique_database):
# This test takes about a minute to complete due to the Hive commands that are
# executed. To cut down on runtime, limit the test to exhaustive exploration
# strategy.
if self.exploration_strategy() != 'exhaustive': pytest.skip()
- db_name = 'bool_partitions'
- tbl_name = 'tbl'
- self.execute_query("create database " + db_name)
- self.execute_query("use " + db_name)
+ full_name = unique_database + ".bool_test"
- self.execute_query("create table %s (i int) partitioned by (b boolean)" % tbl_name)
+ self.execute_query("create table %s (i int) partitioned by (b boolean)" % full_name)
# Insert some data using Hive. Due to HIVE-6590, Hive may create multiple
# partitions, mapping to the same boolean literal value.
# For example, Hive may create partitions: /b=FALSE and /b=false, etc
- call(["hive", "-e", "INSERT OVERWRITE TABLE %s.%s PARTITION(b=false) SELECT 1 from "\
- "functional.alltypes limit 1" % (db_name, tbl_name)])
- call(["hive", "-e", "INSERT OVERWRITE TABLE %s.%s PARTITION(b=FALSE) SELECT 2 from "\
- "functional.alltypes limit 1" % (db_name, tbl_name)])
- call(["hive", "-e", "INSERT OVERWRITE TABLE %s.%s PARTITION(b=true) SELECT 10 from "\
- "functional.alltypes limit 1" % (db_name, tbl_name)])
+ self.run_stmt_in_hive("INSERT INTO TABLE %s PARTITION(b=false) SELECT 1 from "\
+ "functional.alltypes limit 1" % full_name)
+ self.run_stmt_in_hive("INSERT INTO TABLE %s PARTITION(b=FALSE) SELECT 2 from "\
+ "functional.alltypes limit 1" % full_name)
+ self.run_stmt_in_hive("INSERT INTO TABLE %s PARTITION(b=true) SELECT 10 from "\
+ "functional.alltypes limit 1" % full_name)
# Update the Impala metadata
- self.execute_query("invalidate metadata " + tbl_name)
+ self.execute_query("invalidate metadata " + full_name)
# List the partitions. Show table stats returns 1 row for each partition + 1 summary
# row
- result = self.execute_query("show table stats %s" % tbl_name)
+ result = self.execute_query("show table stats %s" % full_name)
assert len(result.data) == 3 + 1
# Verify Impala properly merges the results of the bad Hive metadata.
- assert '13' == self.execute_scalar("select sum(i) from %s" % tbl_name);
- assert '10' == self.execute_scalar("select sum(i) from %s where b=true" % tbl_name)
- assert '3' == self.execute_scalar("select sum(i) from %s where b=false" % tbl_name)
+ assert '13' == self.execute_scalar("select sum(i) from %s" % full_name);
+ assert '10' == self.execute_scalar("select sum(i) from %s where b=true" % full_name)
+ assert '3' == self.execute_scalar("select sum(i) from %s where b=false" % full_name)
# INSERT into a boolean column is disabled in Impala due to this Hive bug.
try:
- self.execute_query("insert into %s partition(bool_col=true) select 1" % tbl_name)
+ self.execute_query("insert into %s partition(bool_col=true) select 1" % full_name)
except ImpalaBeeswaxException, e:
assert 'AnalysisException: INSERT into table with BOOLEAN partition column (%s) '\
- 'is not supported: %s.%s' % ('b', db_name, tbl_name) in str(e)
+ 'is not supported: %s' % ('b', full_name) in str(e)
[2/7] incubator-impala git commit: IMPALA-3481: Use Kudu ScanToken
API for scan ranges
Posted by ab...@apache.org.
IMPALA-3481: Use Kudu ScanToken API for scan ranges
Switches the planner and KuduScanNode to use Kudu's new
ScanToken API instead of explicitly constructing scan ranges
for all tablets of a table, regardless of whether they were
needed. The ScanToken API allows Impala to specify the
projected columns and predicates during planning, and Kudu
returns a set of 'scan tokens' that represent a scanner for
each tablet that needs to be scanned. The scan tokens can
be serialized and distributed to the scan nodes, which can
then deserialize them into Kudu scanner objects. Upon
deserialization, the scan token has all scan parameters
already, including the 'pushed down' predicates. Impala no
longer needs to send the Kudu predicates to the BE and
convert them at the scan node.
This change also fixes:
1) IMPALA-4016: Avoid materializing slots only referenced
by Kudu conjuncts
2) IMPALA-3874: Predicates are not always pushed to Kudu
TODO: Consider additional planning improvements.
Testing: Updated the existing tests, verified everything
works as expected. Some BE tests no longer make sense and
they were removed.
TODO: When KUDU-1065 is resolved, add tests that demonstrate pruning.
Change-Id: I160e5849d372755748ff5ba3c90a4651c804b220
Reviewed-on: http://gerrit.cloudera.org:8080/4120
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/157c8005
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/157c8005
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/157c8005
Branch: refs/heads/master
Commit: 157c80056c62c89193a04d147d8c94fcb58610c4
Parents: 94ef5b1
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Fri Aug 19 08:49:25 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Sep 8 01:50:51 2016 +0000
----------------------------------------------------------------------
be/src/exec/kudu-scan-node-test.cc | 358 +++--------------
be/src/exec/kudu-scan-node.cc | 247 ++----------
be/src/exec/kudu-scan-node.h | 108 ++---
be/src/exec/kudu-scanner.cc | 101 ++---
be/src/exec/kudu-scanner.h | 60 ++-
be/src/exec/kudu-util.cc | 132 +------
be/src/exec/kudu-util.h | 27 +-
be/src/scheduling/simple-scheduler.cc | 2 +-
bin/impala-config.sh | 2 +-
common/thrift/PlanNodes.thrift | 13 +-
fe/pom.xml | 2 +-
.../java/com/cloudera/impala/analysis/Expr.java | 4 +-
.../cloudera/impala/analysis/LiteralExpr.java | 6 +-
.../com/cloudera/impala/catalog/KuduTable.java | 6 +-
.../catalog/delegates/KuduDdlDelegate.java | 16 +-
.../cloudera/impala/planner/KuduScanNode.java | 392 ++++++++++---------
.../java/com/cloudera/impala/util/KuduUtil.java | 10 +-
.../impala/planner/KuduPlannerTest.java | 52 ---
.../cloudera/impala/planner/PlannerTest.java | 20 +
.../impala/planner/PlannerTestBase.java | 37 +-
.../com/cloudera/impala/util/KuduUtilTest.java | 10 +-
.../queries/PlannerTest/kudu-selectivity.test | 4 +-
.../queries/PlannerTest/kudu-update.test | 12 +-
.../queries/PlannerTest/kudu.test | 51 +--
.../queries/PlannerTest/tpch-kudu.test | 35 +-
25 files changed, 522 insertions(+), 1185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-scan-node-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node-test.cc b/be/src/exec/kudu-scan-node-test.cc
index 0cda0ec..972ae80 100644
--- a/be/src/exec/kudu-scan-node-test.cc
+++ b/be/src/exec/kudu-scan-node-test.cc
@@ -37,12 +37,6 @@
#include "util/cpu-info.h"
#include "util/test-info.h"
-DEFINE_bool(run_scan_bench, false, "Whether to run the Kudu scan micro benchmarks. Note:"
- " These tests are very slow.");
-DEFINE_bool(bench_strings, true, "Whether to scan string columns in benchmarks.");
-DEFINE_int32(bench_num_rows, 10 * 1000 * 1000, "Num rows to insert in benchmarks.");
-DEFINE_int32(bench_num_splits, 40, "Num tablets to create in benchmarks");
-DEFINE_int32(bench_num_runs, 10, "Num times to run each benchmark.");
DEFINE_bool(skip_delete_table, false, "Skips deleting the tables at the end of the tests.");
DEFINE_string(use_existing_table, "", "The name of the existing table to use.");
DECLARE_bool(pick_only_leaders_for_tests);
@@ -56,6 +50,7 @@ static const char* BASE_TABLE_NAME = "TestScanNodeTable";
// The id of the slot that contains the key, in the test schema.
const int KEY_SLOT_ID = 1;
+// TODO: Remove after the Kudu EE tests are providing sufficient coverage (IMPALA-3718).
class KuduScanNodeTest : public testing::Test {
public:
virtual void SetUp() {
@@ -94,8 +89,6 @@ class KuduScanNodeTest : public testing::Test {
runtime_state_->InitMemTrackers(TUniqueId(), NULL, -1);
TKuduScanNode kudu_scan_node_;
- kudu_scan_node_.__set_kudu_conjuncts(pushable_conjuncts_);
-
kudu_node_.reset(new TPlanNode);
kudu_node_->__set_kudu_scan_node(kudu_scan_node_);
@@ -176,9 +169,41 @@ class KuduScanNodeTest : public testing::Test {
}
};
- void ScanAndVerify(const vector<TScanRangeParams>& params, int first_row,
- int expected_num_rows, int num_cols_to_materialize,
+ void CreateScanRangeParams(int num_cols_to_materialize,
+ vector<TScanRangeParams>* params) {
+ DCHECK_GE(num_cols_to_materialize, 0);
+ DCHECK_LE(num_cols_to_materialize, 3);
+
+ kudu::client::KuduScanTokenBuilder b(kudu_test_helper_.table().get());
+ vector<string> col_names;
+ if (num_cols_to_materialize > 0) col_names.push_back("key");
+ if (num_cols_to_materialize > 1) col_names.push_back("int_val");
+ if (num_cols_to_materialize > 2) col_names.push_back("string_val");
+ KUDU_ASSERT_OK(b.SetProjectedColumnNames(col_names));
+
+ vector<kudu::client::KuduScanToken*> tokens;
+ KUDU_ASSERT_OK(b.Build(&tokens));
+
+ for (kudu::client::KuduScanToken* token: tokens) {
+ TScanRange range;
+ string buf;
+ KUDU_ASSERT_OK(token->Serialize(&buf));
+ range.__set_kudu_scan_token(buf);
+
+ TScanRangeParams scan_range_params;
+ scan_range_params.__set_is_cached(false);
+ scan_range_params.__set_is_remote(false);
+ scan_range_params.__set_volume_id(0);
+ scan_range_params.__set_scan_range(range);
+ params->push_back(scan_range_params);
+ }
+ }
+
+ void ScanAndVerify(int first_row, int expected_num_rows, int num_cols_to_materialize,
int num_notnull_cols = 3, int limit = NO_LIMIT, bool verify = true) {
+ vector<TScanRangeParams> params;
+ CreateScanRangeParams(num_cols_to_materialize, ¶ms);
+
BuildRuntimeStateForScans(num_cols_to_materialize);
if (limit != NO_LIMIT) kudu_node_->__set_limit(limit);
@@ -220,49 +245,6 @@ class KuduScanNodeTest : public testing::Test {
scanner.Close(runtime_state_.get());
}
- // Adds a new TScanRangeParams to 'params'.
- // If start_key and/or stop_key are '-1' an empty key string is passed
- // instead.
- void AddScanRange(int start_key, int stop_key,
- vector<TScanRangeParams>* params) {
-
- string encoded_start_key;
- if (start_key != -1) {
- scoped_ptr<KuduPartialRow> start_key_row(kudu_test_helper_.test_schema().NewRow());
- KUDU_ASSERT_OK(start_key_row->SetInt32(0, start_key));
- start_key_row->EncodeRowKey(&encoded_start_key);
- } else {
- encoded_start_key = "";
- }
-
- string encoded_stop_key;
- if (stop_key != -1) {
- gscoped_ptr<KuduPartialRow> stop_key_row(kudu_test_helper_.test_schema().NewRow());
- KUDU_ASSERT_OK(stop_key_row->SetInt32(0, stop_key));
- stop_key_row->EncodeRowKey(&encoded_stop_key);
- } else {
- encoded_stop_key = "";
- }
-
- TKuduKeyRange kudu_key_range;
- kudu_key_range.__set_range_start_key(encoded_start_key);
- kudu_key_range.__set_range_stop_key(encoded_stop_key);
-
- TScanRange scan_range;
- scan_range.__set_kudu_key_range(kudu_key_range);
-
- TScanRangeParams scan_range_params;
- scan_range_params.__set_is_cached(false);
- scan_range_params.__set_is_remote(false);
- scan_range_params.__set_volume_id(0);
- scan_range_params.__set_scan_range(scan_range);
-
- params->push_back(scan_range_params);
- }
-
- void DoBenchmarkScan(const string& name, const vector<TScanRangeParams>& params,
- int num_cols);
-
protected:
KuduTestHelper kudu_test_helper_;
boost::scoped_ptr<MemTracker> mem_tracker_;
@@ -272,7 +254,6 @@ class KuduScanNodeTest : public testing::Test {
boost::scoped_ptr<TPlanNode> kudu_node_;
TTableDescriptor t_tbl_desc_;
DescriptorTbl* desc_tbl_;
- vector<TExpr> pushable_conjuncts_;
};
TEST_F(KuduScanNodeTest, TestScanNode) {
@@ -286,17 +267,10 @@ TEST_F(KuduScanNodeTest, TestScanNode) {
NUM_ROWS);
- // Test having multiple scan ranges, the range is split by tablet boundaries. Default
- // split is at '5'.
- int mid_key = 5;
- vector<TScanRangeParams> params;
- AddScanRange(FIRST_ROW, mid_key, ¶ms);
- AddScanRange(mid_key, NUM_ROWS, ¶ms);
-
// Test materializing all, some, or none of the slots.
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 3);
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 2);
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 0, /* 0 non-null cols */0, NO_LIMIT,
+ ScanAndVerify(FIRST_ROW, NUM_ROWS, 3);
+ ScanAndVerify(FIRST_ROW, NUM_ROWS, 2);
+ ScanAndVerify(FIRST_ROW, NUM_ROWS, 0, /* 0 non-null cols */0, NO_LIMIT,
/* Don't verify */false);
}
@@ -311,164 +285,10 @@ TEST_F(KuduScanNodeTest, TestScanNullColValues) {
kudu_test_helper_.table().get(),
NUM_ROWS, 0, 1);
- vector<TScanRangeParams> params;
- AddScanRange(-1, -1, ¶ms);
-
// Try scanning including and not including the null columns.
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 3, 1);
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 2, 1);
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 1, 1);
-}
-
-namespace {
-
-// Sets a binary predicate, based on 'type', to the plan node. This will be transformed
-// into a range predicate by KuduScanNode.
-void AddExpressionNodesToExpression(TExpr* expression, int slot_id, const string& op_name,
- TExprNodeType::type constant_type, const void* value) {
-
- vector<TExprNode> nodes;
-
- // Build the op node.
- TExprNode function_node;
- function_node.__set_node_type(TExprNodeType::FUNCTION_CALL);
- TFunctionName function_name;
- function_name.__set_function_name(op_name);
- TFunction function;
- function.__set_name(function_name);
- function_node.__set_fn(function);
-
- nodes.push_back(function_node);
-
- // Add the slof ref.
- TExprNode slot_ref_node;
- slot_ref_node.__set_node_type(TExprNodeType::SLOT_REF);
- TSlotRef slot_ref;
- slot_ref.__set_slot_id(slot_id);
- slot_ref_node.__set_slot_ref(slot_ref);
-
- nodes.push_back(slot_ref_node);
-
- TExprNode constant_node;
- constant_node.__set_node_type(constant_type);
- // Add the constant part.
- switch(constant_type) {
- // We only add the boilerplate for the types used in the test schema.
- case TExprNodeType::INT_LITERAL: {
- TIntLiteral int_literal;
- int_literal.__set_value(*reinterpret_cast<const int64_t*>(value));
- constant_node.__set_int_literal(int_literal);
- break;
- }
- case TExprNodeType::STRING_LITERAL: {
- TStringLiteral string_literal;
- string_literal.__set_value(*reinterpret_cast<const string*>(value));
- constant_node.__set_string_literal(string_literal);
- break;
- }
- default:
- LOG(FATAL) << "Unsupported function type";
- }
-
- nodes.push_back(constant_node);
- expression->__set_nodes(nodes);
-}
-
-} // anonymous namespace
-
-
-// Test a >= predicate on the Key.
-TEST_F(KuduScanNodeTest, TestPushIntGEPredicateOnKey) {
- kudu_test_helper_.CreateTable(BASE_TABLE_NAME);
- const int NUM_ROWS_TO_INSERT = 1000;
- const int SLOT_ID = 1;
- const int MAT_COLS = 3;
- const int FIRST_ROW = 500;
- const int EXPECTED_NUM_ROWS = 500;
-
- // Insert kNumRows rows for this test.
- kudu_test_helper_.InsertTestRows(kudu_test_helper_.client().get(),
- kudu_test_helper_.table().get(),
- NUM_ROWS_TO_INSERT);
-
- const int64_t PREDICATE_VALUE = 500;
-
- // Test having a pushable predicate on the key (key >= PREDICATE_VALUE).
- TExpr conjunct;
- AddExpressionNodesToExpression(&conjunct, SLOT_ID, KuduScanNode::GE_FN,
- TExprNodeType::INT_LITERAL, &PREDICATE_VALUE);
- pushable_conjuncts_.push_back(conjunct);
-
- vector<TScanRangeParams> params;
- AddScanRange(-1, -1, ¶ms);
-
- ScanAndVerify(params, FIRST_ROW, EXPECTED_NUM_ROWS, MAT_COLS);
-}
-
-// Test a == predicate on the 2nd column.
-TEST_F(KuduScanNodeTest, TestPushIntEQPredicateOn2ndColumn) {
- kudu_test_helper_.CreateTable(BASE_TABLE_NAME);
- const int NUM_ROWS_TO_INSERT = 1000;
- const int SLOT_ID = 2;
- const int MAT_COLS = 3;
-
- const int FIRST_ROW = 500;
- const int EXPECTED_NUM_ROWS = 1;
-
- // Insert kNumRows rows for this test.
- kudu_test_helper_.InsertTestRows(kudu_test_helper_.client().get(),
- kudu_test_helper_.table().get(),
- NUM_ROWS_TO_INSERT);
-
- // When rows are added col2 is 2 * col1 so we multiply by two to get back
- // first row = 500.
- const int64_t PREDICATE_VAL = FIRST_ROW * 2;
-
- // Now test having a pushable predicate on the 2nd column.
- TExpr conjunct;
- AddExpressionNodesToExpression(&conjunct, SLOT_ID, KuduScanNode::EQ_FN,
- TExprNodeType::INT_LITERAL, &PREDICATE_VAL);
- pushable_conjuncts_.push_back(conjunct);
-
- vector<TScanRangeParams> params;
- AddScanRange(-1, -1, ¶ms);
-
- ScanAndVerify(params, FIRST_ROW, EXPECTED_NUM_ROWS, MAT_COLS);
-}
-
-TEST_F(KuduScanNodeTest, TestPushStringLEPredicateOn3rdColumn) {
- kudu_test_helper_.CreateTable(BASE_TABLE_NAME);
- const int NUM_ROWS_TO_INSERT = 1000;
- const int SLOT_ID = 3;
- const int MAT_COLS = 3;
- // This predicate won't return consecutive rows so we just assert on the count.
- const bool VERIFY_ROWS = false;
-
- const int FIRST_ROW = 0;
- const string PREDICATE_VAL = "hello_500";
-
- // We expect 448 rows as Kudu will lexicographically compare the predicate value to
- // column values. We can expect to obtain rows 1-500, inclusive, except numbers
- // 6-9 (hello_6 > hello_500) and numbers 51-99 for a total of 52 excluded numbers.
- const int EXPECTED_NUM_ROWS = 448;
-
-
- // Insert kNumRows rows for this test.
- kudu_test_helper_.InsertTestRows(kudu_test_helper_.client().get(),
- kudu_test_helper_.table().get(),
- NUM_ROWS_TO_INSERT);
-
- // Now test having a pushable predicate on the third column.
- TExpr conjunct;
- AddExpressionNodesToExpression(&conjunct, SLOT_ID, KuduScanNode::LE_FN,
- TExprNodeType::STRING_LITERAL, &PREDICATE_VAL);
- pushable_conjuncts_.push_back(conjunct);
-
- vector<TScanRangeParams> params;
- AddScanRange(-1, -1, ¶ms);
-
- ScanAndVerify(params, FIRST_ROW, EXPECTED_NUM_ROWS, MAT_COLS, MAT_COLS,
- NO_LIMIT, VERIFY_ROWS);
+ ScanAndVerify(FIRST_ROW, NUM_ROWS, 3, 1);
+ ScanAndVerify(FIRST_ROW, NUM_ROWS, 2, 1);
+ ScanAndVerify(FIRST_ROW, NUM_ROWS, 1, 1);
}
// Test for a bug where we would mishandle getting an empty string from
@@ -485,15 +305,21 @@ TEST_F(KuduScanNodeTest, TestScanEmptyString) {
KUDU_ASSERT_OK(session->Flush());
ASSERT_FALSE(session->HasPendingOperations());
- BuildRuntimeStateForScans(3);
+ int num_cols_to_materialize = 3;
+ BuildRuntimeStateForScans(num_cols_to_materialize);
KuduScanNode scanner(obj_pool_.get(), *kudu_node_, *desc_tbl_);
+
vector<TScanRangeParams> params;
- AddScanRange(-1, -1, ¶ms);
+ CreateScanRangeParams(num_cols_to_materialize, ¶ms);
SetUpScanner(&scanner, params);
bool eos = false;
RowBatch* batch = obj_pool_->Add(
new RowBatch(scanner.row_desc(), DEFAULT_ROWS_PER_BATCH, mem_tracker_.get()));
- ASSERT_OK(scanner.GetNext(runtime_state_.get(), batch, &eos));
+ for (int i = 0; i < 2 && batch->num_rows() == 0; ++i) {
+ // Allow for up to 2 empty row batches since there are scanners created for all
+ // tablets (1 split point), and only one row was inserted.
+ ASSERT_OK(scanner.GetNext(runtime_state_.get(), batch, &eos));
+ }
ASSERT_EQ(1, batch->num_rows());
ASSERT_OK(scanner.GetNext(runtime_state_.get(), NULL, &eos));
@@ -513,91 +339,13 @@ TEST_F(KuduScanNodeTest, TestLimitsAreEnforced) {
kudu_test_helper_.table().get(),
NUM_ROWS);
- vector<TScanRangeParams> params;
- AddScanRange(-1, -1, ¶ms);
-
// Try scanning but limit the number of returned rows to several different values.
int limit_rows_to = 0;
- ScanAndVerify(params, FIRST_ROW, limit_rows_to, 3, 3, limit_rows_to);
+ ScanAndVerify(FIRST_ROW, limit_rows_to, 3, 3, limit_rows_to);
limit_rows_to = 1;
- ScanAndVerify(params, FIRST_ROW, limit_rows_to, 3, 3, limit_rows_to);
+ ScanAndVerify(FIRST_ROW, limit_rows_to, 3, 3, limit_rows_to);
limit_rows_to = 2000;
- ScanAndVerify(params, FIRST_ROW, 1000, 3, 3, limit_rows_to);
-}
-
-void KuduScanNodeTest::DoBenchmarkScan(const string& name,
- const vector<TScanRangeParams>& params, int num_cols) {
-
- const int NUM_ROWS = FLAGS_bench_num_rows;
- const int FIRST_ROW = 0;
-
- double avg = 0;
- int num_runs = 0;
-
- LOG(INFO) << "===== Starting benchmark: " << name;
- for (int i = 0; i < FLAGS_bench_num_runs; ++i) {
- MonotonicStopWatch watch;
- watch.Start();
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, num_cols, 3, NO_LIMIT, false);
- watch.Stop();
- int64_t total = watch.ElapsedTime();
- avg += total;
- ++num_runs;
- LOG(INFO) << "Run took: " << (total / (1000 * 1000)) << " msecs.";
- }
-
- avg = (avg / num_runs) / (1000 * 1000);
- LOG(INFO) << "===== Benchmark: " << name << " took(avg): " << avg << " msecs.";
-}
-
-TEST_F(KuduScanNodeTest, BenchmarkScanNode) {
- if (!FLAGS_run_scan_bench) return;
-
- // Generate some more splits and insert a lot more rows.
- const int NUM_ROWS = FLAGS_bench_num_rows;
- const int NUM_SPLITS = FLAGS_bench_num_splits;
- const int FIRST_ROW = 0;
-
- if (FLAGS_use_existing_table == "") {
- vector<const KuduPartialRow*> split_rows;
- for (int i = 1; i < NUM_SPLITS; ++i) {
- int split_key = (NUM_ROWS / NUM_SPLITS) * i;
- KuduPartialRow* row = kudu_test_helper_.test_schema().NewRow();
- KUDU_ASSERT_OK(row->SetInt32(0, split_key));
- split_rows.push_back(row);
- }
- kudu_test_helper_.CreateTable(BASE_TABLE_NAME, &split_rows);
- // Insert NUM_ROWS rows for this test.
- kudu_test_helper_.InsertTestRows(kudu_test_helper_.client().get(),
- kudu_test_helper_.table().get(),
- NUM_ROWS);
- LOG(INFO) << "Inserted: " << NUM_ROWS << " rows into " << NUM_SPLITS << " tablets.";
- } else {
- kudu_test_helper_.OpenTable(FLAGS_use_existing_table);
- }
-
- // TODO We calculate the scan ranges based on the test params and not by actually
- // querying the tablet servers since Kudu doesn't have an API to get them.
- vector<TScanRangeParams> params;
- int previous_split_key = -1;
- for (int i = 1; i < NUM_SPLITS; ++i) {
- int split_key = (NUM_ROWS / NUM_SPLITS) * i;
- AddScanRange(previous_split_key, split_key, ¶ms);
- previous_split_key = split_key;
- if (i == NUM_SPLITS -1) {
- AddScanRange(split_key, -1, ¶ms);
- }
- }
-
- LOG(INFO) << "Warming up scans.";
-
- // Scan all columns once to warmup.
- ScanAndVerify(params, FIRST_ROW, NUM_ROWS, 3, 3, NO_LIMIT, false);
-
- DoBenchmarkScan("No cols", params, 0);
- DoBenchmarkScan("Key only", params, 1);
- DoBenchmarkScan("Int cols", params, 2);
- DoBenchmarkScan("All cols with strings", params, 3);
+ ScanAndVerify(FIRST_ROW, 1000, 3, 3, limit_rows_to);
}
} // namespace impala
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.cc b/be/src/exec/kudu-scan-node.cc
index a7a068c..b247c67 100644
--- a/be/src/exec/kudu-scan-node.cc
+++ b/be/src/exec/kudu-scan-node.cc
@@ -65,25 +65,29 @@ namespace impala {
const string KuduScanNode::KUDU_READ_TIMER = "TotalKuduReadTime";
const string KuduScanNode::KUDU_ROUND_TRIPS = "TotalKuduScanRoundTrips";
-const std::string KuduScanNode::GE_FN = "ge";
-const std::string KuduScanNode::LE_FN = "le";
-const std::string KuduScanNode::EQ_FN = "eq";
-
KuduScanNode::KuduScanNode(ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs)
: ScanNode(pool, tnode, descs),
tuple_id_(tnode.kudu_scan_node.tuple_id),
- next_scan_range_idx_(0),
+ next_scan_token_idx_(0),
num_active_scanners_(0),
done_(false),
- pushable_conjuncts_(tnode.kudu_scan_node.kudu_conjuncts),
thread_avail_cb_id_(-1) {
DCHECK(KuduIsAvailable());
+
+ int max_row_batches = FLAGS_kudu_max_row_batches;
+ if (max_row_batches <= 0) {
+ // TODO: See comment on hdfs-scan-node.
+ // This value is built the same way as it assumes that the scan node runs co-located
+ // with a Kudu tablet server and that the tablet server is using disks similarly as
+ // a datanode would.
+ max_row_batches = 10 * (DiskInfo::num_disks() + DiskIoMgr::REMOTE_NUM_DISKS);
+ }
+ materialized_row_batches_.reset(new RowBatchQueue(max_row_batches));
}
KuduScanNode::~KuduScanNode() {
DCHECK(is_closed());
- STLDeleteElements(&kudu_predicates_);
}
Status KuduScanNode::Prepare(RuntimeState* state) {
@@ -100,24 +104,11 @@ Status KuduScanNode::Prepare(RuntimeState* state) {
tuple_desc_ = state->desc_tbl().GetTupleDescriptor(tuple_id_);
- // Convert TScanRangeParams to ScanRanges.
- CHECK(scan_range_params_ != NULL)
- << "Must call SetScanRanges() before calling Prepare()";
+ // Initialize the list of scan tokens to process from the TScanRangeParams.
+ DCHECK(scan_range_params_ != NULL);
for (const TScanRangeParams& params: *scan_range_params_) {
- const TKuduKeyRange& key_range = params.scan_range.kudu_key_range;
- key_ranges_.push_back(key_range);
+ scan_tokens_.push_back(params.scan_range.kudu_scan_token);
}
- max_materialized_row_batches_ = FLAGS_kudu_max_row_batches;
-
- if (max_materialized_row_batches_ <= 0) {
- // TODO: See comment on hdfs-scan-node.
- // This value is built the same way as it assumes that the scan node runs co-located
- // with a Kudu tablet server and that the tablet server is using disks similarly as
- // a datanode would.
- max_materialized_row_batches_ =
- 10 * (DiskInfo::num_disks() + DiskIoMgr::REMOTE_NUM_DISKS);
- }
- materialized_row_batches_.reset(new RowBatchQueue(max_materialized_row_batches_));
return Status::OK();
}
@@ -139,15 +130,11 @@ Status KuduScanNode::Open(RuntimeState* state) {
KUDU_RETURN_IF_ERROR(client_->OpenTable(table_desc->table_name(), &table_),
"Unable to open Kudu table");
- RETURN_IF_ERROR(ProjectedColumnsFromTupleDescriptor(*tuple_desc_, &projected_columns_,
- table_->schema()));
- // Must happen after table_ is opened.
- RETURN_IF_ERROR(TransformPushableConjunctsToRangePredicates());
num_scanner_threads_started_counter_ =
ADD_COUNTER(runtime_profile(), NUM_SCANNER_THREADS_STARTED, TUnit::UNIT);
- // Reserve one thread token.
+ // Reserve one thread.
state->resource_pool()->ReserveOptionalTokens(1);
if (state->query_options().num_scanner_threads > 0) {
state->resource_pool()->set_max_quota(
@@ -155,8 +142,8 @@ Status KuduScanNode::Open(RuntimeState* state) {
}
thread_avail_cb_id_ = state->resource_pool()->AddThreadAvailableCb(
- bind<void>(mem_fn(&KuduScanNode::ThreadTokenAvailableCb), this, _1));
- ThreadTokenAvailableCb(state->resource_pool());
+ bind<void>(mem_fn(&KuduScanNode::ThreadAvailableCb), this, _1));
+ ThreadAvailableCb(state->resource_pool());
return Status::OK();
}
@@ -167,7 +154,7 @@ Status KuduScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos
SCOPED_TIMER(runtime_profile_->total_time_counter());
SCOPED_TIMER(materialize_tuple_timer());
- if (ReachedLimit() || key_ranges_.empty()) {
+ if (ReachedLimit() || scan_tokens_.empty()) {
*eos = true;
return Status::OK();
}
@@ -203,145 +190,6 @@ Status KuduScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos
return status;
}
-Status KuduScanNode::TransformPushableConjunctsToRangePredicates() {
- // The only supported pushable predicates are binary operators with a SlotRef and a
- // literal as the left and right operands respectively.
- for (const TExpr& predicate: pushable_conjuncts_) {
- DCHECK_EQ(predicate.nodes.size(), 3);
- const TExprNode& function_call = predicate.nodes[0];
-
- DCHECK_EQ(function_call.node_type, TExprNodeType::FUNCTION_CALL);
-
- IdxByLowerCaseColName idx_by_lc_name;
- RETURN_IF_ERROR(MapLowercaseKuduColumnNamesToIndexes(table_->schema(),
- &idx_by_lc_name));
-
- string impala_col_name;
- GetSlotRefColumnName(predicate.nodes[1], &impala_col_name);
-
- IdxByLowerCaseColName::const_iterator iter;
- if ((iter = idx_by_lc_name.find(impala_col_name)) ==
- idx_by_lc_name.end()) {
- return Status(Substitute("Could not find col: '$0' in the table schema",
- impala_col_name));
- }
-
- KuduColumnSchema column = table_->schema().Column(iter->second);
-
- KuduValue* bound;
- RETURN_IF_ERROR(GetExprLiteralBound(predicate.nodes[2], column.type(), &bound));
- DCHECK(bound != NULL);
-
- const string& function_name = function_call.fn.name.function_name;
- if (function_name == GE_FN) {
- kudu_predicates_.push_back(table_->NewComparisonPredicate(column.name(),
- KuduPredicate::GREATER_EQUAL, bound));
- } else if (function_name == LE_FN) {
- kudu_predicates_.push_back(table_->NewComparisonPredicate(column.name(),
- KuduPredicate::LESS_EQUAL, bound));
- } else if (function_name == EQ_FN) {
- kudu_predicates_.push_back(table_->NewComparisonPredicate(column.name(),
- KuduPredicate::EQUAL, bound));
- } else {
- DCHECK(false) << "Received unpushable operator to push down: " << function_name;
- }
- }
- return Status::OK();
-}
-
-void KuduScanNode::GetSlotRefColumnName(const TExprNode& node, string* col_name) {
- const KuduTableDescriptor* table_desc =
- static_cast<const KuduTableDescriptor*>(tuple_desc_->table_desc());
- TSlotId slot_id = node.slot_ref.slot_id;
- for (SlotDescriptor* slot: tuple_desc_->slots()) {
- if (slot->id() == slot_id) {
- int col_idx = slot->col_pos();
- *col_name = table_desc->col_descs()[col_idx].name();
- return;
- }
- }
-
- DCHECK(false) << "Could not find a slot with slot id: " << slot_id;
-}
-
-namespace {
-
-typedef std::map<int, const char*> TypeNamesMap;
-
-// Gets the name of an Expr node type.
-string NodeTypeToString(TExprNodeType::type type) {
- const TypeNamesMap& type_names_map =
- impala::_TExprNodeType_VALUES_TO_NAMES;
- TypeNamesMap::const_iterator iter = type_names_map.find(type);
-
- if (iter == type_names_map.end()) {
- return Substitute("Unknown type: $0", type);
- }
-
- return (*iter).second;
-}
-
-} // anonymous namespace
-
-Status KuduScanNode::GetExprLiteralBound(const TExprNode& node,
- KuduColumnSchema::DataType type, KuduValue** value) {
-
- // Build the Kudu values based on the type of the Kudu column.
- // We're restrictive regarding which types we accept as the planner does all casting
- // in the frontend and predicates only get pushed down if the types match.
- switch (type) {
- // For types BOOL and STRING we expect the expression literal to match perfectly.
- case kudu::client::KuduColumnSchema::BOOL: {
- if (node.node_type != TExprNodeType::BOOL_LITERAL) {
- return Status(Substitute("Cannot create predicate over column of type BOOL with "
- "value of type: $0", NodeTypeToString(node.node_type)));
- }
- *value = KuduValue::FromBool(node.bool_literal.value);
- return Status::OK();
- }
- case kudu::client::KuduColumnSchema::STRING: {
- if (node.node_type != TExprNodeType::STRING_LITERAL) {
- return Status(Substitute("Cannot create predicate over column of type STRING"
- " with value of type: $0", NodeTypeToString(node.node_type)));
- }
- *value = KuduValue::CopyString(node.string_literal.value);
- return Status::OK();
- }
- case kudu::client::KuduColumnSchema::INT8:
- case kudu::client::KuduColumnSchema::INT16:
- case kudu::client::KuduColumnSchema::INT32:
- case kudu::client::KuduColumnSchema::INT64: {
- if (node.node_type != TExprNodeType::INT_LITERAL) {
- return Status(Substitute("Cannot create predicate over column of type INT with "
- "value of type: $0", NodeTypeToString(node.node_type)));
- }
- *value = KuduValue::FromInt(node.int_literal.value);
- return Status::OK();
- }
- case kudu::client::KuduColumnSchema::FLOAT: {
- if (node.node_type != TExprNodeType::FLOAT_LITERAL) {
- return Status(Substitute("Cannot create predicate over column of type FLOAT with "
- "value of type: $0", NodeTypeToString(node.node_type)));
- }
- *value = KuduValue::FromFloat(node.float_literal.value);
- return Status::OK();
- }
- case kudu::client::KuduColumnSchema::DOUBLE: {
- if (node.node_type != TExprNodeType::FLOAT_LITERAL) {
- return Status(Substitute("Cannot create predicate over column of type DOUBLE with "
- "value of type: $0", NodeTypeToString(node.node_type)));
- }
- *value = KuduValue::FromDouble(node.float_literal.value);
- return Status::OK();
- }
- default:
- // Should be unreachable.
- LOG(DFATAL) << "Unsupported node type: " << node.node_type;
- }
- // Avoid compiler warning.
- return Status("Unreachable");
-}
-
void KuduScanNode::Close(RuntimeState* state) {
if (is_closed()) return;
SCOPED_TIMER(runtime_profile_->total_time_counter());
@@ -368,30 +216,22 @@ void KuduScanNode::DebugString(int indentation_level, stringstream* out) const {
*out << indent << "KuduScanNode(tupleid=" << tuple_id_ << ")";
}
-TKuduKeyRange* KuduScanNode::GetNextKeyRange() {
+const string* KuduScanNode::GetNextScanToken() {
unique_lock<mutex> lock(lock_);
- if (next_scan_range_idx_ >= key_ranges_.size()) return NULL;
- TKuduKeyRange* range = &key_ranges_[next_scan_range_idx_++];
- return range;
+ if (next_scan_token_idx_ >= scan_tokens_.size()) return NULL;
+ const string* token = &scan_tokens_[next_scan_token_idx_++];
+ return token;
}
Status KuduScanNode::GetConjunctCtxs(vector<ExprContext*>* ctxs) {
return Expr::CloneIfNotExists(conjunct_ctxs_, runtime_state_, ctxs);
}
-void KuduScanNode::ClonePredicates(vector<KuduPredicate*>* predicates) {
- unique_lock<mutex> l(lock_);
- for (KuduPredicate* predicate: kudu_predicates_) {
- predicates->push_back(predicate->Clone());
- }
-}
-
-void KuduScanNode::ThreadTokenAvailableCb(
- ThreadResourceMgr::ResourcePool* pool) {
+void KuduScanNode::ThreadAvailableCb(ThreadResourceMgr::ResourcePool* pool) {
while (true) {
unique_lock<mutex> lock(lock_);
- // All done or all ranges are assigned.
- if (done_ || next_scan_range_idx_ >= key_ranges_.size()) break;
+ // All done or all tokens are assigned.
+ if (done_ || next_scan_token_idx_ >= scan_tokens_.size()) break;
// Check if we can get a token.
if (!pool->TryAcquireThreadToken()) break;
@@ -399,14 +239,14 @@ void KuduScanNode::ThreadTokenAvailableCb(
++num_active_scanners_;
COUNTER_ADD(num_scanner_threads_started_counter_, 1);
- // Reserve the first range so no other thread picks it up.
- TKuduKeyRange* range = &key_ranges_[next_scan_range_idx_++];
+ // Reserve the first token so no other thread picks it up.
+ const string* token = &scan_tokens_[next_scan_token_idx_++];
string name = Substitute("scanner-thread($0)",
num_scanner_threads_started_counter_->value());
- VLOG(1) << "Thread started: " << name;
+ VLOG_RPC << "Thread started: " << name;
scanner_threads_.AddThread(new Thread("kudu-scan-node", name,
- &KuduScanNode::ScannerThread, this, name, range));
+ &KuduScanNode::RunScannerThread, this, name, token));
if (runtime_state_->query_resource_mgr() != NULL) {
runtime_state_->query_resource_mgr()->NotifyThreadUsageChange(1);
@@ -414,12 +254,12 @@ void KuduScanNode::ThreadTokenAvailableCb(
}
}
-Status KuduScanNode::ProcessRange(KuduScanner* scanner, const TKuduKeyRange* key_range) {
- RETURN_IF_ERROR(scanner->OpenNextRange(*key_range));
+Status KuduScanNode::ProcessScanToken(KuduScanner* scanner, const string& scan_token) {
+ RETURN_IF_ERROR(scanner->OpenNextScanToken(scan_token));
bool eos = false;
while (!eos) {
gscoped_ptr<RowBatch> row_batch(new RowBatch(
- row_desc(), runtime_state_->batch_size(), mem_tracker()));
+ row_desc(), runtime_state_->batch_size(), mem_tracker()));
RETURN_IF_ERROR(scanner->GetNext(row_batch.get(), &eos));
while (!done_) {
scanner->KeepKuduScannerAlive();
@@ -429,13 +269,12 @@ Status KuduScanNode::ProcessRange(KuduScanner* scanner, const TKuduKeyRange* key
}
}
}
- // Mark the current scan range as complete.
if (eos) scan_ranges_complete_counter()->Add(1);
return Status::OK();
}
-void KuduScanNode::ScannerThread(const string& name, const TKuduKeyRange* initial_range) {
- DCHECK(initial_range != NULL);
+void KuduScanNode::RunScannerThread(const string& name, const string* initial_token) {
+ DCHECK(initial_token != NULL);
SCOPED_THREAD_COUNTER_MEASUREMENT(scanner_thread_counters());
SCOPED_TIMER(runtime_state_->total_cpu_timer());
@@ -443,29 +282,29 @@ void KuduScanNode::ScannerThread(const string& name, const TKuduKeyRange* initia
// exceeded and is exiting early.
bool optional_thread_exiting = false;
KuduScanner scanner(this, runtime_state_);
- Status status = scanner.Open();
+ const string* scan_token = initial_token;
+ Status status = scanner.Open();
if (status.ok()) {
- const TKuduKeyRange* key_range = initial_range;
- while (!done_ && key_range != NULL) {
- status = ProcessRange(&scanner, key_range);
+ while (!done_ && scan_token != NULL) {
+ status = ProcessScanToken(&scanner, *scan_token);
if (!status.ok()) break;
// Check if the number of optional threads has been exceeded.
if (runtime_state_->resource_pool()->optional_exceeded()) {
unique_lock<mutex> l(lock_);
// Don't exit if this is the last thread. Otherwise, the scan will indicate it's
- // done before all ranges have been processed.
+ // done before all scan tokens have been processed.
if (num_active_scanners_ > 1) {
--num_active_scanners_;
optional_thread_exiting = true;
break;
}
}
- key_range = GetNextKeyRange();
+ scan_token = GetNextScanToken();
}
- scanner.Close();
}
+ scanner.Close();
{
unique_lock<mutex> l(lock_);
@@ -485,8 +324,8 @@ void KuduScanNode::ScannerThread(const string& name, const TKuduKeyRange* initia
}
// lock_ is released before calling ThreadResourceMgr::ReleaseThreadToken() which
- // invokes ThreadTokenAvailableCb() which attempts to take the same lock.
- VLOG(1) << "Thread done: " << name;
+ // invokes ThreadAvailableCb() which attempts to take the same lock.
+ VLOG_RPC << "Thread done: " << name;
runtime_state_->resource_pool()->ReleaseThreadToken(false);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scan-node.h b/be/src/exec/kudu-scan-node.h
index d243cb8..b1848d7 100644
--- a/be/src/exec/kudu-scan-node.h
+++ b/be/src/exec/kudu-scan-node.h
@@ -31,50 +31,30 @@
namespace impala {
class KuduScanner;
-class Tuple;
-/// A scan node that scans Kudu TabletServers.
+/// A scan node that scans a Kudu table.
///
-/// This scan node takes a set of ranges and uses a Kudu client to retrieve the data
-/// belonging to those ranges from Kudu. The client's schema is rebuilt from the
-/// TupleDescriptors forwarded by the frontend so that we're sure all the scan nodes
-/// use the same schema, for the same scan.
+/// This takes a set of serialized Kudu scan tokens which encode the information needed
+/// for this scan. A Kudu client deserializes the tokens into kudu scanners, and those
+/// are used to retrieve the rows for this scan.
class KuduScanNode : public ScanNode {
public:
KuduScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
~KuduScanNode();
- /// Create Kudu schema and columns to slots mapping.
virtual Status Prepare(RuntimeState* state);
-
- /// Start Kudu scan.
virtual Status Open(RuntimeState* state);
-
- /// Fill the next row batch by fetching more data from the KuduScanner.
virtual Status GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos);
-
- /// Close connections to Kudu.
virtual void Close(RuntimeState* state);
protected:
- /// Write debug string of this into out.
virtual void DebugString(int indentation_level, std::stringstream* out) const;
private:
- FRIEND_TEST(KuduScanNodeTest, TestPushIntGEPredicateOnKey);
- FRIEND_TEST(KuduScanNodeTest, TestPushIntEQPredicateOn2ndColumn);
- FRIEND_TEST(KuduScanNodeTest, TestPushStringLEPredicateOn3rdColumn);
- FRIEND_TEST(KuduScanNodeTest, TestPushTwoPredicatesOnNonMaterializedColumn);
friend class KuduScanner;
- kudu::client::KuduClient* kudu_client() {
- return client_.get();
- }
-
- kudu::client::KuduTable* kudu_table() {
- return table_.get();
- }
+ kudu::client::KuduClient* kudu_client() { return client_.get(); }
/// Tuple id resolved in Prepare() to set tuple_desc_.
const TupleId tuple_id_;
@@ -84,19 +64,15 @@ class KuduScanNode : public ScanNode {
/// Descriptor of tuples read from Kudu table.
const TupleDescriptor* tuple_desc_;
- /// The list of Kudu columns to project for the scan, extracted from the TupleDescriptor
- /// and translated into Kudu format, i.e. matching the case of the Kudu schema.
- std::vector<std::string> projected_columns_;
-
/// The Kudu client and table. Scanners share these instances.
std::tr1::shared_ptr<kudu::client::KuduClient> client_;
std::tr1::shared_ptr<kudu::client::KuduTable> table_;
- /// Set of ranges to be scanned.
- std::vector<TKuduKeyRange> key_ranges_;
+ /// Set of scan tokens to be deserialized into Kudu scanners.
+ std::vector<std::string> scan_tokens_;
- /// The next index in 'key_ranges_' to be assigned to a scanner.
- int next_scan_range_idx_;
+ /// The next index in 'scan_tokens_' to be assigned. Protected by lock_.
+ int next_scan_token_idx_;
// Outgoing row batches queue. Row batches are produced asynchronously by the scanner
// threads and consumed by the main thread.
@@ -106,8 +82,8 @@ class KuduScanNode : public ScanNode {
/// 'num_active_scanners_'.
boost::mutex lock_;
- /// The current status of the scan, set to non-OK if any problems occur, e.g. if an error
- /// occurs in a scanner.
+ /// The current status of the scan, set to non-OK if any problems occur, e.g. if an
+ /// error occurs in a scanner.
/// Protected by lock_
Status status_;
@@ -115,14 +91,11 @@ class KuduScanNode : public ScanNode {
/// Protected by lock_
int num_active_scanners_;
- /// Set to true when the scan is complete (either because all ranges were scanned, the limit
- /// was reached or some error occurred).
+ /// Set to true when the scan is complete (either because all scan tokens have been
+ /// processed, the limit was reached or some error occurred).
/// Protected by lock_
volatile bool done_;
- /// Maximum size of materialized_row_batches_.
- int max_materialized_row_batches_;
-
/// Thread group for all scanner worker threads
ThreadGroup scanner_threads_;
@@ -131,55 +104,29 @@ class KuduScanNode : public ScanNode {
static const std::string KUDU_READ_TIMER;
static const std::string KUDU_ROUND_TRIPS;
- /// The function names of the supported predicates.
- static const std::string GE_FN;
- static const std::string LE_FN;
- static const std::string EQ_FN;
-
- /// The set of conjuncts, in TExpr form, to be pushed to Kudu.
- /// Received in TPlanNode::kudu_scan_node.
- const std::vector<TExpr> pushable_conjuncts_;
-
- /// The id of the callback added to the thread resource manager when thread token
+ /// The id of the callback added to the thread resource manager when a thread
/// is available. Used to remove the callback before this scan node is destroyed.
/// -1 if no callback is registered.
int thread_avail_cb_id_;
- /// The set of conjuncts, in KuduPredicate form, to be pushed to Kudu.
- /// Derived from 'pushable_conjuncts_'.
- std::vector<kudu::client::KuduPredicate*> kudu_predicates_;
-
- /// Returns a KuduValue with 'type' built from a literal in 'node'.
- /// Expects that 'node' is a literal value.
- Status GetExprLiteralBound(const TExprNode& node,
- kudu::client::KuduColumnSchema::DataType type, kudu::client::KuduValue** value);
-
- /// Returns a string with the name of the column that 'node' refers to.
- void GetSlotRefColumnName(const TExprNode& node, string* col_name);
-
- /// Transforms 'pushable_conjuncts_' received from the frontend into 'kudu_predicates_' that will
- /// be set in all scanners.
- Status TransformPushableConjunctsToRangePredicates();
-
/// Called when scanner threads are available for this scan node. This will
/// try to spin up as many scanner threads as the quota allows.
- void ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool);
+ void ThreadAvailableCb(ThreadResourceMgr::ResourcePool* pool);
/// Main function for scanner thread which executes a KuduScanner. Begins by processing
- /// 'initial_range', and continues processing ranges returned by 'GetNextKeyRange()'
- /// until there are no more ranges, an error occurs, or the limit is reached.
- void ScannerThread(const string& name, const TKuduKeyRange* initial_range);
-
- /// Processes a single scan range. Row batches are fetched using 'scanner' and enqueued
- /// in 'materialized_row_batches_' until the scanner reports eos for 'key_range', an
- /// error occurs, or the limit is reached.
- Status ProcessRange(KuduScanner* scanner, const TKuduKeyRange* key_range);
+ /// 'initial_token', and continues processing scan tokens returned by
+ /// 'GetNextScanToken()' until there are none left, an error occurs, or the limit is
+ /// reached.
+ void RunScannerThread(const std::string& name, const std::string* initial_token);
- /// Returns the next partition key range to read. Thread safe. Returns NULL if there are
- /// no more ranges.
- TKuduKeyRange* GetNextKeyRange();
+ /// Processes a single scan token. Row batches are fetched using 'scanner' and enqueued
+ /// in 'materialized_row_batches_' until the scanner reports eos, an error occurs, or
+ /// the limit is reached.
+ Status ProcessScanToken(KuduScanner* scanner, const std::string& scan_token);
- const std::vector<std::string>& projected_columns() const { return projected_columns_; }
+ /// Returns the next scan token. Thread safe. Returns NULL if there are no more scan
+ /// tokens.
+ const std::string* GetNextScanToken();
const TupleDescriptor* tuple_desc() const { return tuple_desc_; }
@@ -187,9 +134,6 @@ class KuduScanNode : public ScanNode {
// have been open previously.
Status GetConjunctCtxs(vector<ExprContext*>* ctxs);
- // Clones the set of predicates to be set on scanners.
- void ClonePredicates(vector<kudu::client::KuduPredicate*>* predicates);
-
RuntimeProfile::Counter* kudu_read_timer() const { return kudu_read_timer_; }
RuntimeProfile::Counter* kudu_round_trips() const { return kudu_round_trips_; }
};
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scanner.cc b/be/src/exec/kudu-scanner.cc
index 66e17fa..57ba356 100644
--- a/be/src/exec/kudu-scanner.cc
+++ b/be/src/exec/kudu-scanner.cc
@@ -39,8 +39,6 @@
#include "common/names.h"
using kudu::client::KuduClient;
-using kudu::client::KuduColumnSchema;
-using kudu::client::KuduPredicate;
using kudu::client::KuduScanBatch;
using kudu::client::KuduSchema;
using kudu::client::KuduTable;
@@ -57,40 +55,15 @@ DEFINE_int32(kudu_scanner_timeout_sec, 60,
namespace impala {
-namespace {
-
-// Sets up the scan range predicate on the scanner, i.e. the partition start/stop keys
-// of the partition the scanner is supposed to scan.
-Status SetupScanRangePredicate(const TKuduKeyRange& key_range,
- kudu::client::KuduScanner* scanner) {
- if (key_range.range_start_key.empty() && key_range.range_stop_key.empty()) {
- return Status::OK();
- }
-
- if (!key_range.range_start_key.empty()) {
- KUDU_RETURN_IF_ERROR(scanner->AddLowerBoundPartitionKeyRaw(
- key_range.range_start_key), "adding scan range lower bound");
- }
- if (!key_range.range_stop_key.empty()) {
- KUDU_RETURN_IF_ERROR(scanner->AddExclusiveUpperBoundPartitionKeyRaw(
- key_range.range_stop_key), "adding scan range upper bound");
- }
- return Status::OK();
-}
-
-} // anonymous namespace
-
KuduScanner::KuduScanner(KuduScanNode* scan_node, RuntimeState* state)
: scan_node_(scan_node),
state_(state),
- rows_scanned_current_block_(0),
- last_alive_time_micros_(0) {
+ cur_kudu_batch_num_read_(0),
+ last_alive_time_micros_(0),
+ tuple_num_null_bytes_(scan_node_->tuple_desc()->num_null_bytes()) {
}
Status KuduScanner::Open() {
- tuple_byte_size_ = scan_node_->tuple_desc()->byte_size();
- tuple_num_null_bytes_ = scan_node_->tuple_desc()->num_null_bytes();
-
// Store columns that need relocation when materialized into the
// destination row batch.
for (int i = 0; i < scan_node_->tuple_desc_->slots().size(); ++i) {
@@ -129,60 +102,48 @@ Status KuduScanner::GetNext(RowBatch* row_batch, bool* eos) {
Tuple* tuple = reinterpret_cast<Tuple*>(tuple_buffer);
// Main scan loop:
- // Tries to fill 'row_batch' with rows from the last fetched block.
- // If there are no rows to decode tries to get the next block from kudu.
- // If there are no more blocks in the current range tries to get the next range.
- // If there aren't any more rows, blocks or ranges then we're done.
- while(true) {
+ // Tries to fill 'row_batch' with rows from cur_kudu_batch_.
+ // If there are no rows to decode, tries to get the next row batch from kudu.
+ // If this scanner has no more rows, the scanner is closed and eos is returned.
+ while (!*eos) {
RETURN_IF_CANCELLED(state_);
- // If the last fetched block has more rows, decode and if we filled up the batch
- // return.
- if (CurrentBlockHasMoreRows()) {
+
+ if (cur_kudu_batch_num_read_ < cur_kudu_batch_.NumRows()) {
bool batch_done = false;
RETURN_IF_ERROR(DecodeRowsIntoRowBatch(row_batch, &tuple, &batch_done));
- if (batch_done) return Status::OK();
+ if (batch_done) break;
}
- // If the current scanner has more blocks, fetch them.
- if (CurrentRangeHasMoreBlocks()) {
- RETURN_IF_ERROR(GetNextBlock());
+
+ if (scanner_->HasMoreRows()) {
+ RETURN_IF_ERROR(GetNextScannerBatch());
continue;
}
- // No more blocks in the current scanner, close it.
- CloseCurrentRange();
- // No more rows or blocks, we're done.
+
+ CloseCurrentClientScanner();
*eos = true;
- return Status::OK();
}
return Status::OK();
}
void KuduScanner::Close() {
- if (scanner_) CloseCurrentRange();
+ if (scanner_) CloseCurrentClientScanner();
Expr::Close(conjunct_ctxs_, state_);
}
-Status KuduScanner::OpenNextRange(const TKuduKeyRange& key_range) {
+Status KuduScanner::OpenNextScanToken(const string& scan_token) {
DCHECK(scanner_ == NULL);
- scanner_.reset(new kudu::client::KuduScanner(scan_node_->kudu_table()));
- KUDU_RETURN_IF_ERROR(scanner_->SetProjectedColumns(scan_node_->projected_columns()),
- "Unable to set projected columns");
-
- RETURN_IF_ERROR(SetupScanRangePredicate(key_range, scanner_.get()));
-
- vector<KuduPredicate*> predicates;
- scan_node_->ClonePredicates(&predicates);
- for (KuduPredicate* predicate: predicates) {
- KUDU_RETURN_IF_ERROR(scanner_->AddConjunctPredicate(predicate),
- "Unable to add conjunct predicate.");
- }
+ kudu::client::KuduScanner* scanner;
+ KUDU_RETURN_IF_ERROR(kudu::client::KuduScanToken::DeserializeIntoScanner(
+ scan_node_->kudu_client(), scan_token, &scanner),
+ "Unable to deserialize scan token");
+ scanner_.reset(scanner);
if (UNLIKELY(FLAGS_pick_only_leaders_for_tests)) {
KUDU_RETURN_IF_ERROR(scanner_->SetSelection(kudu::client::KuduClient::LEADER_ONLY),
"Could not set replica selection.");
}
- KUDU_RETURN_IF_ERROR(scanner_->SetTimeoutMillis(
- FLAGS_kudu_scanner_timeout_sec * 1000),
+ KUDU_RETURN_IF_ERROR(scanner_->SetTimeoutMillis(FLAGS_kudu_scanner_timeout_sec * 1000),
"Could not set scanner timeout");
{
@@ -192,17 +153,17 @@ Status KuduScanner::OpenNextRange(const TKuduKeyRange& key_range) {
return Status::OK();
}
-void KuduScanner::CloseCurrentRange() {
+void KuduScanner::CloseCurrentClientScanner() {
DCHECK_NOTNULL(scanner_.get());
scanner_->Close();
scanner_.reset();
}
Status KuduScanner::HandleEmptyProjection(RowBatch* row_batch, bool* batch_done) {
- int rem_in_block = cur_kudu_batch_.NumRows() - rows_scanned_current_block_;
+ int num_rows_remaining = cur_kudu_batch_.NumRows() - cur_kudu_batch_num_read_;
int rows_to_add = std::min(row_batch->capacity() - row_batch->num_rows(),
- rem_in_block);
- rows_scanned_current_block_ += rows_to_add;
+ num_rows_remaining);
+ cur_kudu_batch_num_read_ += rows_to_add;
row_batch->CommitRows(rows_to_add);
// If we've reached the capacity, or the LIMIT for the scan, return.
if (row_batch->AtCapacity() || scan_node_->ReachedLimit()) {
@@ -228,14 +189,14 @@ Status KuduScanner::DecodeRowsIntoRowBatch(RowBatch* row_batch,
int num_rows = cur_kudu_batch_.NumRows();
// Now iterate through the Kudu rows.
- for (int krow_idx = rows_scanned_current_block_; krow_idx < num_rows; ++krow_idx) {
+ for (int krow_idx = cur_kudu_batch_num_read_; krow_idx < num_rows; ++krow_idx) {
// Clear any NULL indicators set by a previous iteration.
(*tuple_mem)->Init(tuple_num_null_bytes_);
// Transform a Kudu row into an Impala row.
KuduScanBatch::RowPtr krow = cur_kudu_batch_.Row(krow_idx);
RETURN_IF_ERROR(KuduRowToImpalaTuple(krow, row_batch, *tuple_mem));
- ++rows_scanned_current_block_;
+ ++cur_kudu_batch_num_read_;
// Evaluate the conjuncts that haven't been pushed down to Kudu.
if (conjunct_ctxs_.empty() ||
@@ -373,12 +334,12 @@ Status KuduScanner::KuduRowToImpalaTuple(const KuduScanBatch::RowPtr& row,
}
-Status KuduScanner::GetNextBlock() {
+Status KuduScanner::GetNextScannerBatch() {
SCOPED_TIMER(scan_node_->kudu_read_timer());
int64_t now = MonotonicMicros();
KUDU_RETURN_IF_ERROR(scanner_->NextBatch(&cur_kudu_batch_), "Unable to advance iterator");
COUNTER_ADD(scan_node_->kudu_round_trips(), 1);
- rows_scanned_current_block_ = 0;
+ cur_kudu_batch_num_read_ = 0;
COUNTER_ADD(scan_node_->rows_read_counter(), cur_kudu_batch_.NumRows());
last_alive_time_micros_ = now;
return Status::OK();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-scanner.h b/be/src/exec/kudu-scanner.h
index beddd05..b31bb73 100644
--- a/be/src/exec/kudu-scanner.h
+++ b/be/src/exec/kudu-scanner.h
@@ -31,7 +31,9 @@ class RowBatch;
class RuntimeState;
class Tuple;
-/// Executes scans in Kudu based on the provided set of scan ranges.
+/// Wraps a Kudu client scanner to fetch row batches from Kudu. The Kudu client scanner
+/// is created from a scan token in OpenNextScanToken(), which then provides rows fetched
+/// by GetNext() until it reaches eos, and the caller may open another scan token.
class KuduScanner {
public:
KuduScanner(KuduScanNode* scan_node, RuntimeState* state);
@@ -40,17 +42,17 @@ class KuduScanner {
/// Does not actually open a kudu::client::KuduScanner.
Status Open();
- /// Opens a new kudu::client::KuduScanner to scan 'range'.
- Status OpenNextRange(const TKuduKeyRange& range);
+ /// Opens a new kudu::client::KuduScanner using 'scan_token'.
+ Status OpenNextScanToken(const std::string& scan_token);
/// Fetches the next batch from the current kudu::client::KuduScanner.
Status GetNext(RowBatch* row_batch, bool* eos);
- /// Sends a "Ping" to the Kudu TabletServer servicing the current scan, if there is one.
- /// This serves the purpose of making the TabletServer keep the server side scanner alive
- /// if the batch queue is full and no batches can be queued. If there are any errors,
- /// they are ignored here, since we assume that we will just fail the next time we
- /// try to read a batch.
+ /// Sends a "Ping" to the Kudu TabletServer servicing the current scan, if there is
+ /// one. This serves the purpose of making the TabletServer keep the server side
+ /// scanner alive if the batch queue is full and no batches can be queued. If there are
+ /// any errors, they are ignored here, since we assume that we will just fail the next
+ /// time we try to read a batch.
void KeepKuduScannerAlive();
/// Closes this scanner.
@@ -67,32 +69,22 @@ class KuduScanner {
/// Returns true if 'slot' is Null in 'tuple'.
bool IsSlotNull(Tuple* tuple, const SlotDescriptor& slot);
- /// Returns true if the current block hasn't been fully scanned.
- bool CurrentBlockHasMoreRows() {
- return rows_scanned_current_block_ < cur_kudu_batch_.NumRows();
- }
-
/// Decodes rows previously fetched from kudu, now in 'cur_rows_' into a RowBatch.
/// - 'batch' is the batch that will point to the new tuples.
/// - *tuple_mem should be the location to output tuples.
- /// - Sets 'batch_done' to true to indicate that the batch was filled to capacity or the limit
- /// was reached.
+ /// - Sets 'batch_done' to true to indicate that the batch was filled to capacity or
+ /// the limit was reached.
Status DecodeRowsIntoRowBatch(RowBatch* batch, Tuple** tuple_mem, bool* batch_done);
- /// Returns true of the current kudu::client::KuduScanner has more rows.
- bool CurrentRangeHasMoreBlocks() {
- return scanner_->HasMoreRows();
- }
-
- /// Fetches the next block of the current kudu::client::KuduScanner.
- Status GetNextBlock();
+ /// Fetches the next batch of rows from the current kudu::client::KuduScanner.
+ Status GetNextScannerBatch();
/// Closes the current kudu::client::KuduScanner.
- void CloseCurrentRange();
+ void CloseCurrentClientScanner();
- /// Given a tuple, copies the values of those columns that require additional memory from memory
- /// owned by the kudu::client::KuduScanner into memory owned by the RowBatch. Assumes that the
- /// other columns are already materialized.
+ /// Given a tuple, copies the values of those columns that require additional memory
+ /// from memory owned by the kudu::client::KuduScanner into memory owned by the
+ /// RowBatch. Assumes that the other columns are already materialized.
Status RelocateValuesFromKudu(Tuple* tuple, MemPool* mem_pool);
/// Transforms a kudu row into an Impala row. Columns that don't require auxiliary
@@ -113,16 +105,15 @@ class KuduScanner {
KuduScanNode* scan_node_;
RuntimeState* state_;
- /// The set of key ranges being serviced by this scanner.
- const vector<TKuduKeyRange> scan_ranges_;
-
- /// The kudu::client::KuduScanner for the current range.
- /// One such scanner is required per range as per-range start/stop keys can only be set once.
+ /// The kudu::client::KuduScanner for the current scan token. A new KuduScanner is
+ /// created for each scan token using KuduScanToken::DeserializeIntoScanner().
boost::scoped_ptr<kudu::client::KuduScanner> scanner_;
/// The current batch of retrieved rows.
kudu::client::KuduScanBatch cur_kudu_batch_;
- int rows_scanned_current_block_;
+
+ /// The number of rows already read from cur_kudu_batch_.
+ int cur_kudu_batch_num_read_;
/// The last time a keepalive request or successful RPC was sent.
int64_t last_alive_time_micros_;
@@ -130,11 +121,6 @@ class KuduScanner {
/// The scanner's cloned copy of the conjuncts to apply.
vector<ExprContext*> conjunct_ctxs_;
- std::vector<std::string> projected_columns_;
-
- /// Size of the materialized tuple in the row batch.
- int tuple_byte_size_;
-
/// Number of bytes needed to represent the null bits in the tuple.
int tuple_num_null_bytes_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-util.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-util.cc b/be/src/exec/kudu-util.cc
index 8edf1cb..aedf229 100644
--- a/be/src/exec/kudu-util.cc
+++ b/be/src/exec/kudu-util.cc
@@ -17,19 +17,16 @@
#include "exec/kudu-util.h"
-#include <algorithm>
+#include <string>
+#include <sstream>
-#include <boost/algorithm/string.hpp>
-#include <boost/unordered_set.hpp>
#include <kudu/client/callbacks.h>
#include <kudu/client/schema.h>
-#include "runtime/descriptors.h"
-#include "gutil/strings/substitute.h"
-
+#include "common/logging.h"
#include "common/names.h"
+#include "common/status.h"
-using boost::algorithm::to_lower_copy;
using kudu::client::KuduSchema;
using kudu::client::KuduColumnSchema;
@@ -57,122 +54,13 @@ Status CheckKuduAvailability() {
return Status(TErrorCode::KUDU_NOT_SUPPORTED_ON_OS);
}
-Status ImpalaToKuduType(const ColumnType& impala_type,
- kudu::client::KuduColumnSchema::DataType* kudu_type) {
- using kudu::client::KuduColumnSchema;
-
- switch (impala_type.type) {
- case TYPE_VARCHAR:
- case TYPE_STRING:
- *kudu_type = KuduColumnSchema::STRING;
- break;
- case TYPE_TINYINT:
- *kudu_type = KuduColumnSchema::INT8;
- break;
- case TYPE_SMALLINT:
- *kudu_type = KuduColumnSchema::INT16;
- break;
- case TYPE_INT:
- *kudu_type = KuduColumnSchema::INT32;
- break;
- case TYPE_BIGINT:
- *kudu_type = KuduColumnSchema::INT64;
- break;
- case TYPE_FLOAT:
- *kudu_type = KuduColumnSchema::FLOAT;
- break;
- case TYPE_DOUBLE:
- *kudu_type = KuduColumnSchema::DOUBLE;
- break;
- case TYPE_BOOLEAN:
- *kudu_type = KuduColumnSchema::BOOL;
- break;
- default:
- return Status(TErrorCode::IMPALA_KUDU_TYPE_MISSING, TypeToString(impala_type.type));
- }
- return Status::OK();
-}
-
-Status KuduToImpalaType(const kudu::client::KuduColumnSchema::DataType& kudu_type,
- ColumnType* impala_type) {
- using kudu::client::KuduColumnSchema;
-
- switch (kudu_type) {
- case KuduColumnSchema::STRING:
- *impala_type = TYPE_STRING;
- break;
- case KuduColumnSchema::INT8:
- *impala_type = TYPE_TINYINT;
- break;
- case KuduColumnSchema::INT16:
- *impala_type = TYPE_SMALLINT;
- break;
- case KuduColumnSchema::INT32:
- *impala_type = TYPE_INT;
- break;
- case KuduColumnSchema::INT64:
- *impala_type = TYPE_BIGINT;
- break;
- case KuduColumnSchema::FLOAT:
- *impala_type = TYPE_FLOAT;
- break;
- case KuduColumnSchema::DOUBLE:
- *impala_type = TYPE_DOUBLE;
- break;
- default:
- return Status(TErrorCode::KUDU_IMPALA_TYPE_MISSING,
- KuduColumnSchema::DataTypeToString(kudu_type));
+string KuduSchemaDebugString(const KuduSchema& schema) {
+ stringstream ss;
+ for (int i = 0; i < schema.num_columns(); ++i) {
+ const KuduColumnSchema& col = schema.Column(i);
+ ss << col.name() << " " << KuduColumnSchema::DataTypeToString(col.type()) << "\n";
}
- return Status::OK();
-}
-
-/// Returns a map of lower case column names to column indexes in 'map'.
-/// Returns an error Status if 'schema' had more than one column with the same lower
-/// case name.
-Status MapLowercaseKuduColumnNamesToIndexes(const kudu::client::KuduSchema& schema,
- IdxByLowerCaseColName* map) {
- DCHECK(map != NULL);
- for(size_t i = 0; i < schema.num_columns(); ++i) {
- string lower_case_col_name = to_lower_copy(schema.Column(i).name());
- if (map->find(lower_case_col_name) != map->end()) {
- return Status(strings::Substitute("There was already a column with name: '$0' "
- "in the schema", lower_case_col_name));
- }
- (*map)[lower_case_col_name] = i;
- }
- return Status::OK();
-}
-
-Status ProjectedColumnsFromTupleDescriptor(const TupleDescriptor& tuple_desc,
- vector<string>* projected_columns, const KuduSchema& schema) {
- DCHECK(projected_columns != NULL);
- projected_columns->clear();
-
- IdxByLowerCaseColName idx_by_lc_name;
- RETURN_IF_ERROR(MapLowercaseKuduColumnNamesToIndexes(schema, &idx_by_lc_name));
-
- // In debug mode try a dynamic cast. If it fails it means that the
- // TableDescriptor is not an instance of KuduTableDescriptor.
- DCHECK(dynamic_cast<const KuduTableDescriptor*>(tuple_desc.table_desc()) != NULL)
- << "TableDescriptor must be an instance KuduTableDescriptor.";
-
- const KuduTableDescriptor* table_desc =
- static_cast<const KuduTableDescriptor*>(tuple_desc.table_desc());
- LOG(INFO) << "Table desc for schema: " << table_desc->DebugString();
-
- const std::vector<SlotDescriptor*>& slots = tuple_desc.slots();
- for (int i = 0; i < slots.size(); ++i) {
- int col_idx = slots[i]->col_pos();
- string impala_col_name = to_lower_copy(table_desc->col_descs()[col_idx].name());
- IdxByLowerCaseColName::const_iterator iter = idx_by_lc_name.find(impala_col_name);
- if (iter == idx_by_lc_name.end()) {
- return Status(strings::Substitute("Could not find column: $0 in the Kudu schema.",
- impala_col_name));
- }
- projected_columns->push_back(schema.Column((*iter).second).name());
- }
-
- return Status::OK();
+ return ss.str();
}
void LogKuduMessage(void* unused, kudu::client::KuduLogSeverity severity,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/exec/kudu-util.h
----------------------------------------------------------------------
diff --git a/be/src/exec/kudu-util.h b/be/src/exec/kudu-util.h
index fd779cb..7957f7a 100644
--- a/be/src/exec/kudu-util.h
+++ b/be/src/exec/kudu-util.h
@@ -21,15 +21,9 @@
#include <kudu/client/callbacks.h>
#include <kudu/client/client.h>
-#include <boost/unordered_map.hpp>
-
namespace impala {
-class TExpr;
-class KuduTableDescriptor;
class Status;
-class TupleDescriptor;
-struct ColumnType;
/// Returns false when running on an operating system that Kudu doesn't support. If this
/// check fails, there is no way Kudu should be expected to work. Exposed for testing.
@@ -43,25 +37,8 @@ Status CheckKuduAvailability();
/// Convenience function for the bool equivalent of CheckKuduAvailability().
bool KuduIsAvailable();
-Status ImpalaToKuduType(const ColumnType& impala_type,
- kudu::client::KuduColumnSchema::DataType* kudu_type);
-
-Status KuduToImpalaType(const kudu::client::KuduColumnSchema::DataType& kudu_type,
- ColumnType* impala_type);
-
-typedef boost::unordered_map<std::string, int> IdxByLowerCaseColName;
-
-/// Returns a map of lower case column names to column indexes in 'map'.
-/// Returns an error Status if 'schema' had more than one column with the same lower
-/// case name.
-Status MapLowercaseKuduColumnNamesToIndexes(const kudu::client::KuduSchema& schema,
- IdxByLowerCaseColName* map);
-
-/// Gets the projected columns from the TupleDescriptor.
-/// Translates Impala's lower case column names to the version used by Kudu.
-/// 'projected_columns' is expected to be not NULL and will be cleared.
-Status ProjectedColumnsFromTupleDescriptor(const TupleDescriptor& tuple_desc,
- std::vector<std::string>* projected_columns, const kudu::client::KuduSchema& schema);
+/// Returns a debug string for the KuduSchema.
+std::string KuduSchemaDebugString(const kudu::client::KuduSchema& schema);
/// Initializes Kudu's logging by binding a callback that logs back to Impala's glog. This
/// also sets Kudu's verbose logging to whatever level is set in Impala.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/be/src/scheduling/simple-scheduler.cc
----------------------------------------------------------------------
diff --git a/be/src/scheduling/simple-scheduler.cc b/be/src/scheduling/simple-scheduler.cc
index 9a865f0..f4b90cc 100644
--- a/be/src/scheduling/simple-scheduler.cc
+++ b/be/src/scheduling/simple-scheduler.cc
@@ -970,7 +970,7 @@ void SimpleScheduler::AssignmentCtx::RecordScanRangeAssignment(
int64_t scan_range_length = 0;
if (scan_range_locations.scan_range.__isset.hdfs_file_split) {
scan_range_length = scan_range_locations.scan_range.hdfs_file_split.length;
- } else if (scan_range_locations.scan_range.__isset.kudu_key_range) {
+ } else if (scan_range_locations.scan_range.__isset.kudu_scan_token) {
// Hack so that kudu ranges are well distributed.
// TODO: KUDU-1133 Use the tablet size instead.
scan_range_length = 1000;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 7f4df7f..39750c0 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -285,7 +285,7 @@ export KUDU_MASTER_PORT="${KUDU_MASTER_PORT:-7051}"
# TODO: Figure out a way to use a snapshot version without causing a lot of breakage due
# to nightly changes from Kudu. The version below is the last released version but
# before release this needs to be updated to the version about to be released.
-export KUDU_JAVA_VERSION=0.10.0-SNAPSHOT
+export KUDU_JAVA_VERSION=1.0.0-SNAPSHOT
if [[ $OSTYPE == "darwin"* ]]; then
IMPALA_CYRUS_SASL_VERSION=2.1.26
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/common/thrift/PlanNodes.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index 3e31120..0219bcd 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -173,19 +173,13 @@ struct THBaseKeyRange {
2: optional string stopKey
}
-// Key range to query a Kudu table
-struct TKuduKeyRange {
- 1: optional binary range_start_key
- 2: optional binary range_stop_key
-}
-
// Specification of an individual data range which is held in its entirety
// by a storage server
struct TScanRange {
- // one of these must be set for every TScanRange2
+ // one of these must be set for every TScanRange
1: optional THdfsFileSplit hdfs_file_split
2: optional THBaseKeyRange hbase_key_range
- 3: optional TKuduKeyRange kudu_key_range
+ 3: optional binary kudu_scan_token
}
struct THdfsScanNode {
@@ -247,9 +241,6 @@ struct THBaseScanNode {
struct TKuduScanNode {
1: required Types.TTupleId tuple_id
-
- // List of conjuncts that can be pushed down to Kudu.
- 2: optional list<Exprs.TExpr> kudu_conjuncts
}
struct TEqJoinCondition {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/pom.xml
----------------------------------------------------------------------
diff --git a/fe/pom.xml b/fe/pom.xml
index 3870262..f7bffb9 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -272,7 +272,7 @@ under the License.
</dependency>
<dependency>
- <groupId>org.kududb</groupId>
+ <groupId>org.apache.kudu</groupId>
<artifactId>kudu-client</artifactId>
<version>${kudu.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/main/java/com/cloudera/impala/analysis/Expr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/Expr.java b/fe/src/main/java/com/cloudera/impala/analysis/Expr.java
index 91d240a..fdc5bf1 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/Expr.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/Expr.java
@@ -1190,8 +1190,8 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
/**
- * For children of 'this' that are constant expressions capable of being expressed
- * as LiteralExprs, evaluate them in the BE and substitute the child with the
+ * For children of 'this' that are constant expressions and the type of which has a
+ * LiteralExpr subclass, evaluate them in the BE and substitute the child with the
* resulting LiteralExpr. Modifies 'this' in place and does not re-analyze it. Hence,
* it is not safe to evaluate the modified expr in the BE as the resolved fn_ may be
* incorrect given the new arguments.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java b/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java
index d387b54..f5eedbb 100644
--- a/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java
+++ b/fe/src/main/java/com/cloudera/impala/analysis/LiteralExpr.java
@@ -49,8 +49,8 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
}
/**
- * Returns an analyzed literal of 'type'. Returns null for types that cannot be
- * expressed as literals, e.g. TIMESTAMP.
+ * Returns an analyzed literal of 'type'. Returns null for types that do not have a
+ * LiteralExpr subclass, e.g. TIMESTAMP.
*/
public static LiteralExpr create(String value, Type type) throws AnalysisException {
Preconditions.checkArgument(type.isValid());
@@ -151,7 +151,7 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
/**
* Evaluates the given constant expr and returns its result as a LiteralExpr.
* Assumes expr has been analyzed. Returns constExpr if is it already a LiteralExpr.
- * Returns null for types that cannot be expressed as literals, e.g. TIMESTAMP.
+ * Returns null for types that do not have a LiteralExpr subclass, e.g. TIMESTAMP.
* TODO: Support non-scalar types.
*/
public static LiteralExpr create(Expr constExpr, TQueryCtx queryCtx)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/main/java/com/cloudera/impala/catalog/KuduTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/KuduTable.java b/fe/src/main/java/com/cloudera/impala/catalog/KuduTable.java
index 28f4133..71d897d 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/KuduTable.java
@@ -28,8 +28,8 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.log4j.Logger;
-import org.kududb.client.KuduClient;
-import org.kududb.client.LocatedTablet;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.LocatedTablet;
import com.cloudera.impala.common.ImpalaRuntimeException;
import com.cloudera.impala.thrift.TCatalogObjectType;
@@ -248,7 +248,7 @@ public class KuduTable extends Table {
try (KuduClient client = new KuduClient.KuduClientBuilder(
getKuduMasterAddresses()).build()) {
- org.kududb.client.KuduTable kuduTable = client.openTable(kuduTableName_);
+ org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName_);
List<LocatedTablet> tablets =
kuduTable.getTabletsLocations(KUDU_RPC_TIMEOUT_MS);
for (LocatedTablet tab: tablets) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/157c8005/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java b/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
index 919eebb..ecfeb1a 100644
--- a/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
+++ b/fe/src/main/java/com/cloudera/impala/catalog/delegates/KuduDdlDelegate.java
@@ -28,13 +28,13 @@ import java.util.List;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Table;
-import org.kududb.ColumnSchema;
-import org.kududb.ColumnSchema.ColumnSchemaBuilder;
-import org.kududb.Schema;
-import org.kududb.Type;
-import org.kududb.client.CreateTableOptions;
-import org.kududb.client.KuduClient;
-import org.kududb.client.PartialRow;
+import org.apache.kudu.ColumnSchema;
+import org.apache.kudu.ColumnSchema.ColumnSchemaBuilder;
+import org.apache.kudu.Schema;
+import org.apache.kudu.Type;
+import org.apache.kudu.client.CreateTableOptions;
+import org.apache.kudu.client.KuduClient;
+import org.apache.kudu.client.PartialRow;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,7 +85,7 @@ public class KuduDdlDelegate extends DdlDelegate {
}
// Check if the external table matches the schema
- org.kududb.client.KuduTable kuduTable = client.openTable(kuduTableName);
+ org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName);
if (!compareSchema(msTbl_, kuduTable)) {
throw new ImpalaRuntimeException(String.format(
"Table %s (%s) has a different schema in Kudu than in Hive.",
[7/7] incubator-impala git commit: IMPALA-3905: Add single-threaded
scan node.
Posted by ab...@apache.org.
IMPALA-3905: Add single-threaded scan node.
Adds a new single-threaded scan node HdfsScanNodeMt that
materializes tuples in the thread calling GetNext().
The new scan node uses the HdfsScanner::GetNext() interface,
which is currently only implemented for Parquet.
As before, I/O is performed asynchronously via the I/O manager.
The new scan node is enabled if the mt_dop query option is
set to a value greater than 1. Otherwise, the existing
multi-threaded scan node is used.
The changes are mostly a refactoring of the existing multi-threaded
scan node to separate out the common code between the existing
multi-threaded scan node and the new single-threaded one.
Summary of changes:
- Move code from hdfs-scan-node.h/cc into a new hdfs-scan-node-base.h/cc
- Add new single-threaded scan node in hdfs-scan-node-mt.h/cc
- Both scan nodes inherit from HdfsScanNodeBase
- Rework the allocation of templates tuples such that the memory is drawn
from a new mem pool in the scanners, and that each scanner clones the
partition exprs contexts. Before, the memory was taken from the parent
scan node's mem pool, and there was only one instance of the partition
exprs contexts. Their access was protected under a lock, however, not
in all instances, so their use was not always obviously correct.
The change in this patch makes thread safety obvious and helps move
a lock into the multi-threaded scan node which would otherwise have
to remain in the HdfsScanNodeBase class.
- Simplify a couple of loops with C++11 for-each
Testing: A private core/hdfs run passed. I ran TPC-H/DS and test_scanners.py
on ASAN several times locally.
Change-Id: I98cc7f970e1575dd83875609985e1877ada3d5e0
Reviewed-on: http://gerrit.cloudera.org:8080/4113
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Alex Behm <al...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/faebfebd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/faebfebd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/faebfebd
Branch: refs/heads/master
Commit: faebfebdfd227b8f72055e60e13ac8726eafc8fc
Parents: 8c37bf3
Author: Alex Behm <al...@cloudera.com>
Authored: Fri Jul 29 10:24:42 2016 -0700
Committer: Alex Behm <al...@cloudera.com>
Committed: Thu Sep 8 04:34:52 2016 +0000
----------------------------------------------------------------------
be/src/exec/CMakeLists.txt | 2 +
be/src/exec/base-sequence-scanner.cc | 31 +-
be/src/exec/base-sequence-scanner.h | 4 +-
be/src/exec/exec-node.cc | 6 +
be/src/exec/hdfs-avro-scanner.cc | 28 +-
be/src/exec/hdfs-avro-scanner.h | 9 +-
be/src/exec/hdfs-lzo-text-scanner.cc | 8 +-
be/src/exec/hdfs-lzo-text-scanner.h | 11 +-
be/src/exec/hdfs-parquet-scanner.cc | 58 +-
be/src/exec/hdfs-parquet-scanner.h | 10 +-
be/src/exec/hdfs-rcfile-scanner.cc | 5 +-
be/src/exec/hdfs-rcfile-scanner.h | 3 +-
be/src/exec/hdfs-scan-node-base.cc | 893 ++++++++++++++++++
be/src/exec/hdfs-scan-node-base.h | 459 +++++++++
be/src/exec/hdfs-scan-node-mt.cc | 125 +++
be/src/exec/hdfs-scan-node-mt.h | 59 ++
be/src/exec/hdfs-scan-node.cc | 920 +------------------
be/src/exec/hdfs-scan-node.h | 453 +--------
be/src/exec/hdfs-scanner-ir.cc | 2 +-
be/src/exec/hdfs-scanner.cc | 58 +-
be/src/exec/hdfs-scanner.h | 87 +-
be/src/exec/hdfs-sequence-scanner.cc | 8 +-
be/src/exec/hdfs-sequence-scanner.h | 6 +-
be/src/exec/hdfs-text-scanner.cc | 22 +-
be/src/exec/hdfs-text-scanner.h | 8 +-
be/src/exec/scanner-context.cc | 6 +-
be/src/exec/scanner-context.h | 11 +-
be/src/exprs/expr-context.h | 5 +-
be/src/runtime/tuple.h | 5 +-
common/thrift/ImpalaInternalService.thrift | 7 +-
.../com/cloudera/impala/service/Frontend.java | 2 +-
31 files changed, 1837 insertions(+), 1474 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
index 091b9b7..8f12656 100644
--- a/be/src/exec/CMakeLists.txt
+++ b/be/src/exec/CMakeLists.txt
@@ -46,6 +46,8 @@ add_library(Exec
hbase-table-sink.cc
hbase-table-writer.cc
hdfs-scan-node.cc
+ hdfs-scan-node-base.cc
+ hdfs-scan-node-mt.cc
hdfs-scanner.cc
hdfs-scanner-ir.cc
hdfs-table-sink.cc
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/base-sequence-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/base-sequence-scanner.cc b/be/src/exec/base-sequence-scanner.cc
index 5429e04..5f6bad6 100644
--- a/be/src/exec/base-sequence-scanner.cc
+++ b/be/src/exec/base-sequence-scanner.cc
@@ -41,7 +41,7 @@ static const int MIN_SYNC_READ_SIZE = 64 * 1024; // bytes
// Macro to convert between SerdeUtil errors to Status returns.
#define RETURN_IF_FALSE(x) if (UNLIKELY(!(x))) return parse_status_
-Status BaseSequenceScanner::IssueInitialRanges(HdfsScanNode* scan_node,
+Status BaseSequenceScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const vector<HdfsFileDesc*>& files) {
// Issue just the header range for each file. When the header is complete,
// we'll issue the splits for that file. Splits cannot be processed until the
@@ -49,7 +49,7 @@ Status BaseSequenceScanner::IssueInitialRanges(HdfsScanNode* scan_node,
vector<DiskIoMgr::ScanRange*> header_ranges;
for (int i = 0; i < files.size(); ++i) {
ScanRangeMetadata* metadata =
- reinterpret_cast<ScanRangeMetadata*>(files[i]->splits[0]->meta_data());
+ static_cast<ScanRangeMetadata*>(files[i]->splits[0]->meta_data());
int64_t header_size = min<int64_t>(HEADER_SIZE, files[i]->file_length);
// The header is almost always a remote read. Set the disk id to -1 and indicate
// it is not cached.
@@ -66,9 +66,8 @@ Status BaseSequenceScanner::IssueInitialRanges(HdfsScanNode* scan_node,
return Status::OK();
}
-BaseSequenceScanner::BaseSequenceScanner(HdfsScanNode* node, RuntimeState* state,
- bool add_batches_to_queue)
- : HdfsScanner(node, state, add_batches_to_queue),
+BaseSequenceScanner::BaseSequenceScanner(HdfsScanNodeBase* node, RuntimeState* state)
+ : HdfsScanner(node, state),
header_(NULL),
block_start_(0),
total_block_size_(0),
@@ -110,12 +109,23 @@ void BaseSequenceScanner::Close(RowBatch* row_batch) {
if (row_batch != NULL) {
row_batch->tuple_data_pool()->AcquireData(data_buffer_pool_.get(), false);
context_->ReleaseCompletedResources(row_batch, true);
- if (add_batches_to_queue_) scan_node_->AddMaterializedRowBatch(row_batch);
+ if (scan_node_->HasRowBatchQueue()) {
+ static_cast<HdfsScanNode*>(scan_node_)->AddMaterializedRowBatch(row_batch);
+ }
+ }
+ // Transfer template tuple pool to scan node pool. The scanner may be closed and
+ // subsequently re-used for another range, so we need to ensure that the template
+ // tuples are backed by live memory.
+ if (template_tuple_pool_.get() != NULL) {
+ static_cast<HdfsScanNode*>(scan_node_)->TransferToScanNodePool(
+ template_tuple_pool_.get());
}
+
// Verify all resources (if any) have been transferred.
+ DCHECK_EQ(template_tuple_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(data_buffer_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(context_->num_completed_io_buffers(), 0);
- // 'header_' can be NULL if HdfsScanNode::CreateAndPrepareScanner() failed.
+ // 'header_' can be NULL if HdfsScanNodeBase::CreateAndOpenScanner() failed.
if (!only_parsing_header_ && header_ != NULL) {
scan_node_->RangeComplete(file_format(), header_->compression_type);
}
@@ -123,9 +133,9 @@ void BaseSequenceScanner::Close(RowBatch* row_batch) {
}
Status BaseSequenceScanner::ProcessSplit() {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
header_ = reinterpret_cast<FileHeader*>(
- scan_node_->GetFileMetadata(stream_->filename()));
+ static_cast<HdfsScanNode*>(scan_node_)->GetFileMetadata(stream_->filename()));
if (header_ == NULL) {
// This is the initial scan range just to parse the header
only_parsing_header_ = true;
@@ -139,7 +149,8 @@ Status BaseSequenceScanner::ProcessSplit() {
}
// Header is parsed, set the metadata in the scan node and issue more ranges
- scan_node_->SetFileMetadata(stream_->filename(), header_);
+ static_cast<HdfsScanNode*>(scan_node_)->SetFileMetadata(
+ stream_->filename(), header_);
HdfsFileDesc* desc = scan_node_->GetFileDesc(stream_->filename());
scan_node_->AddDiskIoRanges(desc);
return Status::OK();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/base-sequence-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/base-sequence-scanner.h b/be/src/exec/base-sequence-scanner.h
index 55e3728..a909e00 100644
--- a/be/src/exec/base-sequence-scanner.h
+++ b/be/src/exec/base-sequence-scanner.h
@@ -38,7 +38,7 @@ class ScannerContext;
class BaseSequenceScanner : public HdfsScanner {
public:
/// Issue the initial ranges for all sequence container files.
- static Status IssueInitialRanges(HdfsScanNode* scan_node,
+ static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files);
virtual Status Open(ScannerContext* context);
@@ -102,7 +102,7 @@ class BaseSequenceScanner : public HdfsScanner {
/// Returns type of scanner: e.g. rcfile, seqfile
virtual THdfsFileFormat::type file_format() const = 0;
- BaseSequenceScanner(HdfsScanNode*, RuntimeState*, bool);
+ BaseSequenceScanner(HdfsScanNodeBase*, RuntimeState*);
/// Read and validate sync marker against header_->sync. Returns non-ok if the sync
/// marker did not match. Scanners should always use this function to read sync markers,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/exec-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index 0996037..59ac997 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -35,6 +35,7 @@
#include "exec/hash-join-node.h"
#include "exec/hbase-scan-node.h"
#include "exec/hdfs-scan-node.h"
+#include "exec/hdfs-scan-node-mt.h"
#include "exec/kudu-scan-node.h"
#include "exec/kudu-util.h"
#include "exec/nested-loop-join-node.h"
@@ -264,6 +265,11 @@ Status ExecNode::CreateNode(ObjectPool* pool, const TPlanNode& tnode,
switch (tnode.node_type) {
case TPlanNodeType::HDFS_SCAN_NODE:
*node = pool->Add(new HdfsScanNode(pool, tnode, descs));
+ if (state->query_options().mt_num_cores > 0) {
+ *node = pool->Add(new HdfsScanNodeMt(pool, tnode, descs));
+ } else {
+ *node = pool->Add(new HdfsScanNode(pool, tnode, descs));
+ }
// If true, this node requests codegen over interpretation for conjuncts
// evaluation whenever possible. Turn codegen on for expr evaluation for
// the entire fragment.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-avro-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-avro-scanner.cc b/be/src/exec/hdfs-avro-scanner.cc
index ff278de..f511b12 100644
--- a/be/src/exec/hdfs-avro-scanner.cc
+++ b/be/src/exec/hdfs-avro-scanner.cc
@@ -55,9 +55,8 @@ const string AVRO_MEM_LIMIT_EXCEEDED = "HdfsAvroScanner::$0() failed to allocate
#define RETURN_IF_FALSE(x) if (UNLIKELY(!(x))) return parse_status_
-HdfsAvroScanner::HdfsAvroScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue)
- : BaseSequenceScanner(scan_node, state, add_batches_to_queue),
+HdfsAvroScanner::HdfsAvroScanner(HdfsScanNodeBase* scan_node, RuntimeState* state)
+ : BaseSequenceScanner(scan_node, state),
avro_header_(NULL),
codegend_decode_avro_data_(NULL) {
}
@@ -78,7 +77,7 @@ Status HdfsAvroScanner::Open(ScannerContext* context) {
return Status::OK();
}
-Status HdfsAvroScanner::Codegen(HdfsScanNode* node,
+Status HdfsAvroScanner::Codegen(HdfsScanNodeBase* node,
const vector<ExprContext*>& conjunct_ctxs, Function** decode_avro_data_fn) {
*decode_avro_data_fn = NULL;
if (!node->runtime_state()->codegen_enabled()) {
@@ -291,8 +290,12 @@ Status HdfsAvroScanner::ResolveSchemas(const AvroSchemaElement& table_root,
Status HdfsAvroScanner::WriteDefaultValue(
SlotDescriptor* slot_desc, avro_datum_t default_value, const char* field_name) {
if (avro_header_->template_tuple == NULL) {
- avro_header_->template_tuple = template_tuple_ != NULL ?
- template_tuple_ : scan_node_->InitEmptyTemplateTuple(*scan_node_->tuple_desc());
+ if (template_tuple_ != NULL) {
+ avro_header_->template_tuple = template_tuple_;
+ } else {
+ avro_header_->template_tuple =
+ Tuple::Create(tuple_byte_size_, template_tuple_pool_.get());
+ }
}
switch (default_value->type) {
case AVRO_BOOLEAN: {
@@ -336,14 +339,11 @@ Status HdfsAvroScanner::WriteDefaultValue(
case AVRO_STRING:
case AVRO_BYTES: {
RETURN_IF_ERROR(VerifyTypesMatch(slot_desc, default_value));
- // Mempools aren't thread safe so make a local one and transfer it
- // to the scan node pool.
- MemPool pool(scan_node_->mem_tracker());
char* v;
if (avro_string_get(default_value, &v)) DCHECK(false);
StringValue sv(v);
- RawValue::Write(&sv, avro_header_->template_tuple, slot_desc, &pool);
- scan_node_->TransferToScanNodePool(&pool);
+ RawValue::Write(&sv, avro_header_->template_tuple, slot_desc,
+ template_tuple_pool_.get());
break;
}
case AVRO_NULL:
@@ -772,8 +772,8 @@ void HdfsAvroScanner::SetStatusValueOverflow(TErrorCode::type error_code, int64_
// bail_out: ; preds = %read_field11, %end_field3, %read_field2, %end_field,
// ret i1 false ; %read_field, %entry
// }
-Status HdfsAvroScanner::CodegenMaterializeTuple(HdfsScanNode* node, LlvmCodeGen* codegen,
- Function** materialize_tuple_fn) {
+Status HdfsAvroScanner::CodegenMaterializeTuple(HdfsScanNodeBase* node,
+ LlvmCodeGen* codegen, Function** materialize_tuple_fn) {
LLVMContext& context = codegen->context();
LlvmCodeGen::LlvmBuilder builder(context);
@@ -839,7 +839,7 @@ Status HdfsAvroScanner::CodegenMaterializeTuple(HdfsScanNode* node, LlvmCodeGen*
}
Status HdfsAvroScanner::CodegenReadRecord(
- const SchemaPath& path, const AvroSchemaElement& record, HdfsScanNode* node,
+ const SchemaPath& path, const AvroSchemaElement& record, HdfsScanNodeBase* node,
LlvmCodeGen* codegen, void* void_builder, Function* fn, BasicBlock* insert_before,
BasicBlock* bail_out, Value* this_val, Value* pool_val, Value* tuple_val,
Value* data_val, Value* data_end_val) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-avro-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-avro-scanner.h b/be/src/exec/hdfs-avro-scanner.h
index f95cd10..274cf4d 100644
--- a/be/src/exec/hdfs-avro-scanner.h
+++ b/be/src/exec/hdfs-avro-scanner.h
@@ -89,13 +89,14 @@ class HdfsAvroScanner : public BaseSequenceScanner {
/// Avro file: {'O', 'b', 'j', 1}
static const uint8_t AVRO_VERSION_HEADER[4];
- HdfsAvroScanner(HdfsScanNode* scan_node, RuntimeState* state, bool add_batches_to_queue);
+ HdfsAvroScanner(HdfsScanNodeBase* scan_node, RuntimeState* state);
virtual Status Open(ScannerContext* context);
/// Codegen DecodeAvroData(). Stores the resulting function in 'decode_avro_data_fn' if
/// codegen was successful or NULL otherwise.
- static Status Codegen(HdfsScanNode*, const std::vector<ExprContext*>& conjunct_ctxs,
+ static Status Codegen(HdfsScanNodeBase* node,
+ const std::vector<ExprContext*>& conjunct_ctxs,
llvm::Function** decode_avro_data_fn);
protected:
@@ -205,7 +206,7 @@ class HdfsAvroScanner : public BaseSequenceScanner {
/// schema. Stores the resulting function in 'materialize_tuple_fn' if codegen was
/// successful or returns an error.
/// TODO: Codegen a function for each unique file schema.
- static Status CodegenMaterializeTuple(HdfsScanNode* node, LlvmCodeGen* codegen,
+ static Status CodegenMaterializeTuple(HdfsScanNodeBase* node, LlvmCodeGen* codegen,
llvm::Function** materialize_tuple_fn);
/// Used by CodegenMaterializeTuple to recursively create the IR for reading an Avro
@@ -223,7 +224,7 @@ class HdfsAvroScanner : public BaseSequenceScanner {
/// - this_val, pool_val, tuple_val, data_val, data_end_val: arguments to
/// MaterializeTuple()
static Status CodegenReadRecord(
- const SchemaPath& path, const AvroSchemaElement& record, HdfsScanNode* node,
+ const SchemaPath& path, const AvroSchemaElement& record, HdfsScanNodeBase* node,
LlvmCodeGen* codegen, void* builder, llvm::Function* fn,
llvm::BasicBlock* insert_before, llvm::BasicBlock* bail_out, llvm::Value* this_val,
llvm::Value* pool_val, llvm::Value* tuple_val, llvm::Value* data_val,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-lzo-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-lzo-text-scanner.cc b/be/src/exec/hdfs-lzo-text-scanner.cc
index b5f0525..88ae295 100644
--- a/be/src/exec/hdfs-lzo-text-scanner.cc
+++ b/be/src/exec/hdfs-lzo-text-scanner.cc
@@ -45,13 +45,13 @@ SpinLock HdfsLzoTextScanner::lzo_load_lock_;
const char* (*GetImpalaLzoBuildVersion)();
HdfsScanner* (*HdfsLzoTextScanner::CreateLzoTextScanner)(
- HdfsScanNode* scan_node, RuntimeState* state);
+ HdfsScanNodeBase* scan_node, RuntimeState* state);
Status (*HdfsLzoTextScanner::LzoIssueInitialRanges)(
- HdfsScanNode* scan_node, const std::vector<HdfsFileDesc*>& files);
+ HdfsScanNodeBase* scan_node, const std::vector<HdfsFileDesc*>& files);
HdfsScanner* HdfsLzoTextScanner::GetHdfsLzoTextScanner(
- HdfsScanNode* scan_node, RuntimeState* state) {
+ HdfsScanNodeBase* scan_node, RuntimeState* state) {
// If the scanner was not loaded then no scans could be issued so we should
// never get here without having loaded the scanner.
@@ -60,7 +60,7 @@ HdfsScanner* HdfsLzoTextScanner::GetHdfsLzoTextScanner(
return (*CreateLzoTextScanner)(scan_node, state);
}
-Status HdfsLzoTextScanner::IssueInitialRanges(HdfsScanNode* scan_node,
+Status HdfsLzoTextScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const vector<HdfsFileDesc*>& files) {
if (LzoIssueInitialRanges == NULL) {
lock_guard<SpinLock> l(lzo_load_lock_);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-lzo-text-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-lzo-text-scanner.h b/be/src/exec/hdfs-lzo-text-scanner.h
index 0c75f97..d6bddf9 100644
--- a/be/src/exec/hdfs-lzo-text-scanner.h
+++ b/be/src/exec/hdfs-lzo-text-scanner.h
@@ -21,7 +21,7 @@
#include "common/status.h"
#include "exec/scan-node.h"
#include "exec/hdfs-scanner.h"
-#include "exec/hdfs-scan-node.h"
+#include "exec/hdfs-scan-node-base.h"
#include "util/spinlock.h"
namespace impala {
@@ -33,8 +33,9 @@ namespace impala {
/// GetHdfsLzoTextScanner -- returns a pointer to the Scanner object.
class HdfsLzoTextScanner {
public:
- static HdfsScanner* GetHdfsLzoTextScanner(HdfsScanNode* scan_node, RuntimeState* state);
- static Status IssueInitialRanges(HdfsScanNode* scan_node,
+ static HdfsScanner* GetHdfsLzoTextScanner(HdfsScanNodeBase* scan_node,
+ RuntimeState* state);
+ static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files);
private:
@@ -49,11 +50,11 @@ class HdfsLzoTextScanner {
/// Dynamically linked function to create the Lzo Scanner Object.
static HdfsScanner* (*CreateLzoTextScanner)
- (HdfsScanNode* scan_node, RuntimeState* state);
+ (HdfsScanNodeBase* scan_node, RuntimeState* state);
/// Dynamically linked function to set the initial scan ranges.
static Status (*LzoIssueInitialRanges)(
- HdfsScanNode* scan_node, const std::vector<HdfsFileDesc*>& files);
+ HdfsScanNodeBase* scan_node, const std::vector<HdfsFileDesc*>& files);
/// Dynamically loads CreateLzoTextScanner and LzoIssueInitialRanges.
/// lzo_load_lock_ should be taken before calling this method.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-parquet-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.cc b/be/src/exec/hdfs-parquet-scanner.cc
index 30c07a4..a4f1823 100644
--- a/be/src/exec/hdfs-parquet-scanner.cc
+++ b/be/src/exec/hdfs-parquet-scanner.cc
@@ -71,7 +71,7 @@ const int16_t HdfsParquetScanner::ROW_GROUP_END;
const int16_t HdfsParquetScanner::INVALID_LEVEL;
const int16_t HdfsParquetScanner::INVALID_POS;
-Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNode* scan_node,
+Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files) {
vector<DiskIoMgr::ScanRange*> footer_ranges;
for (int i = 0; i < files.size(); ++i) {
@@ -99,7 +99,7 @@ Status HdfsParquetScanner::IssueInitialRanges(HdfsScanNode* scan_node,
// footer range for the split always.
if (!scan_node->IsZeroSlotTableScan() || footer_split == split) {
ScanRangeMetadata* split_metadata =
- reinterpret_cast<ScanRangeMetadata*>(split->meta_data());
+ static_cast<ScanRangeMetadata*>(split->meta_data());
// Each split is processed by first issuing a scan range for the file footer, which
// is done here, followed by scan ranges for the columns of each row group within
// the actual split (in InitColumns()). The original split is stored in the
@@ -141,9 +141,8 @@ DiskIoMgr::ScanRange* HdfsParquetScanner::FindFooterSplit(HdfsFileDesc* file) {
namespace impala {
-HdfsParquetScanner::HdfsParquetScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue)
- : HdfsScanner(scan_node, state, add_batches_to_queue),
+HdfsParquetScanner::HdfsParquetScanner(HdfsScanNodeBase* scan_node, RuntimeState* state)
+ : HdfsScanner(scan_node, state),
row_group_idx_(-1),
row_group_rows_read_(0),
advance_row_group_(true),
@@ -215,14 +214,22 @@ Status HdfsParquetScanner::Open(ScannerContext* context) {
void HdfsParquetScanner::Close(RowBatch* row_batch) {
if (row_batch != NULL) {
FlushRowGroupResources(row_batch);
- if (add_batches_to_queue_) scan_node_->AddMaterializedRowBatch(row_batch);
- } else if (!FLAGS_enable_partitioned_hash_join ||
- !FLAGS_enable_partitioned_aggregation) {
- // With the legacy aggs/joins the tuple ptrs of the scratch batch are allocated
- // from the scratch batch's mem pool. We can get into this case if Open() fails.
- scratch_batch_->mem_pool()->FreeAll();
+ row_batch->tuple_data_pool()->AcquireData(template_tuple_pool_.get(), false);
+ if (scan_node_->HasRowBatchQueue()) {
+ static_cast<HdfsScanNode*>(scan_node_)->AddMaterializedRowBatch(row_batch);
+ }
+ } else {
+ if (template_tuple_pool_.get() != NULL) template_tuple_pool_->FreeAll();
+ if (!FLAGS_enable_partitioned_hash_join ||
+ !FLAGS_enable_partitioned_aggregation) {
+ // With the legacy aggs/joins the tuple ptrs of the scratch batch are allocated
+ // from the scratch batch's mem pool. We can get into this case if Open() fails.
+ scratch_batch_->mem_pool()->FreeAll();
+ }
}
+
// Verify all resources (if any) have been transferred.
+ DCHECK_EQ(template_tuple_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(dictionary_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(scratch_batch_->mem_pool()->total_allocated_bytes(), 0);
DCHECK_EQ(context_->num_completed_io_buffers(), 0);
@@ -307,25 +314,25 @@ int HdfsParquetScanner::CountScalarColumns(const vector<ParquetColumnReader*>& c
}
Status HdfsParquetScanner::ProcessSplit() {
- DCHECK(add_batches_to_queue_);
- bool scanner_eos = false;
+ DCHECK(scan_node_->HasRowBatchQueue());
+ HdfsScanNode* scan_node = static_cast<HdfsScanNode*>(scan_node_);
do {
StartNewParquetRowBatch();
- RETURN_IF_ERROR(GetNextInternal(batch_, &scanner_eos));
- scan_node_->AddMaterializedRowBatch(batch_);
- } while (!scanner_eos && !scan_node_->ReachedLimit());
+ RETURN_IF_ERROR(GetNextInternal(batch_));
+ scan_node->AddMaterializedRowBatch(batch_);
+ } while (!eos_ && !scan_node_->ReachedLimit());
// Transfer the remaining resources to this new batch in Close().
StartNewParquetRowBatch();
return Status::OK();
}
-Status HdfsParquetScanner::GetNextInternal(RowBatch* row_batch, bool* eos) {
+Status HdfsParquetScanner::GetNextInternal(RowBatch* row_batch) {
if (scan_node_->IsZeroSlotTableScan()) {
// There are no materialized slots, e.g. count(*) over the table. We can serve
// this query from just the file metadata. We don't need to read the column data.
if (row_group_rows_read_ == file_metadata_.num_rows) {
- *eos = true;
+ eos_ = true;
return Status::OK();
}
assemble_rows_timer_.Start();
@@ -363,7 +370,7 @@ Status HdfsParquetScanner::GetNextInternal(RowBatch* row_batch, bool* eos) {
}
RETURN_IF_ERROR(NextRowGroup());
if (row_group_idx_ >= file_metadata_.row_groups.size()) {
- *eos = true;
+ eos_ = true;
DCHECK(parse_status_.ok());
return Status::OK();
}
@@ -374,7 +381,7 @@ Status HdfsParquetScanner::GetNextInternal(RowBatch* row_batch, bool* eos) {
// scan range.
if (!scan_node_->PartitionPassesFilters(context_->partition_descriptor()->id(),
FilterStats::ROW_GROUPS_KEY, context_->filter_ctxs())) {
- *eos = true;
+ eos_ = true;
DCHECK(parse_status_.ok());
return Status::OK();
}
@@ -406,7 +413,7 @@ Status HdfsParquetScanner::NextRowGroup() {
const parquet::RowGroup& row_group = file_metadata_.row_groups[row_group_idx_];
if (row_group.num_rows == 0) continue;
- const DiskIoMgr::ScanRange* split_range = reinterpret_cast<ScanRangeMetadata*>(
+ const DiskIoMgr::ScanRange* split_range = static_cast<ScanRangeMetadata*>(
metadata_range_->meta_data())->original_split;
HdfsFileDesc* file_desc = scan_node_->GetFileDesc(filename());
RETURN_IF_ERROR(ParquetMetadataUtils::ValidateColumnOffsets(
@@ -539,7 +546,7 @@ Status HdfsParquetScanner::AssembleRows(
}
void HdfsParquetScanner::StartNewParquetRowBatch() {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
batch_ = new RowBatch(scan_node_->row_desc(), state_->batch_size(),
scan_node_->mem_tracker());
}
@@ -609,7 +616,7 @@ int HdfsParquetScanner::TransferScratchTuples(RowBatch* dst_batch) {
return num_row_to_commit;
}
-Status HdfsParquetScanner::Codegen(HdfsScanNode* node,
+Status HdfsParquetScanner::Codegen(HdfsScanNodeBase* node,
const vector<ExprContext*>& conjunct_ctxs, Function** process_scratch_batch_fn) {
*process_scratch_batch_fn = NULL;
if (!node->runtime_state()->codegen_enabled()) {
@@ -929,7 +936,8 @@ Status HdfsParquetScanner::CreateColumnReaders(const TupleDescriptor& tuple_desc
// Update the template tuple to put a NULL in this slot.
Tuple** template_tuple = &template_tuple_map_[&tuple_desc];
if (*template_tuple == NULL) {
- *template_tuple = scan_node_->InitEmptyTemplateTuple(tuple_desc);
+ *template_tuple =
+ Tuple::Create(tuple_desc.byte_size(), template_tuple_pool_.get());
}
(*template_tuple)->SetNull(slot_desc->null_indicator_offset());
continue;
@@ -1110,7 +1118,7 @@ Status HdfsParquetScanner::InitColumns(
}
const DiskIoMgr::ScanRange* split_range =
- reinterpret_cast<ScanRangeMetadata*>(metadata_range_->meta_data())->original_split;
+ static_cast<ScanRangeMetadata*>(metadata_range_->meta_data())->original_split;
// Determine if the column is completely contained within a local split.
bool column_range_local = split_range->expected_local() &&
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-parquet-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-parquet-scanner.h b/be/src/exec/hdfs-parquet-scanner.h
index 425444f..7707083 100644
--- a/be/src/exec/hdfs-parquet-scanner.h
+++ b/be/src/exec/hdfs-parquet-scanner.h
@@ -320,13 +320,12 @@ class BoolColumnReader;
/// the ScannerContext.
class HdfsParquetScanner : public HdfsScanner {
public:
- HdfsParquetScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue);
+ HdfsParquetScanner(HdfsScanNodeBase* scan_node, RuntimeState* state);
virtual ~HdfsParquetScanner() {};
/// Issue just the footer range for each file. We'll then parse the footer and pick
/// out the columns we want.
- static Status IssueInitialRanges(HdfsScanNode* scan_node,
+ static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files);
virtual Status Open(ScannerContext* context);
@@ -335,7 +334,8 @@ class HdfsParquetScanner : public HdfsScanner {
/// Codegen ProcessScratchBatch(). Stores the resulting function in
/// 'process_scratch_batch_fn' if codegen was successful or NULL otherwise.
- static Status Codegen(HdfsScanNode*, const std::vector<ExprContext*>& conjunct_ctxs,
+ static Status Codegen(HdfsScanNodeBase* node,
+ const std::vector<ExprContext*>& conjunct_ctxs,
llvm::Function** process_scratch_batch_fn);
/// The repetition level is set to this value to indicate the end of a row group.
@@ -438,7 +438,7 @@ class HdfsParquetScanner : public HdfsScanner {
const char* filename() const { return metadata_range_->file(); }
- virtual Status GetNextInternal(RowBatch* row_batch, bool* eos);
+ virtual Status GetNextInternal(RowBatch* row_batch);
/// Advances 'row_group_idx_' to the next non-empty row group and initializes
/// the column readers to scan it. Recoverable errors are logged to the runtime
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-rcfile-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-rcfile-scanner.cc b/be/src/exec/hdfs-rcfile-scanner.cc
index c670d28..f43b2aa 100644
--- a/be/src/exec/hdfs-rcfile-scanner.cc
+++ b/be/src/exec/hdfs-rcfile-scanner.cc
@@ -53,9 +53,8 @@ const uint8_t HdfsRCFileScanner::RCFILE_VERSION_HEADER[4] = {'R', 'C', 'F', 1};
// Macro to convert between SerdeUtil errors to Status returns.
#define RETURN_IF_FALSE(x) if (UNLIKELY(!(x))) return parse_status_
-HdfsRCFileScanner::HdfsRCFileScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue)
- : BaseSequenceScanner(scan_node, state, add_batches_to_queue) {
+HdfsRCFileScanner::HdfsRCFileScanner(HdfsScanNodeBase* scan_node, RuntimeState* state)
+ : BaseSequenceScanner(scan_node, state) {
}
HdfsRCFileScanner::~HdfsRCFileScanner() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-rcfile-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-rcfile-scanner.h b/be/src/exec/hdfs-rcfile-scanner.h
index fe83648..4e18bc4 100644
--- a/be/src/exec/hdfs-rcfile-scanner.h
+++ b/be/src/exec/hdfs-rcfile-scanner.h
@@ -233,8 +233,7 @@ class Tuple;
/// A scanner for reading RCFiles into tuples.
class HdfsRCFileScanner : public BaseSequenceScanner {
public:
- HdfsRCFileScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue);
+ HdfsRCFileScanner(HdfsScanNodeBase* scan_node, RuntimeState* state);
virtual ~HdfsRCFileScanner();
virtual Status Open(ScannerContext* context);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scan-node-base.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.cc b/be/src/exec/hdfs-scan-node-base.cc
new file mode 100644
index 0000000..dba67b5
--- /dev/null
+++ b/be/src/exec/hdfs-scan-node-base.cc
@@ -0,0 +1,893 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/hdfs-scan-node-base.h"
+#include "exec/base-sequence-scanner.h"
+#include "exec/hdfs-text-scanner.h"
+#include "exec/hdfs-lzo-text-scanner.h"
+#include "exec/hdfs-sequence-scanner.h"
+#include "exec/hdfs-rcfile-scanner.h"
+#include "exec/hdfs-avro-scanner.h"
+#include "exec/hdfs-parquet-scanner.h"
+
+#include <sstream>
+#include <avro/errors.h>
+#include <avro/schema.h>
+#include <boost/filesystem.hpp>
+#include <gutil/strings/substitute.h>
+
+#include "codegen/llvm-codegen.h"
+#include "common/logging.h"
+#include "common/object-pool.h"
+#include "exprs/expr-context.h"
+#include "runtime/descriptors.h"
+#include "runtime/hdfs-fs-cache.h"
+#include "runtime/runtime-filter.inline.h"
+#include "runtime/runtime-state.h"
+#include "runtime/mem-pool.h"
+#include "runtime/mem-tracker.h"
+#include "runtime/raw-value.h"
+#include "runtime/row-batch.h"
+#include "runtime/string-buffer.h"
+#include "scheduling/query-resource-mgr.h"
+#include "util/bit-util.h"
+#include "util/container-util.h"
+#include "util/debug-util.h"
+#include "util/disk-info.h"
+#include "util/error-util.h"
+#include "util/hdfs-util.h"
+#include "util/impalad-metrics.h"
+#include "util/periodic-counter-updater.h"
+#include "util/runtime-profile-counters.h"
+
+#include "gen-cpp/PlanNodes_types.h"
+
+#include "common/names.h"
+
+DEFINE_int32(runtime_filter_wait_time_ms, 1000, "(Advanced) the maximum time, in ms, "
+ "that a scan node will wait for expected runtime filters to arrive.");
+DEFINE_bool(suppress_unknown_disk_id_warnings, false,
+ "Suppress unknown disk id warnings generated when the HDFS implementation does not"
+ " provide volume/disk information.");
+
+#ifndef NDEBUG
+DECLARE_bool(skip_file_runtime_filtering);
+#endif
+
+namespace filesystem = boost::filesystem;
+using namespace impala;
+using namespace llvm;
+using namespace strings;
+using boost::algorithm::join;
+
+const string HdfsScanNodeBase::HDFS_SPLIT_STATS_DESC =
+ "Hdfs split stats (<volume id>:<# splits>/<split lengths>)";
+
+// Determines how many unexpected remote bytes trigger an error in the runtime state
+const int UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD = 64 * 1024 * 1024;
+
+HdfsScanNodeBase::HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode,
+ const DescriptorTbl& descs)
+ : ScanNode(pool, tnode, descs),
+ runtime_state_(NULL),
+ skip_header_line_count_(tnode.hdfs_scan_node.__isset.skip_header_line_count ?
+ tnode.hdfs_scan_node.skip_header_line_count : 0),
+ tuple_id_(tnode.hdfs_scan_node.tuple_id),
+ reader_context_(NULL),
+ tuple_desc_(NULL),
+ hdfs_table_(NULL),
+ unknown_disk_id_warned_(false),
+ initial_ranges_issued_(false),
+ counters_running_(false),
+ max_compressed_text_file_length_(NULL),
+ disks_accessed_bitmap_(TUnit::UNIT, 0),
+ bytes_read_local_(NULL),
+ bytes_read_short_circuit_(NULL),
+ bytes_read_dn_cache_(NULL),
+ num_remote_ranges_(NULL),
+ unexpected_remote_bytes_(NULL) {
+}
+
+HdfsScanNodeBase::~HdfsScanNodeBase() {
+}
+
+Status HdfsScanNodeBase::Init(const TPlanNode& tnode, RuntimeState* state) {
+ RETURN_IF_ERROR(ExecNode::Init(tnode, state));
+
+ // Add collection item conjuncts
+ for (const auto& entry: tnode.hdfs_scan_node.collection_conjuncts) {
+ DCHECK(conjuncts_map_[entry.first].empty());
+ RETURN_IF_ERROR(
+ Expr::CreateExprTrees(pool_, entry.second, &conjuncts_map_[entry.first]));
+ }
+
+ const TQueryOptions& query_options = state->query_options();
+ for (const TRuntimeFilterDesc& filter: tnode.runtime_filters) {
+ auto it = filter.planid_to_target_ndx.find(tnode.node_id);
+ DCHECK(it != filter.planid_to_target_ndx.end());
+ const TRuntimeFilterTargetDesc& target = filter.targets[it->second];
+ if (state->query_options().runtime_filter_mode == TRuntimeFilterMode::LOCAL &&
+ !target.is_local_target) {
+ continue;
+ }
+ if (query_options.disable_row_runtime_filtering &&
+ !target.is_bound_by_partition_columns) {
+ continue;
+ }
+
+ FilterContext filter_ctx;
+ RETURN_IF_ERROR(Expr::CreateExprTree(pool_, target.target_expr, &filter_ctx.expr));
+ filter_ctx.filter = state->filter_bank()->RegisterFilter(filter, false);
+
+ string filter_profile_title = Substitute("Filter $0 ($1)", filter.filter_id,
+ PrettyPrinter::Print(filter_ctx.filter->filter_size(), TUnit::BYTES));
+ RuntimeProfile* profile = state->obj_pool()->Add(
+ new RuntimeProfile(state->obj_pool(), filter_profile_title));
+ runtime_profile_->AddChild(profile);
+ filter_ctx.stats = state->obj_pool()->Add(new FilterStats(profile,
+ target.is_bound_by_partition_columns));
+
+ filter_ctxs_.push_back(filter_ctx);
+ }
+
+ // Add row batch conjuncts
+ DCHECK(conjuncts_map_[tuple_id_].empty());
+ conjuncts_map_[tuple_id_] = conjunct_ctxs_;
+
+ return Status::OK();
+}
+
+/// TODO: Break up this very long function.
+Status HdfsScanNodeBase::Prepare(RuntimeState* state) {
+ SCOPED_TIMER(runtime_profile_->total_time_counter());
+ runtime_state_ = state;
+ RETURN_IF_ERROR(ScanNode::Prepare(state));
+
+ tuple_desc_ = state->desc_tbl().GetTupleDescriptor(tuple_id_);
+ DCHECK(tuple_desc_ != NULL);
+
+ // Prepare collection conjuncts
+ for (const auto& entry: conjuncts_map_) {
+ TupleDescriptor* tuple_desc = state->desc_tbl().GetTupleDescriptor(entry.first);
+ // conjuncts_ are already prepared in ExecNode::Prepare(), don't try to prepare again
+ if (tuple_desc == tuple_desc_) continue;
+ RowDescriptor* collection_row_desc =
+ state->obj_pool()->Add(new RowDescriptor(tuple_desc, /* is_nullable */ false));
+ RETURN_IF_ERROR(
+ Expr::Prepare(entry.second, state, *collection_row_desc, expr_mem_tracker()));
+ }
+
+ // One-time initialisation of state that is constant across scan ranges
+ DCHECK(tuple_desc_->table_desc() != NULL);
+ hdfs_table_ = static_cast<const HdfsTableDescriptor*>(tuple_desc_->table_desc());
+ scan_node_pool_.reset(new MemPool(mem_tracker()));
+
+ for (FilterContext& filter: filter_ctxs_) {
+ RETURN_IF_ERROR(filter.expr->Prepare(state, row_desc(), expr_mem_tracker()));
+ AddExprCtxToFree(filter.expr);
+ }
+
+ // Parse Avro table schema if applicable
+ const string& avro_schema_str = hdfs_table_->avro_schema();
+ if (!avro_schema_str.empty()) {
+ avro_schema_t avro_schema;
+ int error = avro_schema_from_json_length(
+ avro_schema_str.c_str(), avro_schema_str.size(), &avro_schema);
+ if (error != 0) {
+ return Status(Substitute("Failed to parse table schema: $0", avro_strerror()));
+ }
+ RETURN_IF_ERROR(AvroSchemaElement::ConvertSchema(avro_schema, avro_schema_.get()));
+ }
+
+ // Gather materialized partition-key slots and non-partition slots.
+ const vector<SlotDescriptor*>& slots = tuple_desc_->slots();
+ for (size_t i = 0; i < slots.size(); ++i) {
+ if (hdfs_table_->IsClusteringCol(slots[i])) {
+ partition_key_slots_.push_back(slots[i]);
+ } else {
+ materialized_slots_.push_back(slots[i]);
+ }
+ }
+
+ // Order the materialized slots such that for schemaless file formats (e.g. text) the
+ // order corresponds to the physical order in files. For formats where the file schema
+ // is independent of the table schema (e.g. Avro, Parquet), this step is not necessary.
+ sort(materialized_slots_.begin(), materialized_slots_.end(),
+ SlotDescriptor::ColPathLessThan);
+
+ // Populate mapping from slot path to index into materialized_slots_.
+ for (int i = 0; i < materialized_slots_.size(); ++i) {
+ path_to_materialized_slot_idx_[materialized_slots_[i]->col_path()] = i;
+ }
+
+ // Initialize is_materialized_col_
+ is_materialized_col_.resize(hdfs_table_->num_cols());
+ for (int i = 0; i < hdfs_table_->num_cols(); ++i) {
+ is_materialized_col_[i] = GetMaterializedSlotIdx(vector<int>(1, i)) != SKIP_COLUMN;
+ }
+
+ HdfsFsCache::HdfsFsMap fs_cache;
+ // Convert the TScanRangeParams into per-file DiskIO::ScanRange objects and populate
+ // partition_ids_, file_descs_, and per_type_files_.
+ DCHECK(scan_range_params_ != NULL)
+ << "Must call SetScanRanges() before calling Prepare()";
+ int num_ranges_missing_volume_id = 0;
+ for (const TScanRangeParams& params: *scan_range_params_) {
+ DCHECK(params.scan_range.__isset.hdfs_file_split);
+ const THdfsFileSplit& split = params.scan_range.hdfs_file_split;
+ partition_ids_.insert(split.partition_id);
+ HdfsPartitionDescriptor* partition_desc =
+ hdfs_table_->GetPartition(split.partition_id);
+ if (partition_desc == NULL) {
+ // TODO: this should be a DCHECK but we sometimes hit it. It's likely IMPALA-1702.
+ LOG(ERROR) << "Bad table descriptor! table_id=" << hdfs_table_->id()
+ << " partition_id=" << split.partition_id
+ << "\n" << PrintThrift(state->fragment_params());
+ return Status("Query encountered invalid metadata, likely due to IMPALA-1702."
+ " Try rerunning the query.");
+ }
+
+ filesystem::path file_path(partition_desc->location());
+ file_path.append(split.file_name, filesystem::path::codecvt());
+ const string& native_file_path = file_path.native();
+
+ HdfsFileDesc* file_desc = NULL;
+ FileDescMap::iterator file_desc_it = file_descs_.find(native_file_path);
+ if (file_desc_it == file_descs_.end()) {
+ // Add new file_desc to file_descs_ and per_type_files_
+ file_desc = runtime_state_->obj_pool()->Add(new HdfsFileDesc(native_file_path));
+ file_descs_[native_file_path] = file_desc;
+ file_desc->file_length = split.file_length;
+ file_desc->mtime = split.mtime;
+ file_desc->file_compression = split.file_compression;
+ RETURN_IF_ERROR(HdfsFsCache::instance()->GetConnection(
+ native_file_path, &file_desc->fs, &fs_cache));
+ num_unqueued_files_.Add(1);
+ per_type_files_[partition_desc->file_format()].push_back(file_desc);
+ } else {
+ // File already processed
+ file_desc = file_desc_it->second;
+ }
+
+ bool expected_local = params.__isset.is_remote && !params.is_remote;
+ if (expected_local && params.volume_id == -1) {
+ if (!FLAGS_suppress_unknown_disk_id_warnings && !unknown_disk_id_warned_) {
+ runtime_profile()->AppendExecOption("Missing Volume Id");
+ runtime_state()->LogError(ErrorMsg(TErrorCode::HDFS_SCAN_NODE_UNKNOWN_DISK));
+ unknown_disk_id_warned_ = true;
+ }
+ ++num_ranges_missing_volume_id;
+ }
+
+ bool try_cache = params.is_cached;
+ if (runtime_state_->query_options().disable_cached_reads) {
+ DCHECK(!try_cache) << "Params should not have had this set.";
+ }
+ file_desc->splits.push_back(
+ AllocateScanRange(file_desc->fs, file_desc->filename.c_str(), split.length,
+ split.offset, split.partition_id, params.volume_id,
+ try_cache, expected_local, file_desc->mtime));
+ }
+
+ // Prepare all the partitions scanned by the scan node
+ for (int64_t partition_id: partition_ids_) {
+ HdfsPartitionDescriptor* partition_desc = hdfs_table_->GetPartition(partition_id);
+ // This is IMPALA-1702, but will have been caught earlier in this method.
+ DCHECK(partition_desc != NULL) << "table_id=" << hdfs_table_->id()
+ << " partition_id=" << partition_id
+ << "\n" << PrintThrift(state->fragment_params());
+ RETURN_IF_ERROR(partition_desc->PrepareExprs(state));
+ }
+
+ // Update server wide metrics for number of scan ranges and ranges that have
+ // incomplete metadata.
+ ImpaladMetrics::NUM_RANGES_PROCESSED->Increment(scan_range_params_->size());
+ ImpaladMetrics::NUM_RANGES_MISSING_VOLUME_ID->Increment(num_ranges_missing_volume_id);
+
+ // Add per volume stats to the runtime profile
+ PerVolumnStats per_volume_stats;
+ stringstream str;
+ UpdateHdfsSplitStats(*scan_range_params_, &per_volume_stats);
+ PrintHdfsSplitStats(per_volume_stats, &str);
+ runtime_profile()->AddInfoString(HDFS_SPLIT_STATS_DESC, str.str());
+
+ // Create codegen'd functions
+ for (int format = THdfsFileFormat::TEXT;
+ format <= THdfsFileFormat::PARQUET; ++format) {
+ vector<HdfsFileDesc*>& file_descs =
+ per_type_files_[static_cast<THdfsFileFormat::type>(format)];
+
+ if (file_descs.empty()) continue;
+
+ // Randomize the order this node processes the files. We want to do this to avoid
+ // issuing remote reads to the same DN from different impalads. In file formats such
+ // as avro/seq/rc (i.e. splittable with a header), every node first reads the header.
+ // If every node goes through the files in the same order, all the remote reads are
+ // for the same file meaning a few DN serves a lot of remote reads at the same time.
+ random_shuffle(file_descs.begin(), file_descs.end());
+
+ // Create reusable codegen'd functions for each file type type needed
+ // TODO: do this for conjuncts_map_
+ Function* fn;
+ Status status;
+ switch (format) {
+ case THdfsFileFormat::TEXT:
+ status = HdfsTextScanner::Codegen(this, conjunct_ctxs_, &fn);
+ break;
+ case THdfsFileFormat::SEQUENCE_FILE:
+ status = HdfsSequenceScanner::Codegen(this, conjunct_ctxs_, &fn);
+ break;
+ case THdfsFileFormat::AVRO:
+ status = HdfsAvroScanner::Codegen(this, conjunct_ctxs_, &fn);
+ break;
+ case THdfsFileFormat::PARQUET:
+ status = HdfsParquetScanner::Codegen(this, conjunct_ctxs_, &fn);
+ break;
+ default:
+ // No codegen for this format
+ fn = NULL;
+ status = Status("Not implemented for this format.");
+ }
+ DCHECK(fn != NULL || !status.ok());
+
+ const char* format_name = _THdfsFileFormat_VALUES_TO_NAMES.find(format)->second;
+ if (!status.ok()) {
+ runtime_profile()->AddCodegenMsg(false, status, format_name);
+ } else {
+ runtime_profile()->AddCodegenMsg(true, status, format_name);
+ LlvmCodeGen* codegen;
+ RETURN_IF_ERROR(runtime_state_->GetCodegen(&codegen));
+ codegen->AddFunctionToJit(
+ fn, &codegend_fn_map_[static_cast<THdfsFileFormat::type>(format)]);
+ }
+ }
+
+ return Status::OK();
+}
+
+Status HdfsScanNodeBase::Open(RuntimeState* state) {
+ RETURN_IF_ERROR(ExecNode::Open(state));
+
+ if (file_descs_.empty()) return Status::OK();
+
+ // Open collection conjuncts
+ for (const auto& entry: conjuncts_map_) {
+ // conjuncts_ are already opened in ExecNode::Open()
+ if (entry.first == tuple_id_) continue;
+ RETURN_IF_ERROR(Expr::Open(entry.second, state));
+ }
+
+ for (FilterContext& filter: filter_ctxs_) RETURN_IF_ERROR(filter.expr->Open(state));
+
+ // Open all the partition exprs used by the scan node and create template tuples.
+ for (int64_t partition_id: partition_ids_) {
+ HdfsPartitionDescriptor* partition_desc = hdfs_table_->GetPartition(partition_id);
+ DCHECK(partition_desc != NULL) << "table_id=" << hdfs_table_->id()
+ << " partition_id=" << partition_id
+ << "\n" << PrintThrift(state->fragment_params());
+ RETURN_IF_ERROR(partition_desc->OpenExprs(state));
+ vector<ExprContext*> partition_key_value_ctxs;
+ RETURN_IF_ERROR(Expr::CloneIfNotExists(
+ partition_desc->partition_key_value_ctxs(), state, &partition_key_value_ctxs));
+ partition_template_tuple_map_[partition_id] =
+ InitTemplateTuple(partition_key_value_ctxs, scan_node_pool_.get(), state);
+ Expr::Close(partition_key_value_ctxs, state);
+ }
+
+ RETURN_IF_ERROR(runtime_state_->io_mgr()->RegisterContext(
+ &reader_context_, mem_tracker()));
+
+ // Initialize HdfsScanNode specific counters
+ // TODO: Revisit counters and move the counters specific to multi-threaded scans
+ // into HdfsScanNode.
+ read_timer_ = ADD_TIMER(runtime_profile(), TOTAL_HDFS_READ_TIMER);
+ per_read_thread_throughput_counter_ = runtime_profile()->AddDerivedCounter(
+ PER_READ_THREAD_THROUGHPUT_COUNTER, TUnit::BYTES_PER_SECOND,
+ bind<int64_t>(&RuntimeProfile::UnitsPerSecond, bytes_read_counter_, read_timer_));
+ scan_ranges_complete_counter_ =
+ ADD_COUNTER(runtime_profile(), SCAN_RANGES_COMPLETE_COUNTER, TUnit::UNIT);
+ if (DiskInfo::num_disks() < 64) {
+ num_disks_accessed_counter_ =
+ ADD_COUNTER(runtime_profile(), NUM_DISKS_ACCESSED_COUNTER, TUnit::UNIT);
+ } else {
+ num_disks_accessed_counter_ = NULL;
+ }
+ num_scanner_threads_started_counter_ =
+ ADD_COUNTER(runtime_profile(), NUM_SCANNER_THREADS_STARTED, TUnit::UNIT);
+
+ runtime_state_->io_mgr()->set_bytes_read_counter(reader_context_, bytes_read_counter());
+ runtime_state_->io_mgr()->set_read_timer(reader_context_, read_timer());
+ runtime_state_->io_mgr()->set_active_read_thread_counter(reader_context_,
+ &active_hdfs_read_thread_counter_);
+ runtime_state_->io_mgr()->set_disks_access_bitmap(reader_context_,
+ &disks_accessed_bitmap_);
+
+ average_scanner_thread_concurrency_ = runtime_profile()->AddSamplingCounter(
+ AVERAGE_SCANNER_THREAD_CONCURRENCY, &active_scanner_thread_counter_);
+ average_hdfs_read_thread_concurrency_ = runtime_profile()->AddSamplingCounter(
+ AVERAGE_HDFS_READ_THREAD_CONCURRENCY, &active_hdfs_read_thread_counter_);
+
+ bytes_read_local_ = ADD_COUNTER(runtime_profile(), "BytesReadLocal",
+ TUnit::BYTES);
+ bytes_read_short_circuit_ = ADD_COUNTER(runtime_profile(), "BytesReadShortCircuit",
+ TUnit::BYTES);
+ bytes_read_dn_cache_ = ADD_COUNTER(runtime_profile(), "BytesReadDataNodeCache",
+ TUnit::BYTES);
+ num_remote_ranges_ = ADD_COUNTER(runtime_profile(), "RemoteScanRanges",
+ TUnit::UNIT);
+ unexpected_remote_bytes_ = ADD_COUNTER(runtime_profile(), "BytesReadRemoteUnexpected",
+ TUnit::BYTES);
+
+ max_compressed_text_file_length_ = runtime_profile()->AddHighWaterMarkCounter(
+ "MaxCompressedTextFileLength", TUnit::BYTES);
+
+ for (int i = 0; i < state->io_mgr()->num_total_disks() + 1; ++i) {
+ hdfs_read_thread_concurrency_bucket_.push_back(
+ pool_->Add(new RuntimeProfile::Counter(TUnit::DOUBLE_VALUE, 0)));
+ }
+ runtime_profile()->RegisterBucketingCounters(&active_hdfs_read_thread_counter_,
+ &hdfs_read_thread_concurrency_bucket_);
+
+ counters_running_ = true;
+
+ int64_t total_splits = 0;
+ for (const auto& fd: file_descs_) total_splits += fd.second->splits.size();
+ progress_.Init(Substitute("Splits complete (node=$0)", total_splits), total_splits);
+ return Status::OK();
+}
+
+Status HdfsScanNodeBase::Reset(RuntimeState* state) {
+ DCHECK(false) << "Internal error: Scan nodes should not appear in subplans.";
+ return Status("Internal error: Scan nodes should not appear in subplans.");
+}
+
+void HdfsScanNodeBase::Close(RuntimeState* state) {
+ if (is_closed()) return;
+
+ if (reader_context_ != NULL) {
+ // There may still be io buffers used by parent nodes so we can't unregister the
+ // reader context yet. The runtime state keeps a list of all the reader contexts and
+ // they are unregistered when the fragment is closed.
+ state->reader_contexts()->push_back(reader_context_);
+ // Need to wait for all the active scanner threads to finish to ensure there is no
+ // more memory tracked by this scan node's mem tracker.
+ state->io_mgr()->CancelContext(reader_context_, true);
+ }
+
+ StopAndFinalizeCounters();
+
+ // There should be no active scanner threads and hdfs read threads.
+ DCHECK_EQ(active_scanner_thread_counter_.value(), 0);
+ DCHECK_EQ(active_hdfs_read_thread_counter_.value(), 0);
+
+ if (scan_node_pool_.get() != NULL) scan_node_pool_->FreeAll();
+
+ // Close all the partitions scanned by the scan node
+ for (int64_t partition_id: partition_ids_) {
+ HdfsPartitionDescriptor* partition_desc = hdfs_table_->GetPartition(partition_id);
+ if (partition_desc == NULL) {
+ // TODO: Revert when IMPALA-1702 is fixed.
+ LOG(ERROR) << "Bad table descriptor! table_id=" << hdfs_table_->id()
+ << " partition_id=" << partition_id
+ << "\n" << PrintThrift(state->fragment_params());
+ continue;
+ }
+ partition_desc->CloseExprs(state);
+ }
+
+ // Open collection conjuncts
+ for (const auto& tid_conjunct: conjuncts_map_) {
+ // conjuncts_ are already closed in ExecNode::Close()
+ if (tid_conjunct.first == tuple_id_) continue;
+ Expr::Close(tid_conjunct.second, state);
+ }
+
+ for (auto& filter_ctx: filter_ctxs_) filter_ctx.expr->Close(state);
+ ScanNode::Close(state);
+}
+
+Status HdfsScanNodeBase::IssueInitialScanRanges(RuntimeState* state) {
+ DCHECK(!initial_ranges_issued_);
+ initial_ranges_issued_ = true;
+
+ // No need to issue ranges with limit 0.
+ if (ReachedLimit()) {
+ DCHECK_EQ(limit_, 0);
+ return Status::OK();
+ }
+
+ int32 wait_time_ms = FLAGS_runtime_filter_wait_time_ms;
+ if (state->query_options().runtime_filter_wait_time_ms > 0) {
+ wait_time_ms = state->query_options().runtime_filter_wait_time_ms;
+ }
+ if (filter_ctxs_.size() > 0) WaitForRuntimeFilters(wait_time_ms);
+ // Apply dynamic partition-pruning per-file.
+ FileFormatsMap matching_per_type_files;
+ for (const FileFormatsMap::value_type& v: per_type_files_) {
+ vector<HdfsFileDesc*>* matching_files = &matching_per_type_files[v.first];
+ for (HdfsFileDesc* file: v.second) {
+ if (FilePassesFilterPredicates(filter_ctxs_, v.first, file)) {
+ matching_files->push_back(file);
+ }
+ }
+ }
+
+ // Issue initial ranges for all file types.
+ RETURN_IF_ERROR(HdfsParquetScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::PARQUET]));
+ RETURN_IF_ERROR(HdfsTextScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::TEXT]));
+ RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::SEQUENCE_FILE]));
+ RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::RC_FILE]));
+ RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
+ matching_per_type_files[THdfsFileFormat::AVRO]));
+
+ return Status::OK();
+}
+
+bool HdfsScanNodeBase::FilePassesFilterPredicates(const vector<FilterContext>& filter_ctxs,
+ const THdfsFileFormat::type& format, HdfsFileDesc* file) {
+#ifndef NDEBUG
+ if (FLAGS_skip_file_runtime_filtering) return true;
+#endif
+ if (filter_ctxs_.size() == 0) return true;
+ ScanRangeMetadata* metadata =
+ static_cast<ScanRangeMetadata*>(file->splits[0]->meta_data());
+ if (!PartitionPassesFilters(metadata->partition_id, FilterStats::FILES_KEY,
+ filter_ctxs)) {
+ for (int j = 0; j < file->splits.size(); ++j) {
+ // Mark range as complete to ensure progress.
+ RangeComplete(format, file->file_compression);
+ }
+ return false;
+ }
+ return true;
+}
+
+bool HdfsScanNodeBase::WaitForRuntimeFilters(int32_t time_ms) {
+ vector<string> arrived_filter_ids;
+ int32_t start = MonotonicMillis();
+ for (auto& ctx: filter_ctxs_) {
+ if (ctx.filter->WaitForArrival(time_ms)) {
+ arrived_filter_ids.push_back(Substitute("$0", ctx.filter->id()));
+ }
+ }
+ int32_t end = MonotonicMillis();
+ const string& wait_time = PrettyPrinter::Print(end - start, TUnit::TIME_MS);
+
+ if (arrived_filter_ids.size() == filter_ctxs_.size()) {
+ runtime_profile()->AddInfoString("Runtime filters",
+ Substitute("All filters arrived. Waited $0", wait_time));
+ VLOG_QUERY << "Filters arrived. Waited " << wait_time;
+ return true;
+ }
+
+ const string& filter_str = Substitute("Only following filters arrived: $0, waited $1",
+ join(arrived_filter_ids, ", "), wait_time);
+ runtime_profile()->AddInfoString("Runtime filters", filter_str);
+ VLOG_QUERY << filter_str;
+ return false;
+}
+
+DiskIoMgr::ScanRange* HdfsScanNodeBase::AllocateScanRange(
+ hdfsFS fs, const char* file, int64_t len, int64_t offset, int64_t partition_id,
+ int disk_id, bool try_cache, bool expected_local, int64_t mtime,
+ const DiskIoMgr::ScanRange* original_split) {
+ DCHECK_GE(disk_id, -1);
+ // Require that the scan range is within [0, file_length). While this cannot be used
+ // to guarantee safety (file_length metadata may be stale), it avoids different
+ // behavior between Hadoop FileSystems (e.g. s3n hdfsSeek() returns error when seeking
+ // beyond the end of the file).
+ DCHECK_GE(offset, 0);
+ DCHECK_GE(len, 0);
+ DCHECK_LE(offset + len, GetFileDesc(file)->file_length)
+ << "Scan range beyond end of file (offset=" << offset << ", len=" << len << ")";
+ disk_id = runtime_state_->io_mgr()->AssignQueue(file, disk_id, expected_local);
+
+ ScanRangeMetadata* metadata = runtime_state_->obj_pool()->Add(
+ new ScanRangeMetadata(partition_id, original_split));
+ DiskIoMgr::ScanRange* range =
+ runtime_state_->obj_pool()->Add(new DiskIoMgr::ScanRange());
+ range->Reset(fs, file, len, offset, disk_id, try_cache, expected_local,
+ mtime, metadata);
+ return range;
+}
+
+Status HdfsScanNodeBase::AddDiskIoRanges(const vector<DiskIoMgr::ScanRange*>& ranges,
+ int num_files_queued) {
+ RETURN_IF_ERROR(
+ runtime_state_->io_mgr()->AddScanRanges(reader_context_, ranges));
+ num_unqueued_files_.Add(-num_files_queued);
+ DCHECK_GE(num_unqueued_files_.Load(), 0);
+ return Status::OK();
+}
+
+HdfsFileDesc* HdfsScanNodeBase::GetFileDesc(const string& filename) {
+ DCHECK(file_descs_.find(filename) != file_descs_.end());
+ return file_descs_[filename];
+}
+
+void* HdfsScanNodeBase::GetCodegenFn(THdfsFileFormat::type type) {
+ CodegendFnMap::iterator it = codegend_fn_map_.find(type);
+ if (it == codegend_fn_map_.end()) return NULL;
+ return it->second;
+}
+
+Status HdfsScanNodeBase::CreateAndOpenScanner(HdfsPartitionDescriptor* partition,
+ ScannerContext* context, scoped_ptr<HdfsScanner>* scanner) {
+ DCHECK(context != NULL);
+ THdfsCompression::type compression =
+ context->GetStream()->file_desc()->file_compression;
+
+ // Create a new scanner for this file format and compression.
+ switch (partition->file_format()) {
+ case THdfsFileFormat::TEXT:
+ // Lzo-compressed text files are scanned by a scanner that it is implemented as a
+ // dynamic library, so that Impala does not include GPL code.
+ if (compression == THdfsCompression::LZO) {
+ scanner->reset(HdfsLzoTextScanner::GetHdfsLzoTextScanner(this, runtime_state_));
+ } else {
+ scanner->reset(new HdfsTextScanner(this, runtime_state_));
+ }
+ break;
+ case THdfsFileFormat::SEQUENCE_FILE:
+ scanner->reset(new HdfsSequenceScanner(this, runtime_state_));
+ break;
+ case THdfsFileFormat::RC_FILE:
+ scanner->reset(new HdfsRCFileScanner(this, runtime_state_));
+ break;
+ case THdfsFileFormat::AVRO:
+ scanner->reset(new HdfsAvroScanner(this, runtime_state_));
+ break;
+ case THdfsFileFormat::PARQUET:
+ scanner->reset(new HdfsParquetScanner(this, runtime_state_));
+ break;
+ default:
+ return Status(Substitute("Unknown Hdfs file format type: $0",
+ partition->file_format()));
+ }
+ DCHECK(scanner->get() != NULL);
+ Status status = ExecDebugAction(TExecNodePhase::PREPARE_SCANNER, runtime_state_);
+ if (status.ok()) {
+ status = scanner->get()->Open(context);
+ if (!status.ok()) scanner->get()->Close(scanner->get()->batch());
+ } else {
+ context->ClearStreams();
+ }
+ return status;
+}
+
+Tuple* HdfsScanNodeBase::InitTemplateTuple(const vector<ExprContext*>& value_ctxs,
+ MemPool* pool, RuntimeState* state) const {
+ if (partition_key_slots_.empty()) return NULL;
+ Tuple* template_tuple = Tuple::Create(tuple_desc_->byte_size(), pool);
+ for (int i = 0; i < partition_key_slots_.size(); ++i) {
+ const SlotDescriptor* slot_desc = partition_key_slots_[i];
+ ExprContext* value_ctx = value_ctxs[slot_desc->col_pos()];
+ /// This function may be called from multiple threads, and we expect each
+ /// thread to pass in their own cloned value contexts.
+ DCHECK(value_ctx->is_clone());
+ // Exprs guaranteed to be literals, so can safely be evaluated without a row.
+ RawValue::Write(value_ctx->GetValue(NULL), template_tuple, slot_desc, NULL);
+ }
+ return template_tuple;
+}
+
+void HdfsScanNodeBase::InitNullCollectionValues(const TupleDescriptor* tuple_desc,
+ Tuple* tuple) const {
+ for (const SlotDescriptor* slot_desc: tuple_desc->collection_slots()) {
+ CollectionValue* slot = reinterpret_cast<CollectionValue*>(
+ tuple->GetSlot(slot_desc->tuple_offset()));
+ if (tuple->IsNull(slot_desc->null_indicator_offset())) {
+ *slot = CollectionValue();
+ continue;
+ }
+ // Recursively traverse collection items.
+ const TupleDescriptor* item_desc = slot_desc->collection_item_descriptor();
+ if (item_desc->collection_slots().empty()) continue;
+ for (int i = 0; i < slot->num_tuples; ++i) {
+ int item_offset = i * item_desc->byte_size();
+ Tuple* collection_item = reinterpret_cast<Tuple*>(slot->ptr + item_offset);
+ InitNullCollectionValues(item_desc, collection_item);
+ }
+ }
+}
+
+void HdfsScanNodeBase::InitNullCollectionValues(RowBatch* row_batch) const {
+ DCHECK_EQ(row_batch->row_desc().tuple_descriptors().size(), 1);
+ const TupleDescriptor& tuple_desc =
+ *row_batch->row_desc().tuple_descriptors()[tuple_idx()];
+ if (tuple_desc.collection_slots().empty()) return;
+ for (int i = 0; i < row_batch->num_rows(); ++i) {
+ Tuple* tuple = row_batch->GetRow(i)->GetTuple(tuple_idx());
+ DCHECK(tuple != NULL);
+ InitNullCollectionValues(&tuple_desc, tuple);
+ }
+}
+
+bool HdfsScanNodeBase::PartitionPassesFilters(int32_t partition_id,
+ const string& stats_name, const vector<FilterContext>& filter_ctxs) {
+ if (filter_ctxs.size() == 0) return true;
+ DCHECK_EQ(filter_ctxs.size(), filter_ctxs_.size())
+ << "Mismatched number of filter contexts";
+ Tuple* template_tuple = partition_template_tuple_map_[partition_id];
+ // Defensive - if template_tuple is NULL, there can be no filters on partition columns.
+ if (template_tuple == NULL) return true;
+ TupleRow* tuple_row_mem = reinterpret_cast<TupleRow*>(&template_tuple);
+ for (const FilterContext& ctx: filter_ctxs) {
+ int target_ndx = ctx.filter->filter_desc().planid_to_target_ndx.at(id_);
+ if (!ctx.filter->filter_desc().targets[target_ndx].is_bound_by_partition_columns) {
+ continue;
+ }
+ void* e = ctx.expr->GetValue(tuple_row_mem);
+
+ // Not quite right because bitmap could arrive after Eval(), but we're ok with
+ // off-by-one errors.
+ bool processed = ctx.filter->HasBloomFilter();
+ bool passed_filter = ctx.filter->Eval<void>(e, ctx.expr->root()->type());
+ ctx.stats->IncrCounters(stats_name, 1, processed, !passed_filter);
+ if (!passed_filter) return false;
+ }
+
+ return true;
+}
+
+void HdfsScanNodeBase::RangeComplete(const THdfsFileFormat::type& file_type,
+ const THdfsCompression::type& compression_type) {
+ vector<THdfsCompression::type> types;
+ types.push_back(compression_type);
+ RangeComplete(file_type, types);
+}
+
+void HdfsScanNodeBase::RangeComplete(const THdfsFileFormat::type& file_type,
+ const vector<THdfsCompression::type>& compression_types) {
+ scan_ranges_complete_counter()->Add(1);
+ progress_.Update(1);
+ for (int i = 0; i < compression_types.size(); ++i) {
+ ++file_type_counts_[make_pair(file_type, compression_types[i])];
+ }
+}
+
+void HdfsScanNodeBase::ComputeSlotMaterializationOrder(vector<int>* order) const {
+ const vector<ExprContext*>& conjuncts = ExecNode::conjunct_ctxs();
+ // Initialize all order to be conjuncts.size() (after the last conjunct)
+ order->insert(order->begin(), materialized_slots().size(), conjuncts.size());
+
+ const DescriptorTbl& desc_tbl = runtime_state_->desc_tbl();
+
+ vector<SlotId> slot_ids;
+ for (int conjunct_idx = 0; conjunct_idx < conjuncts.size(); ++conjunct_idx) {
+ slot_ids.clear();
+ int num_slots = conjuncts[conjunct_idx]->root()->GetSlotIds(&slot_ids);
+ for (int j = 0; j < num_slots; ++j) {
+ SlotDescriptor* slot_desc = desc_tbl.GetSlotDescriptor(slot_ids[j]);
+ int slot_idx = GetMaterializedSlotIdx(slot_desc->col_path());
+ // slot_idx == -1 means this was a partition key slot which is always
+ // materialized before any slots.
+ if (slot_idx == -1) continue;
+ // If this slot hasn't been assigned an order, assign it be materialized
+ // before evaluating conjuncts[i]
+ if ((*order)[slot_idx] == conjuncts.size()) {
+ (*order)[slot_idx] = conjunct_idx;
+ }
+ }
+ }
+}
+
+void HdfsScanNodeBase::UpdateHdfsSplitStats(
+ const vector<TScanRangeParams>& scan_range_params_list,
+ PerVolumnStats* per_volume_stats) {
+ pair<int, int64_t> init_value(0, 0);
+ for (const TScanRangeParams& scan_range_params: scan_range_params_list) {
+ const TScanRange& scan_range = scan_range_params.scan_range;
+ if (!scan_range.__isset.hdfs_file_split) continue;
+ const THdfsFileSplit& split = scan_range.hdfs_file_split;
+ pair<int, int64_t>* stats =
+ FindOrInsert(per_volume_stats, scan_range_params.volume_id, init_value);
+ ++(stats->first);
+ stats->second += split.length;
+ }
+}
+
+void HdfsScanNodeBase::PrintHdfsSplitStats(const PerVolumnStats& per_volume_stats,
+ stringstream* ss) {
+ for (PerVolumnStats::const_iterator i = per_volume_stats.begin();
+ i != per_volume_stats.end(); ++i) {
+ (*ss) << i->first << ":" << i->second.first << "/"
+ << PrettyPrinter::Print(i->second.second, TUnit::BYTES) << " ";
+ }
+}
+
+void HdfsScanNodeBase::StopAndFinalizeCounters() {
+ if (!counters_running_) return;
+ counters_running_ = false;
+
+ PeriodicCounterUpdater::StopTimeSeriesCounter(bytes_read_timeseries_counter_);
+ PeriodicCounterUpdater::StopRateCounter(total_throughput_counter());
+ PeriodicCounterUpdater::StopSamplingCounter(average_scanner_thread_concurrency_);
+ PeriodicCounterUpdater::StopSamplingCounter(average_hdfs_read_thread_concurrency_);
+ PeriodicCounterUpdater::StopBucketingCounters(&hdfs_read_thread_concurrency_bucket_,
+ true);
+
+ // Output hdfs read thread concurrency into info string
+ stringstream ss;
+ for (int i = 0; i < hdfs_read_thread_concurrency_bucket_.size(); ++i) {
+ ss << i << ":" << setprecision(4)
+ << hdfs_read_thread_concurrency_bucket_[i]->double_value() << "% ";
+ }
+ runtime_profile_->AddInfoString("Hdfs Read Thread Concurrency Bucket", ss.str());
+
+ // Convert disk access bitmap to num of disk accessed
+ uint64_t num_disk_bitmap = disks_accessed_bitmap_.value();
+ int64_t num_disk_accessed = BitUtil::Popcount(num_disk_bitmap);
+ if (num_disks_accessed_counter_ != NULL) {
+ num_disks_accessed_counter_->Set(num_disk_accessed);
+ }
+
+ // output completed file types and counts to info string
+ if (!file_type_counts_.empty()) {
+ stringstream ss;
+ {
+ for (FileTypeCountsMap::const_iterator it = file_type_counts_.begin();
+ it != file_type_counts_.end(); ++it) {
+ ss << it->first.first << "/" << it->first.second << ":" << it->second << " ";
+ }
+ }
+ runtime_profile_->AddInfoString("File Formats", ss.str());
+ }
+
+ // Output fraction of scanners with codegen enabled
+ int num_enabled = num_scanners_codegen_enabled_.Load();
+ int total = num_enabled + num_scanners_codegen_disabled_.Load();
+ runtime_profile()->AppendExecOption(
+ Substitute("Codegen enabled: $0 out of $1", num_enabled, total));
+
+ if (reader_context_ != NULL) {
+ bytes_read_local_->Set(runtime_state_->io_mgr()->bytes_read_local(reader_context_));
+ bytes_read_short_circuit_->Set(
+ runtime_state_->io_mgr()->bytes_read_short_circuit(reader_context_));
+ bytes_read_dn_cache_->Set(
+ runtime_state_->io_mgr()->bytes_read_dn_cache(reader_context_));
+ num_remote_ranges_->Set(static_cast<int64_t>(
+ runtime_state_->io_mgr()->num_remote_ranges(reader_context_)));
+ unexpected_remote_bytes_->Set(
+ runtime_state_->io_mgr()->unexpected_remote_bytes(reader_context_));
+
+ if (unexpected_remote_bytes_->value() >= UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD) {
+ runtime_state_->LogError(ErrorMsg(TErrorCode::GENERAL, Substitute(
+ "Read $0 of data across network that was expected to be local. "
+ "Block locality metadata for table '$1.$2' may be stale. Consider running "
+ "\"INVALIDATE METADATA `$1`.`$2`\".",
+ PrettyPrinter::Print(unexpected_remote_bytes_->value(), TUnit::BYTES),
+ hdfs_table_->database(), hdfs_table_->name())));
+ }
+
+ ImpaladMetrics::IO_MGR_BYTES_READ->Increment(bytes_read_counter()->value());
+ ImpaladMetrics::IO_MGR_LOCAL_BYTES_READ->Increment(
+ bytes_read_local_->value());
+ ImpaladMetrics::IO_MGR_SHORT_CIRCUIT_BYTES_READ->Increment(
+ bytes_read_short_circuit_->value());
+ ImpaladMetrics::IO_MGR_CACHED_BYTES_READ->Increment(
+ bytes_read_dn_cache_->value());
+ }
+}
+
+Status HdfsScanNodeBase::TriggerDebugAction() {
+ return ExecDebugAction(TExecNodePhase::GETNEXT, runtime_state_);
+}
[6/7] incubator-impala git commit: IMPALA-3905: Add single-threaded
scan node.
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scan-node-base.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-base.h b/be/src/exec/hdfs-scan-node-base.h
new file mode 100644
index 0000000..7ea4b9d
--- /dev/null
+++ b/be/src/exec/hdfs-scan-node-base.h
@@ -0,0 +1,459 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#ifndef IMPALA_EXEC_HDFS_SCAN_NODE_BASE_H_
+#define IMPALA_EXEC_HDFS_SCAN_NODE_BASE_H_
+
+#include <stdint.h>
+#include <memory>
+#include <unordered_set>
+#include <vector>
+
+#include <boost/unordered_map.hpp>
+#include <boost/scoped_ptr.hpp>
+
+#include "exec/filter-context.h"
+#include "exec/scan-node.h"
+#include "runtime/descriptors.h"
+#include "runtime/disk-io-mgr.h"
+#include "util/avro-util.h"
+#include "util/progress-updater.h"
+#include "util/spinlock.h"
+
+namespace impala {
+
+class ScannerContext;
+class DescriptorTbl;
+class HdfsScanner;
+class RowBatch;
+class Status;
+class Tuple;
+class TPlanNode;
+class TScanRange;
+
+/// Maintains per file information for files assigned to this scan node. This includes
+/// all the splits for the file. Note that it is not thread-safe.
+struct HdfsFileDesc {
+ HdfsFileDesc(const std::string& filename)
+ : fs(NULL), filename(filename), file_length(0), mtime(0),
+ file_compression(THdfsCompression::NONE) {
+ }
+
+ /// Connection to the filesystem containing the file.
+ hdfsFS fs;
+
+ /// File name including the path.
+ std::string filename;
+
+ /// Length of the file. This is not related to which parts of the file have been
+ /// assigned to this node.
+ int64_t file_length;
+
+ /// Last modified time
+ int64_t mtime;
+
+ THdfsCompression::type file_compression;
+
+ /// Splits (i.e. raw byte ranges) for this file, assigned to this scan node.
+ std::vector<DiskIoMgr::ScanRange*> splits;
+};
+
+/// Struct for additional metadata for scan ranges. This contains the partition id
+/// that this scan range is for.
+struct ScanRangeMetadata {
+ /// The partition id that this range is part of.
+ int64_t partition_id;
+
+ /// For parquet scan ranges we initially create a request for the file footer for each
+ /// split; we store a pointer to the actual split so that we can recover its information
+ /// for the scanner to process.
+ const DiskIoMgr::ScanRange* original_split;
+
+ ScanRangeMetadata(int64_t partition_id, const DiskIoMgr::ScanRange* original_split)
+ : partition_id(partition_id), original_split(original_split) { }
+};
+
+
+/// Base class for all Hdfs scan nodes. Contains common members and functions
+/// that are independent of whether batches are materialized by the main thread
+/// (via HdfsScanner::GexNext()) or by spinning up separate threads that feed
+/// into a RowBatch queue (via HdfsScanner::ProcessSplit()). Those specifics
+/// are expected to be implemented in subclasses.
+///
+/// Subclasses may expect to receive runtime filters produced elsewhere in the plan
+/// (even from remote fragments). These filters arrive asynchronously during execution,
+/// and are applied as soon as they arrive. Filters may be applied by the scan node in
+/// the following scopes:
+///
+/// 1. Per-file (all file formats, partition column filters only) - filtering at this
+/// scope saves IO as the filters are applied before scan ranges are issued.
+/// 2. Per-scan-range (all file formats, partition column filters only) - filtering at
+/// this scope saves CPU as filtered scan ranges are never scanned.
+///
+/// Scanners may also use the same filters to eliminate rows at finer granularities
+/// (e.g. per row).
+///
+/// TODO: Revisit and minimize metrics. Move those specific to legacy multi-threaded
+/// scans into HdfsScanNode.
+/// TODO: Once the legacy scan node has been removed, several functions can be made
+/// non-virtual. Also merge this class with HdfsScanNodeMt.
+class HdfsScanNodeBase : public ScanNode {
+ public:
+ HdfsScanNodeBase(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
+ ~HdfsScanNodeBase();
+
+ virtual Status Init(const TPlanNode& tnode, RuntimeState* state);
+ virtual Status Prepare(RuntimeState* state);
+ virtual Status Open(RuntimeState* state);
+ virtual Status Reset(RuntimeState* state);
+ virtual void Close(RuntimeState* state);
+
+ /// Returns true if this node uses separate threads for scanners that append RowBatches
+ /// to a queue, false otherwise.
+ virtual bool HasRowBatchQueue() const = 0;
+
+ int limit() const { return limit_; }
+
+ const std::vector<SlotDescriptor*>& materialized_slots()
+ const { return materialized_slots_; }
+
+ /// Returns the tuple idx into the row for this scan node to output to.
+ /// Currently this is always 0.
+ int tuple_idx() const { return 0; }
+
+ /// Returns number of partition keys in the table.
+ int num_partition_keys() const { return hdfs_table_->num_clustering_cols(); }
+
+ /// Returns number of partition key slots.
+ int num_materialized_partition_keys() const { return partition_key_slots_.size(); }
+
+ const TupleDescriptor* tuple_desc() const { return tuple_desc_; }
+ const HdfsTableDescriptor* hdfs_table() { return hdfs_table_; }
+ const AvroSchemaElement& avro_schema() { return *avro_schema_.get(); }
+ RuntimeState* runtime_state() { return runtime_state_; }
+ int skip_header_line_count() const { return skip_header_line_count_; }
+ DiskIoRequestContext* reader_context() { return reader_context_; }
+
+ typedef std::map<TupleId, std::vector<ExprContext*>> ConjunctsMap;
+ const ConjunctsMap& conjuncts_map() const { return conjuncts_map_; }
+
+ RuntimeProfile::HighWaterMarkCounter* max_compressed_text_file_length() {
+ return max_compressed_text_file_length_;
+ }
+
+ const static int SKIP_COLUMN = -1;
+
+ /// Returns index into materialized_slots with 'path'. Returns SKIP_COLUMN if
+ /// that path is not materialized.
+ int GetMaterializedSlotIdx(const std::vector<int>& path) const {
+ PathToSlotIdxMap::const_iterator result = path_to_materialized_slot_idx_.find(path);
+ if (result == path_to_materialized_slot_idx_.end()) return SKIP_COLUMN;
+ return result->second;
+ }
+
+ /// The result array is of length hdfs_table_->num_cols(). The i-th element is true iff
+ /// column i should be materialized.
+ const bool* is_materialized_col() {
+ return reinterpret_cast<const bool*>(&is_materialized_col_[0]);
+ }
+
+ /// Returns the per format codegen'd function. Scanners call this to get the
+ /// codegen'd function to use. Returns NULL if codegen should not be used.
+ void* GetCodegenFn(THdfsFileFormat::type);
+
+ inline void IncNumScannersCodegenEnabled() { num_scanners_codegen_enabled_.Add(1); }
+ inline void IncNumScannersCodegenDisabled() { num_scanners_codegen_disabled_.Add(1); }
+
+ /// Allocate a new scan range object, stored in the runtime state's object pool. For
+ /// scan ranges that correspond to the original hdfs splits, the partition id must be
+ /// set to the range's partition id. For other ranges (e.g. columns in parquet, read
+ /// past buffers), the partition_id is unused. expected_local should be true if this
+ /// scan range is not expected to require a remote read. The range must fall within
+ /// the file bounds. That is, the offset must be >= 0, and offset + len <= file_length.
+ /// If not NULL, the 'original_split' pointer is stored for reference in the scan range
+ /// metadata of the scan range that is to be allocated.
+ /// This is thread safe.
+ DiskIoMgr::ScanRange* AllocateScanRange(
+ hdfsFS fs, const char* file, int64_t len, int64_t offset, int64_t partition_id,
+ int disk_id, bool try_cache, bool expected_local, int64_t mtime,
+ const DiskIoMgr::ScanRange* original_split = NULL);
+
+ /// Adds ranges to the io mgr queue. 'num_files_queued' indicates how many file's scan
+ /// ranges have been added completely. A file's scan ranges are added completely if no
+ /// new scanner threads will be needed to process that file besides the additional
+ /// threads needed to process those in 'ranges'.
+ /// Can be overridden to add scan-node specific actions like starting scanner threads.
+ virtual Status AddDiskIoRanges(const std::vector<DiskIoMgr::ScanRange*>& ranges,
+ int num_files_queued);
+
+ /// Adds all splits for file_desc to the io mgr queue and indicates one file has
+ /// been added completely.
+ inline Status AddDiskIoRanges(const HdfsFileDesc* file_desc) {
+ return AddDiskIoRanges(file_desc->splits, 1);
+ }
+
+ /// Allocates and initializes a new template tuple allocated from pool with values
+ /// from the partition columns for the current scan range, if any,
+ /// Returns NULL if there are no partition keys slots.
+ Tuple* InitTemplateTuple(const std::vector<ExprContext*>& value_ctxs,
+ MemPool* pool, RuntimeState* state) const;
+
+ /// Returns the file desc for 'filename'. Returns NULL if filename is invalid.
+ HdfsFileDesc* GetFileDesc(const std::string& filename);
+
+ /// Called by scanners when a range is complete. Used to record progress.
+ /// This *must* only be called after a scanner has completely finished its
+ /// scan range (i.e. context->Flush()), and has returned the final row batch.
+ /// Otherwise, scan nodes using a RowBatch queue may lose the last batch due
+ /// to racing with shutting down the queue.
+ void RangeComplete(const THdfsFileFormat::type& file_type,
+ const THdfsCompression::type& compression_type);
+ /// Same as above except for when multiple compression codecs were used
+ /// in the file. The metrics are incremented for each compression_type.
+ virtual void RangeComplete(const THdfsFileFormat::type& file_type,
+ const std::vector<THdfsCompression::type>& compression_type);
+
+ /// Utility function to compute the order in which to materialize slots to allow for
+ /// computing conjuncts as slots get materialized (on partial tuples).
+ /// 'order' will contain for each slot, the first conjunct it is associated with.
+ /// e.g. order[2] = 1 indicates materialized_slots[2] must be materialized before
+ /// evaluating conjuncts[1]. Slots that are not referenced by any conjuncts will have
+ /// order set to conjuncts.size()
+ void ComputeSlotMaterializationOrder(std::vector<int>* order) const;
+
+ /// Returns true if there are no materialized slots, such as a count(*) over the table.
+ inline bool IsZeroSlotTableScan() const {
+ return materialized_slots().empty() && tuple_desc()->tuple_path().empty();
+ }
+
+ /// map from volume id to <number of split, per volume split lengths>
+ typedef boost::unordered_map<int32_t, std::pair<int, int64_t>> PerVolumnStats;
+
+ /// Update the per volume stats with the given scan range params list
+ static void UpdateHdfsSplitStats(
+ const std::vector<TScanRangeParams>& scan_range_params_list,
+ PerVolumnStats* per_volume_stats);
+
+ /// Output the per_volume_stats to stringstream. The output format is a list of:
+ /// <volume id>:<# splits>/<per volume split lengths>
+ static void PrintHdfsSplitStats(const PerVolumnStats& per_volume_stats,
+ std::stringstream* ss);
+
+ /// Description string for the per volume stats output.
+ static const std::string HDFS_SPLIT_STATS_DESC;
+
+ /// Returns true if partition 'partition_id' passes all the filter predicates in
+ /// 'filter_ctxs' and should not be filtered out. 'stats_name' is the key of one of the
+ /// counter groups in FilterStats, and is used to update the correct statistics.
+ ///
+ /// 'filter_ctxs' is either an empty list, in which case filtering is disabled and the
+ /// function returns true, or a set of filter contexts to evaluate.
+ bool PartitionPassesFilters(int32_t partition_id, const std::string& stats_name,
+ const std::vector<FilterContext>& filter_ctxs);
+
+ const std::vector<FilterContext>& filter_ctxs() const { return filter_ctxs_; }
+
+ protected:
+ friend class ScannerContext;
+ friend class HdfsScanner;
+
+ RuntimeState* runtime_state_;
+
+ // Number of header lines to skip at the beginning of each file of this table. Only set
+ // to values > 0 for hdfs text files.
+ const int skip_header_line_count_;
+
+ /// Tuple id resolved in Prepare() to set tuple_desc_
+ const int tuple_id_;
+
+ /// RequestContext object to use with the disk-io-mgr for reads.
+ DiskIoRequestContext* reader_context_;
+
+ /// Descriptor for tuples this scan node constructs
+ const TupleDescriptor* tuple_desc_;
+
+ /// Map from partition ID to a template tuple (owned by scan_node_pool_) which has only
+ /// the partition columns for that partition materialized. Used to filter files and scan
+ /// ranges on partition-column filters. Populated in Prepare().
+ boost::unordered_map<int64_t, Tuple*> partition_template_tuple_map_;
+
+ /// Descriptor for the hdfs table, including partition and format metadata.
+ /// Set in Prepare, owned by RuntimeState
+ const HdfsTableDescriptor* hdfs_table_;
+
+ /// The root of the table's Avro schema, if we're scanning an Avro table.
+ ScopedAvroSchemaElement avro_schema_;
+
+ /// If true, the warning that some disk ids are unknown was logged. Only log this once
+ /// per scan node since it can be noisy.
+ bool unknown_disk_id_warned_;
+
+ /// Partitions scanned by this scan node.
+ std::unordered_set<int64_t> partition_ids_;
+
+ /// File path => file descriptor (which includes the file's splits)
+ typedef std::map<std::string, HdfsFileDesc*> FileDescMap;
+ FileDescMap file_descs_;
+
+ /// File format => file descriptors.
+ typedef std::map<THdfsFileFormat::type, std::vector<HdfsFileDesc*>> FileFormatsMap;
+ FileFormatsMap per_type_files_;
+
+ /// Conjuncts for each materialized tuple (top-level row batch tuples and collection
+ /// item tuples). Includes a copy of ExecNode.conjuncts_.
+ ConjunctsMap conjuncts_map_;
+
+ /// Set to true when the initial scan ranges are issued to the IoMgr. This happens on
+ /// the first call to GetNext(). The token manager, in a different thread, will read
+ /// this variable.
+ bool initial_ranges_issued_;
+
+ /// Number of files that have not been issued from the scanners.
+ AtomicInt32 num_unqueued_files_;
+
+ /// Per scanner type codegen'd fn.
+ typedef std::map<THdfsFileFormat::type, void*> CodegendFnMap;
+ CodegendFnMap codegend_fn_map_;
+
+ /// Maps from a slot's path to its index into materialized_slots_.
+ typedef boost::unordered_map<std::vector<int>, int> PathToSlotIdxMap;
+ PathToSlotIdxMap path_to_materialized_slot_idx_;
+
+ /// List of contexts for expected runtime filters for this scan node. These contexts are
+ /// cloned by individual scanners to be used in multi-threaded contexts, passed through
+ /// the per-scanner ScannerContext..
+ std::vector<FilterContext> filter_ctxs_;
+
+ /// is_materialized_col_[i] = <true i-th column should be materialized, false otherwise>
+ /// for 0 <= i < total # columns in table
+ //
+ /// This should be a vector<bool>, but bool vectors are special-cased and not stored
+ /// internally as arrays, so instead we store as chars and cast to bools as needed
+ std::vector<char> is_materialized_col_;
+
+ /// Vector containing slot descriptors for all non-partition key slots. These
+ /// descriptors are sorted in order of increasing col_pos.
+ std::vector<SlotDescriptor*> materialized_slots_;
+
+ /// Vector containing slot descriptors for all partition key slots.
+ std::vector<SlotDescriptor*> partition_key_slots_;
+
+ /// Keeps track of total splits and the number finished.
+ ProgressUpdater progress_;
+
+ /// Counters which track the number of scanners that have codegen enabled for the
+ /// materialize and conjuncts evaluation code paths.
+ AtomicInt32 num_scanners_codegen_enabled_;
+ AtomicInt32 num_scanners_codegen_disabled_;
+
+ /// This is the number of io buffers that are owned by the scan node and the scanners.
+ /// This is used just to help debug leaked io buffers to determine if the leak is
+ /// happening in the scanners vs other parts of the execution.
+ /// TODO: Remove this counter when deprecating the multi-threaded scan node.
+ AtomicInt32 num_owned_io_buffers_;
+
+ /// If true, counters are actively running and need to be reported in the runtime
+ /// profile.
+ bool counters_running_;
+
+ /// The size of the largest compressed text file to be scanned. This is used to
+ /// estimate scanner thread memory usage.
+ RuntimeProfile::HighWaterMarkCounter* max_compressed_text_file_length_;
+
+ /// Disk accessed bitmap
+ RuntimeProfile::Counter disks_accessed_bitmap_;
+
+ /// Total number of bytes read locally
+ RuntimeProfile::Counter* bytes_read_local_;
+
+ /// Total number of bytes read via short circuit read
+ RuntimeProfile::Counter* bytes_read_short_circuit_;
+
+ /// Total number of bytes read from data node cache
+ RuntimeProfile::Counter* bytes_read_dn_cache_;
+
+ /// Total number of remote scan ranges
+ RuntimeProfile::Counter* num_remote_ranges_;
+
+ /// Total number of bytes read remotely that were expected to be local
+ RuntimeProfile::Counter* unexpected_remote_bytes_;
+
+ /// Pool for allocating some amounts of memory that is shared between scanners.
+ /// e.g. partition key tuple and their string buffers
+ boost::scoped_ptr<MemPool> scan_node_pool_;
+
+ /// Status of failed operations. This is set in the ScannerThreads
+ /// Returned in GetNext() if an error occurred. An non-ok status triggers cleanup
+ /// scanner threads.
+ Status status_;
+
+ /// Mapping of file formats (file type, compression type) to the number of
+ /// splits of that type and the lock protecting it.
+ typedef std::map<
+ std::pair<THdfsFileFormat::type, THdfsCompression::type>, int> FileTypeCountsMap;
+ FileTypeCountsMap file_type_counts_;
+
+ /// Performs dynamic partition pruning, i.e., applies runtime filters to files, and
+ /// issues initial ranges for all file types. Waits for runtime filters if necessary.
+ /// Only valid to call if !initial_ranges_issued_. Sets initial_ranges_issued_ to true.
+ Status IssueInitialScanRanges(RuntimeState* state);
+
+ /// Create and open new scanner for this partition type.
+ /// If the scanner is successfully created, it is returned in 'scanner'.
+ /// Passes 'add_batches_to_queue' to the scanner constructor.
+ Status CreateAndOpenScanner(HdfsPartitionDescriptor* partition,
+ ScannerContext* context, boost::scoped_ptr<HdfsScanner>* scanner);
+
+ /// Recursively initializes all NULL collection slots to an empty CollectionValue in
+ /// addition to maintaining the null bit. Hack to allow UnnestNode to project out
+ /// collection slots. Assumes that the null bit has already been un/set.
+ /// TODO: Remove this function once the TODOs in UnnestNode regarding projection
+ /// have been addressed.
+ void InitNullCollectionValues(const TupleDescriptor* tuple_desc, Tuple* tuple) const;
+
+ /// Helper to call InitNullCollectionValues() on all tuples produced by this scan
+ /// in 'row_batch'.
+ void InitNullCollectionValues(RowBatch* row_batch) const;
+
+ /// Returns false if, according to filters in 'filter_ctxs', 'file' should be filtered
+ /// and therefore not processed. 'file_type' is the the format of 'file', and is used
+ /// for bookkeeping. Returns true if all filters pass or are not present.
+ bool FilePassesFilterPredicates(const std::vector<FilterContext>& filter_ctxs,
+ const THdfsFileFormat::type& file_type, HdfsFileDesc* file);
+
+ /// Waits for up to time_ms for runtime filters to arrive, checking every 20ms. Returns
+ /// true if all filters arrived within the time limit (as measured from the time of
+ /// RuntimeFilterBank::RegisterFilter()), false otherwise.
+ bool WaitForRuntimeFilters(int32_t time_ms);
+
+ /// Stops periodic counters and aggregates counter values for the entire scan node.
+ /// This should be called as soon as the scan node is complete to get the most accurate
+ /// counter values.
+ /// This can be called multiple times, subsequent calls will be ignored.
+ /// This must be called on Close() to unregister counters.
+ /// Scan nodes with a RowBatch queue may have to synchronize calls to this function.
+ void StopAndFinalizeCounters();
+
+ /// Calls ExecDebugAction(). Returns the status based on the debug action specified
+ /// for the query.
+ Status TriggerDebugAction();
+};
+
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scan-node-mt.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-mt.cc b/be/src/exec/hdfs-scan-node-mt.cc
new file mode 100644
index 0000000..8c61326
--- /dev/null
+++ b/be/src/exec/hdfs-scan-node-mt.cc
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "exec/hdfs-scan-node-mt.h"
+
+#include <sstream>
+
+#include "runtime/runtime-state.h"
+#include "runtime/row-batch.h"
+#include "util/debug-util.h"
+#include "util/runtime-profile-counters.h"
+
+#include "gen-cpp/PlanNodes_types.h"
+
+using std::stringstream;
+
+namespace impala {
+
+HdfsScanNodeMt::HdfsScanNodeMt(ObjectPool* pool, const TPlanNode& tnode,
+ const DescriptorTbl& descs)
+ : HdfsScanNodeBase(pool, tnode, descs),
+ scan_range_(NULL),
+ scanner_(NULL) {
+}
+
+HdfsScanNodeMt::~HdfsScanNodeMt() {
+}
+
+Status HdfsScanNodeMt::Prepare(RuntimeState* state) {
+ RETURN_IF_ERROR(HdfsScanNodeBase::Prepare(state));
+ // Return an error if this scan node has been assigned a range that is not supported
+ // because the scanner of the corresponding file format does implement GetNext().
+ for (const auto& files: per_type_files_) {
+ if (!files.second.empty() && files.first != THdfsFileFormat::PARQUET) {
+ stringstream msg;
+ msg << "Unsupported file format with HdfsScanNodeMt: " << files.first;
+ return Status(msg.str());
+ }
+ }
+ return Status::OK();
+}
+
+Status HdfsScanNodeMt::Open(RuntimeState* state) {
+ SCOPED_TIMER(runtime_profile_->total_time_counter());
+ RETURN_IF_ERROR(HdfsScanNodeBase::Open(state));
+ DCHECK(!initial_ranges_issued_);
+ RETURN_IF_ERROR(IssueInitialScanRanges(state));
+ return Status::OK();
+}
+
+Status HdfsScanNodeMt::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos) {
+ SCOPED_TIMER(runtime_profile_->total_time_counter());
+ RETURN_IF_ERROR(ExecDebugAction(TExecNodePhase::GETNEXT, state));
+ RETURN_IF_CANCELLED(state);
+ RETURN_IF_ERROR(QueryMaintenance(state));
+
+ DCHECK(scan_range_ == NULL || scanner_ != NULL);
+ if (scan_range_ == NULL || scanner_->eos()) {
+ if (scanner_ != NULL && scanner_->eos()) {
+ scanner_->Close(row_batch);
+ scanner_.reset();
+ }
+ RETURN_IF_ERROR(
+ runtime_state_->io_mgr()->GetNextRange(reader_context_, &scan_range_));
+ if (scan_range_ == NULL) {
+ *eos = true;
+ StopAndFinalizeCounters();
+ return Status::OK();
+ }
+ ScanRangeMetadata* metadata =
+ static_cast<ScanRangeMetadata*>(scan_range_->meta_data());
+ int64_t partition_id = metadata->partition_id;
+ HdfsPartitionDescriptor* partition = hdfs_table_->GetPartition(partition_id);
+ scanner_ctx_.reset(new ScannerContext(
+ runtime_state_, this, partition, scan_range_, filter_ctxs()));
+ RETURN_IF_ERROR(CreateAndOpenScanner(partition, scanner_ctx_.get(), &scanner_));
+ }
+
+ Status status = scanner_->GetNext(row_batch);
+ if (!status.ok()) {
+ scanner_->Close(row_batch);
+ scanner_.reset();
+ num_owned_io_buffers_.Add(-row_batch->num_io_buffers());
+ return status;
+ }
+
+ num_rows_returned_ += row_batch->num_rows();
+ if (ReachedLimit()) {
+ int num_rows_over = num_rows_returned_ - limit_;
+ row_batch->set_num_rows(row_batch->num_rows() - num_rows_over);
+ num_rows_returned_ -= num_rows_over;
+ scan_range_ = NULL;
+ scanner_->Close(row_batch);
+ scanner_.reset();
+ *eos = true;
+ }
+ COUNTER_SET(rows_returned_counter_, num_rows_returned_);
+ num_owned_io_buffers_.Add(-row_batch->num_io_buffers());
+
+ if (*eos) StopAndFinalizeCounters();
+ return Status::OK();
+}
+
+void HdfsScanNodeMt::Close(RuntimeState* state) {
+ if (is_closed()) return;
+ scanner_.reset();
+ scanner_ctx_.reset();
+ HdfsScanNodeBase::Close(state);
+}
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scan-node-mt.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node-mt.h b/be/src/exec/hdfs-scan-node-mt.h
new file mode 100644
index 0000000..7829d47
--- /dev/null
+++ b/be/src/exec/hdfs-scan-node-mt.h
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#ifndef IMPALA_EXEC_HDFS_SCAN_NODE_MT_H_
+#define IMPALA_EXEC_HDFS_SCAN_NODE_MT_H_
+
+#include <boost/scoped_ptr.hpp>
+
+#include "exec/hdfs-scanner.h"
+#include "exec/hdfs-scan-node-base.h"
+#include "exec/scanner-context.h"
+
+namespace impala {
+
+class DescriptorTbl;
+class ObjectPool;
+class RuntimeState;
+class RowBatch;
+class TPlanNode;
+
+/// Scan node that materializes tuples, evaluates conjuncts and runtime filters
+/// in the thread calling GetNext(). Uses the HdfsScanner::GetNext() interface.
+class HdfsScanNodeMt : public HdfsScanNodeBase {
+ public:
+ HdfsScanNodeMt(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
+ ~HdfsScanNodeMt();
+
+ virtual Status Prepare(RuntimeState* state);
+ virtual Status Open(RuntimeState* state);
+ virtual Status GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos);
+ virtual void Close(RuntimeState* state);
+
+ virtual bool HasRowBatchQueue() const { return false; }
+
+ private:
+ /// Current scan range and corresponding scanner.
+ DiskIoMgr::ScanRange* scan_range_;
+ boost::scoped_ptr<ScannerContext> scanner_ctx_;
+ boost::scoped_ptr<HdfsScanner> scanner_;
+};
+
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scan-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.cc b/be/src/exec/hdfs-scan-node.cc
index e0cee4d..83f6452 100644
--- a/be/src/exec/hdfs-scan-node.cc
+++ b/be/src/exec/hdfs-scan-node.cc
@@ -16,70 +16,31 @@
// under the License.
#include "exec/hdfs-scan-node.h"
-#include "exec/base-sequence-scanner.h"
-#include "exec/hdfs-text-scanner.h"
-#include "exec/hdfs-lzo-text-scanner.h"
-#include "exec/hdfs-sequence-scanner.h"
-#include "exec/hdfs-rcfile-scanner.h"
-#include "exec/hdfs-avro-scanner.h"
-#include "exec/hdfs-parquet-scanner.h"
#include <sstream>
-#include <avro/errors.h>
-#include <avro/schema.h>
-#include <boost/algorithm/string.hpp>
-#include <boost/filesystem.hpp>
-#include <gutil/strings/substitute.h>
-#include <hdfs.h>
-
-#include "codegen/llvm-codegen.h"
#include "common/logging.h"
-#include "common/object-pool.h"
-#include "exprs/expr-context.h"
+#include "exec/hdfs-scanner.h"
+#include "exec/scanner-context.h"
#include "runtime/descriptors.h"
-#include "runtime/hdfs-fs-cache.h"
#include "runtime/runtime-filter.inline.h"
#include "runtime/runtime-state.h"
-#include "runtime/mem-pool.h"
#include "runtime/mem-tracker.h"
-#include "runtime/raw-value.h"
#include "runtime/row-batch.h"
-#include "runtime/string-buffer.h"
#include "scheduling/query-resource-mgr.h"
-#include "util/bit-util.h"
-#include "util/container-util.h"
#include "util/debug-util.h"
#include "util/disk-info.h"
-#include "util/error-util.h"
-#include "util/hdfs-util.h"
-#include "util/impalad-metrics.h"
-#include "util/periodic-counter-updater.h"
#include "util/runtime-profile-counters.h"
-#include "gen-cpp/PlanNodes_types.h"
-
#include "common/names.h"
DEFINE_int32(max_row_batches, 0, "the maximum size of materialized_row_batches_");
-DEFINE_bool(suppress_unknown_disk_id_warnings, false,
- "Suppress unknown disk id warnings generated when the HDFS implementation does not"
- " provide volume/disk information.");
-DEFINE_int32(runtime_filter_wait_time_ms, 1000, "(Advanced) the maximum time, in ms, "
- "that a scan node will wait for expected runtime filters to arrive.");
#ifndef NDEBUG
DECLARE_bool(skip_file_runtime_filtering);
#endif
-namespace filesystem = boost::filesystem;
using namespace impala;
-using namespace llvm;
-using namespace strings;
-using boost::algorithm::join;
-
-const string HdfsScanNode::HDFS_SPLIT_STATS_DESC =
- "Hdfs split stats (<volume id>:<# splits>/<split lengths>)";
// Amount of memory that we approximate a scanner thread will use not including IoBuffers.
// The memory used does not vary considerably between file formats (just a couple of MBs).
@@ -91,37 +52,17 @@ const int SCANNER_THREAD_MEM_USAGE = 32 * 1024 * 1024;
// estimate scanner thread memory usage.
const int COMPRESSED_TEXT_COMPRESSION_RATIO = 11;
-// Determines how many unexpected remote bytes trigger an error in the runtime state
-const int UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD = 64 * 1024 * 1024;
-
// Amount of time to block waiting for GetNext() to release scanner threads between
// checking if a scanner thread should yield itself back to the global thread pool.
const int SCANNER_THREAD_WAIT_TIME_MS = 20;
HdfsScanNode::HdfsScanNode(ObjectPool* pool, const TPlanNode& tnode,
const DescriptorTbl& descs)
- : ScanNode(pool, tnode, descs),
- runtime_state_(NULL),
- skip_header_line_count_(tnode.hdfs_scan_node.__isset.skip_header_line_count ?
- tnode.hdfs_scan_node.skip_header_line_count : 0),
- tuple_id_(tnode.hdfs_scan_node.tuple_id),
- reader_context_(NULL),
- tuple_desc_(NULL),
- hdfs_table_(NULL),
- unknown_disk_id_warned_(false),
- initial_ranges_issued_(false),
+ : HdfsScanNodeBase(pool, tnode, descs),
ranges_issued_barrier_(1),
scanner_thread_bytes_required_(0),
- max_compressed_text_file_length_(NULL),
- disks_accessed_bitmap_(TUnit::UNIT, 0),
- bytes_read_local_(NULL),
- bytes_read_short_circuit_(NULL),
- bytes_read_dn_cache_(NULL),
- num_remote_ranges_(NULL),
- unexpected_remote_bytes_(NULL),
done_(false),
all_ranges_started_(false),
- counters_running_(false),
thread_avail_cb_id_(-1),
rm_callback_id_(-1),
max_num_scanner_threads_(CpuInfo::num_cores()) {
@@ -139,99 +80,6 @@ HdfsScanNode::HdfsScanNode(ObjectPool* pool, const TPlanNode& tnode,
HdfsScanNode::~HdfsScanNode() {
}
-Status HdfsScanNode::Init(const TPlanNode& tnode, RuntimeState* state) {
- RETURN_IF_ERROR(ExecNode::Init(tnode, state));
-
- // Add collection item conjuncts
- const map<TTupleId, vector<TExpr>>& collection_conjuncts =
- tnode.hdfs_scan_node.collection_conjuncts;
- map<TTupleId, vector<TExpr>>::const_iterator iter = collection_conjuncts.begin();
- for (; iter != collection_conjuncts.end(); ++iter) {
- DCHECK(conjuncts_map_[iter->first].empty());
- RETURN_IF_ERROR(
- Expr::CreateExprTrees(pool_, iter->second, &conjuncts_map_[iter->first]));
- }
-
- const TQueryOptions& query_options = state->query_options();
- for (const TRuntimeFilterDesc& filter: tnode.runtime_filters) {
- auto it = filter.planid_to_target_ndx.find(tnode.node_id);
- DCHECK(it != filter.planid_to_target_ndx.end());
- const TRuntimeFilterTargetDesc& target = filter.targets[it->second];
- if (state->query_options().runtime_filter_mode == TRuntimeFilterMode::LOCAL &&
- !target.is_local_target) {
- continue;
- }
- if (query_options.disable_row_runtime_filtering &&
- !target.is_bound_by_partition_columns) {
- continue;
- }
-
- FilterContext filter_ctx;
- RETURN_IF_ERROR(Expr::CreateExprTree(pool_, target.target_expr, &filter_ctx.expr));
- filter_ctx.filter = state->filter_bank()->RegisterFilter(filter, false);
-
- string filter_profile_title = Substitute("Filter $0 ($1)", filter.filter_id,
- PrettyPrinter::Print(filter_ctx.filter->filter_size(), TUnit::BYTES));
- RuntimeProfile* profile = state->obj_pool()->Add(
- new RuntimeProfile(state->obj_pool(), filter_profile_title));
- runtime_profile_->AddChild(profile);
- filter_ctx.stats = state->obj_pool()->Add(new FilterStats(profile,
- target.is_bound_by_partition_columns));
-
- filter_ctxs_.push_back(filter_ctx);
- }
-
- // Add row batch conjuncts
- DCHECK(conjuncts_map_[tuple_id_].empty());
- conjuncts_map_[tuple_id_] = conjunct_ctxs_;
-
- return Status::OK();
-}
-
-bool HdfsScanNode::FilePassesFilterPredicates(const vector<FilterContext>& filter_ctxs,
- const THdfsFileFormat::type& format, HdfsFileDesc* file) {
-#ifndef NDEBUG
- if (FLAGS_skip_file_runtime_filtering) return true;
-#endif
- if (filter_ctxs_.size() == 0) return true;
- ScanRangeMetadata* metadata =
- reinterpret_cast<ScanRangeMetadata*>(file->splits[0]->meta_data());
- if (!PartitionPassesFilters(metadata->partition_id, FilterStats::FILES_KEY,
- filter_ctxs)) {
- for (int j = 0; j < file->splits.size(); ++j) {
- // Mark range as complete to ensure progress.
- RangeComplete(format, file->file_compression);
- }
- return false;
- }
- return true;
-}
-
-bool HdfsScanNode::WaitForRuntimeFilters(int32_t time_ms) {
- vector<string> arrived_filter_ids;
- int32_t start = MonotonicMillis();
- for (auto& ctx: filter_ctxs_) {
- if (ctx.filter->WaitForArrival(time_ms)) {
- arrived_filter_ids.push_back(Substitute("$0", ctx.filter->id()));
- }
- }
- int32_t end = MonotonicMillis();
- const string& wait_time = PrettyPrinter::Print(end - start, TUnit::TIME_MS);
-
- if (arrived_filter_ids.size() == filter_ctxs_.size()) {
- runtime_profile()->AddInfoString("Runtime filters",
- Substitute("All filters arrived. Waited $0", wait_time));
- VLOG_QUERY << "Filters arrived. Waited " << wait_time;
- return true;
- }
-
- const string& filter_str = Substitute("Only following filters arrived: $0, waited $1",
- join(arrived_filter_ids, ", "), wait_time);
- runtime_profile()->AddInfoString("Runtime filters", filter_str);
- VLOG_QUERY << filter_str;
- return false;
-}
-
Status HdfsScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos) {
SCOPED_TIMER(runtime_profile_->total_time_counter());
@@ -241,35 +89,7 @@ Status HdfsScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos
// so we need to tell them there is work to do.
// TODO: This is probably not worth splitting the organisational cost of splitting
// initialisation across two places. Move to before the scanner threads start.
- initial_ranges_issued_ = true;
-
- int32 wait_time_ms = FLAGS_runtime_filter_wait_time_ms;
- if (state->query_options().runtime_filter_wait_time_ms > 0) {
- wait_time_ms = state->query_options().runtime_filter_wait_time_ms;
- }
- if (filter_ctxs_.size() > 0) WaitForRuntimeFilters(wait_time_ms);
- // Apply dynamic partition-pruning per-file.
- FileFormatsMap matching_per_type_files;
- for (const FileFormatsMap::value_type& v: per_type_files_) {
- vector<HdfsFileDesc*>* matching_files = &matching_per_type_files[v.first];
- for (HdfsFileDesc* file: v.second) {
- if (FilePassesFilterPredicates(filter_ctxs_, v.first, file)) {
- matching_files->push_back(file);
- }
- }
- }
-
- // Issue initial ranges for all file types.
- RETURN_IF_ERROR(HdfsParquetScanner::IssueInitialRanges(this,
- matching_per_type_files[THdfsFileFormat::PARQUET]));
- RETURN_IF_ERROR(HdfsTextScanner::IssueInitialRanges(this,
- matching_per_type_files[THdfsFileFormat::TEXT]));
- RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
- matching_per_type_files[THdfsFileFormat::SEQUENCE_FILE]));
- RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
- matching_per_type_files[THdfsFileFormat::RC_FILE]));
- RETURN_IF_ERROR(BaseSequenceScanner::IssueInitialRanges(this,
- matching_per_type_files[THdfsFileFormat::AVRO]));
+ RETURN_IF_ERROR(IssueInitialScanRanges(state));
// Release the scanner threads
ranges_issued_barrier_.Notify();
@@ -278,7 +98,11 @@ Status HdfsScanNode::GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos
}
Status status = GetNextInternal(state, row_batch, eos);
- if (!status.ok() || *eos) StopAndFinalizeCounters();
+ if (!status.ok() || *eos) {
+ unique_lock<mutex> l(lock_);
+ lock_guard<SpinLock> l2(file_type_counts_);
+ StopAndFinalizeCounters();
+ }
return status;
}
@@ -326,283 +150,16 @@ Status HdfsScanNode::GetNextInternal(
return status_;
}
-DiskIoMgr::ScanRange* HdfsScanNode::AllocateScanRange(
- hdfsFS fs, const char* file, int64_t len, int64_t offset, int64_t partition_id,
- int disk_id, bool try_cache, bool expected_local, int64_t mtime,
- const DiskIoMgr::ScanRange* original_split) {
- DCHECK_GE(disk_id, -1);
- // Require that the scan range is within [0, file_length). While this cannot be used
- // to guarantee safety (file_length metadata may be stale), it avoids different
- // behavior between Hadoop FileSystems (e.g. s3n hdfsSeek() returns error when seeking
- // beyond the end of the file).
- DCHECK_GE(offset, 0);
- DCHECK_GE(len, 0);
- DCHECK_LE(offset + len, GetFileDesc(file)->file_length)
- << "Scan range beyond end of file (offset=" << offset << ", len=" << len << ")";
- disk_id = runtime_state_->io_mgr()->AssignQueue(file, disk_id, expected_local);
-
- ScanRangeMetadata* metadata = runtime_state_->obj_pool()->Add(
- new ScanRangeMetadata(partition_id, original_split));
- DiskIoMgr::ScanRange* range =
- runtime_state_->obj_pool()->Add(new DiskIoMgr::ScanRange());
- range->Reset(fs, file, len, offset, disk_id, try_cache, expected_local,
- mtime, metadata);
- return range;
-}
-
-HdfsFileDesc* HdfsScanNode::GetFileDesc(const string& filename) {
- DCHECK(file_descs_.find(filename) != file_descs_.end());
- return file_descs_[filename];
-}
-
-void HdfsScanNode::SetFileMetadata(const string& filename, void* metadata) {
- unique_lock<mutex> l(metadata_lock_);
- DCHECK(per_file_metadata_.find(filename) == per_file_metadata_.end());
- per_file_metadata_[filename] = metadata;
-}
-
-void* HdfsScanNode::GetFileMetadata(const string& filename) {
- unique_lock<mutex> l(metadata_lock_);
- map<string, void*>::iterator it = per_file_metadata_.find(filename);
- if (it == per_file_metadata_.end()) return NULL;
- return it->second;
-}
-
-void* HdfsScanNode::GetCodegenFn(THdfsFileFormat::type type) {
- CodegendFnMap::iterator it = codegend_fn_map_.find(type);
- if (it == codegend_fn_map_.end()) return NULL;
- return it->second;
-}
-
-Status HdfsScanNode::CreateAndOpenScanner(HdfsPartitionDescriptor* partition,
- ScannerContext* context, scoped_ptr<HdfsScanner>* scanner) {
- DCHECK(context != NULL);
- THdfsCompression::type compression =
- context->GetStream()->file_desc()->file_compression;
-
- // Create a new scanner for this file format and compression.
- switch (partition->file_format()) {
- case THdfsFileFormat::TEXT:
- // Lzo-compressed text files are scanned by a scanner that it is implemented as a
- // dynamic library, so that Impala does not include GPL code.
- if (compression == THdfsCompression::LZO) {
- scanner->reset(HdfsLzoTextScanner::GetHdfsLzoTextScanner(this, runtime_state_));
- } else {
- scanner->reset(new HdfsTextScanner(this, runtime_state_, true));
- }
- break;
- case THdfsFileFormat::SEQUENCE_FILE:
- scanner->reset(new HdfsSequenceScanner(this, runtime_state_, true));
- break;
- case THdfsFileFormat::RC_FILE:
- scanner->reset(new HdfsRCFileScanner(this, runtime_state_, true));
- break;
- case THdfsFileFormat::AVRO:
- scanner->reset(new HdfsAvroScanner(this, runtime_state_, true));
- break;
- case THdfsFileFormat::PARQUET:
- scanner->reset(new HdfsParquetScanner(this, runtime_state_, true));
- break;
- default:
- return Status(Substitute("Unknown Hdfs file format type: $0",
- partition->file_format()));
- }
- DCHECK(scanner->get() != NULL);
- Status status = ExecDebugAction(TExecNodePhase::PREPARE_SCANNER, runtime_state_);
- if (status.ok()) {
- status = scanner->get()->Open(context);
- if (!status.ok()) scanner->get()->Close(scanner->get()->batch());
- } else {
- context->ClearStreams();
- }
- return status;
-}
-
-Tuple* HdfsScanNode::InitTemplateTuple(RuntimeState* state,
- const vector<ExprContext*>& value_ctxs) {
- if (partition_key_slots_.empty()) return NULL;
-
- // Lock to protect access to partition_key_pool_ and value_ctxs
- // TODO: we can push the lock to the mempool and exprs_values should not
- // use internal memory.
- Tuple* template_tuple = InitEmptyTemplateTuple(*tuple_desc_);
-
- unique_lock<mutex> l(lock_);
- for (int i = 0; i < partition_key_slots_.size(); ++i) {
- const SlotDescriptor* slot_desc = partition_key_slots_[i];
- // Exprs guaranteed to be literals, so can safely be evaluated without a row context
- void* value = value_ctxs[slot_desc->col_pos()]->GetValue(NULL);
- RawValue::Write(value, template_tuple, slot_desc, NULL);
- }
- return template_tuple;
-}
-
-Tuple* HdfsScanNode::InitEmptyTemplateTuple(const TupleDescriptor& tuple_desc) {
- Tuple* template_tuple = NULL;
- {
- unique_lock<mutex> l(lock_);
- template_tuple = Tuple::Create(tuple_desc.byte_size(), scan_node_pool_.get());
- }
- memset(template_tuple, 0, tuple_desc.byte_size());
- return template_tuple;
-}
-
-void HdfsScanNode::TransferToScanNodePool(MemPool* pool) {
- unique_lock<mutex> l(lock_);
- scan_node_pool_->AcquireData(pool, false);
-}
-
-Status HdfsScanNode::TriggerDebugAction() {
- return ExecDebugAction(TExecNodePhase::GETNEXT, runtime_state_);
-}
-
Status HdfsScanNode::Prepare(RuntimeState* state) {
SCOPED_TIMER(runtime_profile_->total_time_counter());
- runtime_state_ = state;
- RETURN_IF_ERROR(ScanNode::Prepare(state));
-
- tuple_desc_ = state->desc_tbl().GetTupleDescriptor(tuple_id_);
- DCHECK(tuple_desc_ != NULL);
-
- // Prepare collection conjuncts
- ConjunctsMap::const_iterator iter = conjuncts_map_.begin();
- for (; iter != conjuncts_map_.end(); ++iter) {
- TupleDescriptor* tuple_desc = state->desc_tbl().GetTupleDescriptor(iter->first);
-
- // conjuncts_ are already prepared in ExecNode::Prepare(), don't try to prepare again
- if (tuple_desc == tuple_desc_) continue;
-
- RowDescriptor* collection_row_desc =
- state->obj_pool()->Add(new RowDescriptor(tuple_desc, /* is_nullable */ false));
- RETURN_IF_ERROR(
- Expr::Prepare(iter->second, state, *collection_row_desc, expr_mem_tracker()));
- }
+ RETURN_IF_ERROR(HdfsScanNodeBase::Prepare(state));
+ // Assign scanner thread group to cgroup, if any.
if (!state->cgroup().empty()) {
scanner_threads_.SetCgroupsMgr(state->exec_env()->cgroups_mgr());
scanner_threads_.SetCgroup(state->cgroup());
}
- // One-time initialisation of state that is constant across scan ranges
- DCHECK(tuple_desc_->table_desc() != NULL);
- hdfs_table_ = static_cast<const HdfsTableDescriptor*>(tuple_desc_->table_desc());
- scan_node_pool_.reset(new MemPool(mem_tracker()));
-
- for (int i = 0; i < filter_ctxs_.size(); ++i) {
- RETURN_IF_ERROR(
- filter_ctxs_[i].expr->Prepare(state, row_desc(), expr_mem_tracker()));
- AddExprCtxToFree(filter_ctxs_[i].expr);
- }
-
- // Parse Avro table schema if applicable
- const string& avro_schema_str = hdfs_table_->avro_schema();
- if (!avro_schema_str.empty()) {
- avro_schema_t avro_schema;
- int error = avro_schema_from_json_length(
- avro_schema_str.c_str(), avro_schema_str.size(), &avro_schema);
- if (error != 0) {
- return Status(Substitute("Failed to parse table schema: $0", avro_strerror()));
- }
- RETURN_IF_ERROR(AvroSchemaElement::ConvertSchema(avro_schema, avro_schema_.get()));
- }
-
- // Gather materialized partition-key slots and non-partition slots.
- const vector<SlotDescriptor*>& slots = tuple_desc_->slots();
- for (size_t i = 0; i < slots.size(); ++i) {
- if (hdfs_table_->IsClusteringCol(slots[i])) {
- partition_key_slots_.push_back(slots[i]);
- } else {
- materialized_slots_.push_back(slots[i]);
- }
- }
-
- // Order the materialized slots such that for schemaless file formats (e.g. text) the
- // order corresponds to the physical order in files. For formats where the file schema
- // is independent of the table schema (e.g. Avro, Parquet), this step is not necessary.
- sort(materialized_slots_.begin(), materialized_slots_.end(),
- SlotDescriptor::ColPathLessThan);
-
- // Populate mapping from slot path to index into materialized_slots_.
- for (int i = 0; i < materialized_slots_.size(); ++i) {
- path_to_materialized_slot_idx_[materialized_slots_[i]->col_path()] = i;
- }
-
- // Initialize is_materialized_col_
- is_materialized_col_.resize(hdfs_table_->num_cols());
- for (int i = 0; i < hdfs_table_->num_cols(); ++i) {
- is_materialized_col_[i] = GetMaterializedSlotIdx(vector<int>(1, i)) != SKIP_COLUMN;
- }
-
- HdfsFsCache::HdfsFsMap fs_cache;
- // Convert the TScanRangeParams into per-file DiskIO::ScanRange objects and populate
- // partition_ids_, file_descs_, and per_type_files_.
- DCHECK(scan_range_params_ != NULL)
- << "Must call SetScanRanges() before calling Prepare()";
- int num_ranges_missing_volume_id = 0;
- for (int i = 0; i < scan_range_params_->size(); ++i) {
- DCHECK((*scan_range_params_)[i].scan_range.__isset.hdfs_file_split);
- const THdfsFileSplit& split = (*scan_range_params_)[i].scan_range.hdfs_file_split;
- partition_ids_.insert(split.partition_id);
- HdfsPartitionDescriptor* partition_desc =
- hdfs_table_->GetPartition(split.partition_id);
- if (partition_desc == NULL) {
- // TODO: this should be a DCHECK but we sometimes hit it. It's likely IMPALA-1702.
- LOG(ERROR) << "Bad table descriptor! table_id=" << hdfs_table_->id()
- << " partition_id=" << split.partition_id
- << "\n" << PrintThrift(state->fragment_params());
- return Status("Query encountered invalid metadata, likely due to IMPALA-1702."
- " Try rerunning the query.");
- }
-
- if (partition_template_tuple_map_.find(split.partition_id) ==
- partition_template_tuple_map_.end()) {
- partition_template_tuple_map_[split.partition_id] =
- InitTemplateTuple(state, partition_desc->partition_key_value_ctxs());;
- }
-
- filesystem::path file_path(partition_desc->location());
- file_path.append(split.file_name, filesystem::path::codecvt());
- const string& native_file_path = file_path.native();
-
- HdfsFileDesc* file_desc = NULL;
- FileDescMap::iterator file_desc_it = file_descs_.find(native_file_path);
- if (file_desc_it == file_descs_.end()) {
- // Add new file_desc to file_descs_ and per_type_files_
- file_desc = runtime_state_->obj_pool()->Add(new HdfsFileDesc(native_file_path));
- file_descs_[native_file_path] = file_desc;
- file_desc->file_length = split.file_length;
- file_desc->mtime = split.mtime;
- file_desc->file_compression = split.file_compression;
- RETURN_IF_ERROR(HdfsFsCache::instance()->GetConnection(
- native_file_path, &file_desc->fs, &fs_cache));
- num_unqueued_files_.Add(1);
- per_type_files_[partition_desc->file_format()].push_back(file_desc);
- } else {
- // File already processed
- file_desc = file_desc_it->second;
- }
-
- bool expected_local =
- (*scan_range_params_)[i].__isset.is_remote && !(*scan_range_params_)[i].is_remote;
- if (expected_local && (*scan_range_params_)[i].volume_id == -1) {
- if (!FLAGS_suppress_unknown_disk_id_warnings && !unknown_disk_id_warned_) {
- runtime_profile()->AppendExecOption("Missing Volume Id");
- runtime_state()->LogError(ErrorMsg(TErrorCode::HDFS_SCAN_NODE_UNKNOWN_DISK));
- unknown_disk_id_warned_ = true;
- }
- ++num_ranges_missing_volume_id;
- }
-
- bool try_cache = (*scan_range_params_)[i].is_cached;
- if (runtime_state_->query_options().disable_cached_reads) {
- DCHECK(!try_cache) << "Params should not have had this set.";
- }
- file_desc->splits.push_back(
- AllocateScanRange(file_desc->fs, file_desc->filename.c_str(), split.length,
- split.offset, split.partition_id, (*scan_range_params_)[i].volume_id,
- try_cache, expected_local, file_desc->mtime));
- }
-
// Compute the minimum bytes required to start a new thread. This is based on the
// file format.
// The higher the estimate, the less likely it is the query will fail but more likely
@@ -632,100 +189,17 @@ Status HdfsScanNode::Prepare(RuntimeState* state) {
}
}
scanner_thread_bytes_required_ += scanner_thread_mem_usage;
-
- // Prepare all the partitions scanned by the scan node
- for (int64_t partition_id: partition_ids_) {
- HdfsPartitionDescriptor* partition_desc = hdfs_table_->GetPartition(partition_id);
- // This is IMPALA-1702, but will have been caught earlier in this method.
- DCHECK(partition_desc != NULL) << "table_id=" << hdfs_table_->id()
- << " partition_id=" << partition_id
- << "\n" << PrintThrift(state->fragment_params());
- RETURN_IF_ERROR(partition_desc->PrepareExprs(state));
- }
-
- // Update server wide metrics for number of scan ranges and ranges that have
- // incomplete metadata.
- ImpaladMetrics::NUM_RANGES_PROCESSED->Increment(scan_range_params_->size());
- ImpaladMetrics::NUM_RANGES_MISSING_VOLUME_ID->Increment(num_ranges_missing_volume_id);
-
- // Add per volume stats to the runtime profile
- PerVolumnStats per_volume_stats;
- stringstream str;
- UpdateHdfsSplitStats(*scan_range_params_, &per_volume_stats);
- PrintHdfsSplitStats(per_volume_stats, &str);
- runtime_profile()->AddInfoString(HDFS_SPLIT_STATS_DESC, str.str());
-
- // Create codegen'd functions
- for (int format = THdfsFileFormat::TEXT;
- format <= THdfsFileFormat::PARQUET; ++format) {
- vector<HdfsFileDesc*>& file_descs =
- per_type_files_[static_cast<THdfsFileFormat::type>(format)];
-
- if (file_descs.empty()) continue;
-
- // Randomize the order this node processes the files. We want to do this to avoid
- // issuing remote reads to the same DN from different impalads. In file formats such
- // as avro/seq/rc (i.e. splittable with a header), every node first reads the header.
- // If every node goes through the files in the same order, all the remote reads are
- // for the same file meaning a few DN serves a lot of remote reads at the same time.
- random_shuffle(file_descs.begin(), file_descs.end());
-
- // Create reusable codegen'd functions for each file type type needed
- // TODO: do this for conjuncts_map_
- Function* fn;
- Status status;
- switch (format) {
- case THdfsFileFormat::TEXT:
- status = HdfsTextScanner::Codegen(this, conjunct_ctxs_, &fn);
- break;
- case THdfsFileFormat::SEQUENCE_FILE:
- status = HdfsSequenceScanner::Codegen(this, conjunct_ctxs_, &fn);
- break;
- case THdfsFileFormat::AVRO:
- status = HdfsAvroScanner::Codegen(this, conjunct_ctxs_, &fn);
- break;
- case THdfsFileFormat::PARQUET:
- status = HdfsParquetScanner::Codegen(this, conjunct_ctxs_, &fn);
- break;
- default:
- // No codegen for this format
- fn = NULL;
- status = Status("Not implemented for this format.");
- }
- DCHECK(fn != NULL || !status.ok());
-
- const char* format_name = _THdfsFileFormat_VALUES_TO_NAMES.find(format)->second;
- if (!status.ok()) {
- runtime_profile()->AddCodegenMsg(false, status, format_name);
- } else {
- runtime_profile()->AddCodegenMsg(true, status, format_name);
- LlvmCodeGen* codegen;
- RETURN_IF_ERROR(runtime_state_->GetCodegen(&codegen));
- codegen->AddFunctionToJit(
- fn, &codegend_fn_map_[static_cast<THdfsFileFormat::type>(format)]);
- }
- }
-
return Status::OK();
}
-// This function initiates the connection to hdfs and starts up the initial scanner
-// threads. The scanner subclasses are passed the initial splits. Scanners are expected to
-// queue up a non-zero number of those splits to the io mgr (via the ScanNode). Scan
-// ranges are not issued until the first GetNext() call; scanner threads will block on
-// ranges_issued_barrier_ until ranges are issued.
+// This function registers the ThreadTokenAvailableCb to start up the initial scanner
+// threads. Scan ranges are not issued until the first GetNext() call; scanner threads
+// will block on ranges_issued_barrier_ until ranges are issued.
Status HdfsScanNode::Open(RuntimeState* state) {
- RETURN_IF_ERROR(ExecNode::Open(state));
-
- // Open collection conjuncts
- ConjunctsMap::const_iterator iter = conjuncts_map_.begin();
- for (; iter != conjuncts_map_.end(); ++iter) {
- // conjuncts_ are already opened in ExecNode::Open()
- if (iter->first == tuple_id_) continue;
- RETURN_IF_ERROR(Expr::Open(iter->second, state));
- }
+ SCOPED_TIMER(runtime_profile_->total_time_counter());
+ RETURN_IF_ERROR(HdfsScanNodeBase::Open(state));
- for (auto& filter_ctx: filter_ctxs_) RETURN_IF_ERROR(filter_ctx.expr->Open(state));
+ if (file_descs_.empty() || progress_.done()) return Status::OK();
// We need at least one scanner thread to make progress. We need to make this
// reservation before any ranges are issued.
@@ -744,95 +218,9 @@ Status HdfsScanNode::Open(RuntimeState* state) {
runtime_state_->resource_pool()));
}
- if (file_descs_.empty()) {
- SetDone();
- return Status::OK();
- }
-
- // Open all the partition exprs used by the scan node
- for (int64_t partition_id: partition_ids_) {
- HdfsPartitionDescriptor* partition_desc = hdfs_table_->GetPartition(partition_id);
- DCHECK(partition_desc != NULL) << "table_id=" << hdfs_table_->id()
- << " partition_id=" << partition_id
- << "\n" << PrintThrift(state->fragment_params());
- RETURN_IF_ERROR(partition_desc->OpenExprs(state));
- }
-
- RETURN_IF_ERROR(runtime_state_->io_mgr()->RegisterContext(
- &reader_context_, mem_tracker()));
-
- // Initialize HdfsScanNode specific counters
- read_timer_ = ADD_TIMER(runtime_profile(), TOTAL_HDFS_READ_TIMER);
- per_read_thread_throughput_counter_ = runtime_profile()->AddDerivedCounter(
- PER_READ_THREAD_THROUGHPUT_COUNTER, TUnit::BYTES_PER_SECOND,
- bind<int64_t>(&RuntimeProfile::UnitsPerSecond, bytes_read_counter_, read_timer_));
- scan_ranges_complete_counter_ =
- ADD_COUNTER(runtime_profile(), SCAN_RANGES_COMPLETE_COUNTER, TUnit::UNIT);
- if (DiskInfo::num_disks() < 64) {
- num_disks_accessed_counter_ =
- ADD_COUNTER(runtime_profile(), NUM_DISKS_ACCESSED_COUNTER, TUnit::UNIT);
- } else {
- num_disks_accessed_counter_ = NULL;
- }
- num_scanner_threads_started_counter_ =
- ADD_COUNTER(runtime_profile(), NUM_SCANNER_THREADS_STARTED, TUnit::UNIT);
-
- runtime_state_->io_mgr()->set_bytes_read_counter(reader_context_, bytes_read_counter());
- runtime_state_->io_mgr()->set_read_timer(reader_context_, read_timer());
- runtime_state_->io_mgr()->set_active_read_thread_counter(reader_context_,
- &active_hdfs_read_thread_counter_);
- runtime_state_->io_mgr()->set_disks_access_bitmap(reader_context_,
- &disks_accessed_bitmap_);
-
- average_scanner_thread_concurrency_ = runtime_profile()->AddSamplingCounter(
- AVERAGE_SCANNER_THREAD_CONCURRENCY, &active_scanner_thread_counter_);
- average_hdfs_read_thread_concurrency_ = runtime_profile()->AddSamplingCounter(
- AVERAGE_HDFS_READ_THREAD_CONCURRENCY, &active_hdfs_read_thread_counter_);
-
- bytes_read_local_ = ADD_COUNTER(runtime_profile(), "BytesReadLocal",
- TUnit::BYTES);
- bytes_read_short_circuit_ = ADD_COUNTER(runtime_profile(), "BytesReadShortCircuit",
- TUnit::BYTES);
- bytes_read_dn_cache_ = ADD_COUNTER(runtime_profile(), "BytesReadDataNodeCache",
- TUnit::BYTES);
- num_remote_ranges_ = ADD_COUNTER(runtime_profile(), "RemoteScanRanges",
- TUnit::UNIT);
- unexpected_remote_bytes_ = ADD_COUNTER(runtime_profile(), "BytesReadRemoteUnexpected",
- TUnit::BYTES);
-
- max_compressed_text_file_length_ = runtime_profile()->AddHighWaterMarkCounter(
- "MaxCompressedTextFileLength", TUnit::BYTES);
-
- for (int i = 0; i < state->io_mgr()->num_total_disks() + 1; ++i) {
- hdfs_read_thread_concurrency_bucket_.push_back(
- pool_->Add(new RuntimeProfile::Counter(TUnit::DOUBLE_VALUE, 0)));
- }
- runtime_profile()->RegisterBucketingCounters(&active_hdfs_read_thread_counter_,
- &hdfs_read_thread_concurrency_bucket_);
-
- counters_running_ = true;
-
- int total_splits = 0;
- for (FileDescMap::iterator it = file_descs_.begin(); it != file_descs_.end(); ++it) {
- total_splits += it->second->splits.size();
- }
-
- if (total_splits == 0) {
- SetDone();
- return Status::OK();
- }
-
- stringstream ss;
- ss << "Splits complete (node=" << id() << "):";
- progress_.Init(ss.str(), total_splits);
return Status::OK();
}
-Status HdfsScanNode::Reset(RuntimeState* state) {
- DCHECK(false) << "Internal error: Scan nodes should not appear in subplans.";
- return Status("Internal error: Scan nodes should not appear in subplans.");
-}
-
void HdfsScanNode::Close(RuntimeState* state) {
if (is_closed()) return;
SetDone();
@@ -849,47 +237,36 @@ void HdfsScanNode::Close(RuntimeState* state) {
num_owned_io_buffers_.Add(-materialized_row_batches_->Cleanup());
DCHECK_EQ(num_owned_io_buffers_.Load(), 0) << "ScanNode has leaked io buffers";
- if (reader_context_ != NULL) {
- // There may still be io buffers used by parent nodes so we can't unregister the
- // reader context yet. The runtime state keeps a list of all the reader contexts and
- // they are unregistered when the fragment is closed.
- state->reader_contexts()->push_back(reader_context_);
- // Need to wait for all the active scanner threads to finish to ensure there is no
- // more memory tracked by this scan node's mem tracker.
- state->io_mgr()->CancelContext(reader_context_, true);
- }
-
- StopAndFinalizeCounters();
+ HdfsScanNodeBase::Close(state);
+}
- // There should be no active scanner threads and hdfs read threads.
- DCHECK_EQ(active_scanner_thread_counter_.value(), 0);
- DCHECK_EQ(active_hdfs_read_thread_counter_.value(), 0);
+void HdfsScanNode::SetFileMetadata(const string& filename, void* metadata) {
+ unique_lock<mutex> l(metadata_lock_);
+ DCHECK(per_file_metadata_.find(filename) == per_file_metadata_.end());
+ per_file_metadata_[filename] = metadata;
+}
- if (scan_node_pool_.get() != NULL) scan_node_pool_->FreeAll();
+void* HdfsScanNode::GetFileMetadata(const string& filename) {
+ unique_lock<mutex> l(metadata_lock_);
+ map<string, void*>::iterator it = per_file_metadata_.find(filename);
+ if (it == per_file_metadata_.end()) return NULL;
+ return it->second;
+}
- // Close all the partitions scanned by the scan node
- for (int64_t partition_id: partition_ids_) {
- HdfsPartitionDescriptor* partition_desc = hdfs_table_->GetPartition(partition_id);
- if (partition_desc == NULL) {
- // TODO: Revert when IMPALA-1702 is fixed.
- LOG(ERROR) << "Bad table descriptor! table_id=" << hdfs_table_->id()
- << " partition_id=" << partition_id
- << "\n" << PrintThrift(state->fragment_params());
- continue;
- }
- partition_desc->CloseExprs(state);
- }
+void HdfsScanNode::RangeComplete(const THdfsFileFormat::type& file_type,
+ const std::vector<THdfsCompression::type>& compression_type) {
+ lock_guard<SpinLock> l(file_type_counts_);
+ HdfsScanNodeBase::RangeComplete(file_type, compression_type);
+}
- // Close collection conjuncts
- ConjunctsMap::const_iterator iter = conjuncts_map_.begin();
- for (; iter != conjuncts_map_.end(); ++iter) {
- // conjuncts_ are already closed in ExecNode::Close()
- if (iter->first == tuple_id_) continue;
- Expr::Close(iter->second, state);
- }
+void HdfsScanNode::TransferToScanNodePool(MemPool* pool) {
+ unique_lock<mutex> l(lock_);
+ scan_node_pool_->AcquireData(pool, false);
+}
- for (auto& filter_ctx: filter_ctxs_) filter_ctx.expr->Close(state);
- ScanNode::Close(state);
+void HdfsScanNode::AddMaterializedRowBatch(RowBatch* row_batch) {
+ InitNullCollectionValues(row_batch);
+ materialized_row_batches_->AddBatch(row_batch);
}
Status HdfsScanNode::AddDiskIoRanges(const vector<DiskIoMgr::ScanRange*>& ranges,
@@ -902,43 +279,6 @@ Status HdfsScanNode::AddDiskIoRanges(const vector<DiskIoMgr::ScanRange*>& ranges
return Status::OK();
}
-void HdfsScanNode::AddMaterializedRowBatch(RowBatch* row_batch) {
- InitNullCollectionValues(row_batch);
- materialized_row_batches_->AddBatch(row_batch);
-}
-
-void HdfsScanNode::InitNullCollectionValues(const TupleDescriptor* tuple_desc,
- Tuple* tuple) const {
- for (const SlotDescriptor* slot_desc: tuple_desc->collection_slots()) {
- CollectionValue* slot = reinterpret_cast<CollectionValue*>(
- tuple->GetSlot(slot_desc->tuple_offset()));
- if (tuple->IsNull(slot_desc->null_indicator_offset())) {
- *slot = CollectionValue();
- continue;
- }
- // Recursively traverse collection items.
- const TupleDescriptor* item_desc = slot_desc->collection_item_descriptor();
- if (item_desc->collection_slots().empty()) continue;
- for (int i = 0; i < slot->num_tuples; ++i) {
- int item_offset = i * item_desc->byte_size();
- Tuple* collection_item = reinterpret_cast<Tuple*>(slot->ptr + item_offset);
- InitNullCollectionValues(item_desc, collection_item);
- }
- }
-}
-
-void HdfsScanNode::InitNullCollectionValues(RowBatch* row_batch) const {
- DCHECK_EQ(row_batch->row_desc().tuple_descriptors().size(), 1);
- const TupleDescriptor& tuple_desc =
- *row_batch->row_desc().tuple_descriptors()[tuple_idx()];
- if (tuple_desc.collection_slots().empty()) return;
- for (int i = 0; i < row_batch->num_rows(); ++i) {
- Tuple* tuple = row_batch->GetRow(i)->GetTuple(tuple_idx());
- DCHECK(tuple != NULL);
- InitNullCollectionValues(&tuple_desc, tuple);
- }
-}
-
// For controlling the amount of memory used for scanners, we approximate the
// scanner mem usage based on scanner_thread_bytes_required_, rather than the
// consumption in the scan node's mem tracker. The problem with the scan node
@@ -1161,33 +501,6 @@ void HdfsScanNode::ScannerThread() {
runtime_state_->resource_pool()->ReleaseThreadToken(false);
}
-bool HdfsScanNode::PartitionPassesFilters(int32_t partition_id,
- const string& stats_name, const vector<FilterContext>& filter_ctxs) {
- if (filter_ctxs.size() == 0) return true;
- DCHECK_EQ(filter_ctxs.size(), filter_ctxs_.size())
- << "Mismatched number of filter contexts";
- Tuple* template_tuple = partition_template_tuple_map_[partition_id];
- // Defensive - if template_tuple is NULL, there can be no filters on partition columns.
- if (template_tuple == NULL) return true;
- TupleRow* tuple_row_mem = reinterpret_cast<TupleRow*>(&template_tuple);
- for (const FilterContext& ctx: filter_ctxs) {
- int target_ndx = ctx.filter->filter_desc().planid_to_target_ndx.at(id_);
- if (!ctx.filter->filter_desc().targets[target_ndx].is_bound_by_partition_columns) {
- continue;
- }
- void* e = ctx.expr->GetValue(tuple_row_mem);
-
- // Not quite right because bitmap could arrive after Eval(), but we're ok with
- // off-by-one errors.
- bool processed = ctx.filter->HasBloomFilter();
- bool passed_filter = ctx.filter->Eval<void>(e, ctx.expr->root()->type());
- ctx.stats->IncrCounters(stats_name, 1, processed, !passed_filter);
- if (!passed_filter) return false;
- }
-
- return true;
-}
-
namespace {
// Returns true if 'format' uses a scanner derived from BaseSequenceScanner. Used to
@@ -1205,8 +518,7 @@ Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
DCHECK(scan_range != NULL);
- ScanRangeMetadata* metadata =
- reinterpret_cast<ScanRangeMetadata*>(scan_range->meta_data());
+ ScanRangeMetadata* metadata = static_cast<ScanRangeMetadata*>(scan_range->meta_data());
int64_t partition_id = metadata->partition_id;
HdfsPartitionDescriptor* partition = hdfs_table_->GetPartition(partition_id);
DCHECK(partition != NULL) << "table_id=" << hdfs_table_->id()
@@ -1269,26 +581,6 @@ Status HdfsScanNode::ProcessSplit(const vector<FilterContext>& filter_ctxs,
return status;
}
-void HdfsScanNode::RangeComplete(const THdfsFileFormat::type& file_type,
- const THdfsCompression::type& compression_type) {
- vector<THdfsCompression::type> types;
- types.push_back(compression_type);
- RangeComplete(file_type, types);
-}
-
-void HdfsScanNode::RangeComplete(const THdfsFileFormat::type& file_type,
- const vector<THdfsCompression::type>& compression_types) {
- scan_ranges_complete_counter()->Add(1);
- progress_.Update(1);
-
- {
- lock_guard<SpinLock> l(file_type_counts_lock_);
- for (int i = 0; i < compression_types.size(); ++i) {
- ++file_type_counts_[make_pair(file_type, compression_types[i])];
- }
- }
-}
-
void HdfsScanNode::SetDone() {
{
unique_lock<mutex> l(lock_);
@@ -1300,129 +592,3 @@ void HdfsScanNode::SetDone() {
}
materialized_row_batches_->Shutdown();
}
-
-void HdfsScanNode::ComputeSlotMaterializationOrder(vector<int>* order) const {
- const vector<ExprContext*>& conjuncts = ExecNode::conjunct_ctxs();
- // Initialize all order to be conjuncts.size() (after the last conjunct)
- order->insert(order->begin(), materialized_slots().size(), conjuncts.size());
-
- const DescriptorTbl& desc_tbl = runtime_state_->desc_tbl();
-
- vector<SlotId> slot_ids;
- for (int conjunct_idx = 0; conjunct_idx < conjuncts.size(); ++conjunct_idx) {
- slot_ids.clear();
- int num_slots = conjuncts[conjunct_idx]->root()->GetSlotIds(&slot_ids);
- for (int j = 0; j < num_slots; ++j) {
- SlotDescriptor* slot_desc = desc_tbl.GetSlotDescriptor(slot_ids[j]);
- int slot_idx = GetMaterializedSlotIdx(slot_desc->col_path());
- // slot_idx == -1 means this was a partition key slot which is always
- // materialized before any slots.
- if (slot_idx == -1) continue;
- // If this slot hasn't been assigned an order, assign it be materialized
- // before evaluating conjuncts[i]
- if ((*order)[slot_idx] == conjuncts.size()) {
- (*order)[slot_idx] = conjunct_idx;
- }
- }
- }
-}
-
-void HdfsScanNode::StopAndFinalizeCounters() {
- unique_lock<mutex> l(lock_);
- if (!counters_running_) return;
- counters_running_ = false;
-
- PeriodicCounterUpdater::StopTimeSeriesCounter(bytes_read_timeseries_counter_);
- PeriodicCounterUpdater::StopRateCounter(total_throughput_counter());
- PeriodicCounterUpdater::StopSamplingCounter(average_scanner_thread_concurrency_);
- PeriodicCounterUpdater::StopSamplingCounter(average_hdfs_read_thread_concurrency_);
- PeriodicCounterUpdater::StopBucketingCounters(&hdfs_read_thread_concurrency_bucket_,
- true);
-
- // Output hdfs read thread concurrency into info string
- stringstream ss;
- for (int i = 0; i < hdfs_read_thread_concurrency_bucket_.size(); ++i) {
- ss << i << ":" << setprecision(4)
- << hdfs_read_thread_concurrency_bucket_[i]->double_value() << "% ";
- }
- runtime_profile_->AddInfoString("Hdfs Read Thread Concurrency Bucket", ss.str());
-
- // Convert disk access bitmap to num of disk accessed
- uint64_t num_disk_bitmap = disks_accessed_bitmap_.value();
- int64_t num_disk_accessed = BitUtil::Popcount(num_disk_bitmap);
- if (num_disks_accessed_counter_ != NULL) {
- num_disks_accessed_counter_->Set(num_disk_accessed);
- }
-
- // output completed file types and counts to info string
- if (!file_type_counts_.empty()) {
- stringstream ss;
- {
- lock_guard<SpinLock> l2(file_type_counts_lock_);
- for (FileTypeCountsMap::const_iterator it = file_type_counts_.begin();
- it != file_type_counts_.end(); ++it) {
- ss << it->first.first << "/" << it->first.second << ":" << it->second << " ";
- }
- }
- runtime_profile_->AddInfoString("File Formats", ss.str());
- }
-
- // Output fraction of scanners with codegen enabled
- int num_enabled = num_scanners_codegen_enabled_.Load();
- int total = num_enabled + num_scanners_codegen_disabled_.Load();
- runtime_profile()->AppendExecOption(
- Substitute("Codegen enabled: $0 out of $1", num_enabled, total));
-
- if (reader_context_ != NULL) {
- bytes_read_local_->Set(runtime_state_->io_mgr()->bytes_read_local(reader_context_));
- bytes_read_short_circuit_->Set(
- runtime_state_->io_mgr()->bytes_read_short_circuit(reader_context_));
- bytes_read_dn_cache_->Set(
- runtime_state_->io_mgr()->bytes_read_dn_cache(reader_context_));
- num_remote_ranges_->Set(static_cast<int64_t>(
- runtime_state_->io_mgr()->num_remote_ranges(reader_context_)));
- unexpected_remote_bytes_->Set(
- runtime_state_->io_mgr()->unexpected_remote_bytes(reader_context_));
-
- if (unexpected_remote_bytes_->value() >= UNEXPECTED_REMOTE_BYTES_WARN_THRESHOLD) {
- runtime_state_->LogError(ErrorMsg(TErrorCode::GENERAL, Substitute(
- "Read $0 of data across network that was expected to be local. "
- "Block locality metadata for table '$1.$2' may be stale. Consider running "
- "\"INVALIDATE METADATA `$1`.`$2`\".",
- PrettyPrinter::Print(unexpected_remote_bytes_->value(), TUnit::BYTES),
- hdfs_table_->database(), hdfs_table_->name())));
- }
-
- ImpaladMetrics::IO_MGR_BYTES_READ->Increment(bytes_read_counter()->value());
- ImpaladMetrics::IO_MGR_LOCAL_BYTES_READ->Increment(
- bytes_read_local_->value());
- ImpaladMetrics::IO_MGR_SHORT_CIRCUIT_BYTES_READ->Increment(
- bytes_read_short_circuit_->value());
- ImpaladMetrics::IO_MGR_CACHED_BYTES_READ->Increment(
- bytes_read_dn_cache_->value());
- }
-}
-
-void HdfsScanNode::UpdateHdfsSplitStats(
- const vector<TScanRangeParams>& scan_range_params_list,
- PerVolumnStats* per_volume_stats) {
- pair<int, int64_t> init_value(0, 0);
- for (const TScanRangeParams& scan_range_params: scan_range_params_list) {
- const TScanRange& scan_range = scan_range_params.scan_range;
- if (!scan_range.__isset.hdfs_file_split) continue;
- const THdfsFileSplit& split = scan_range.hdfs_file_split;
- pair<int, int64_t>* stats =
- FindOrInsert(per_volume_stats, scan_range_params.volume_id, init_value);
- ++(stats->first);
- stats->second += split.length;
- }
-}
-
-void HdfsScanNode::PrintHdfsSplitStats(const PerVolumnStats& per_volume_stats,
- stringstream* ss) {
- for (PerVolumnStats::const_iterator i = per_volume_stats.begin();
- i != per_volume_stats.end(); ++i) {
- (*ss) << i->first << ":" << i->second.first << "/"
- << PrettyPrinter::Print(i->second.second, TUnit::BYTES) << " ";
- }
-}
[3/7] incubator-impala git commit: IMPALA-3491: Use unique database
fixture in test_insert_parquet.py
Posted by ab...@apache.org.
IMPALA-3491: Use unique database fixture in test_insert_parquet.py
Testing: Ran the test locally in a loop.
Did a private debug/core/hdfs build.
Change-Id: I790b2ed5236640c7263826d1d2a74b64d43ac6f7
Reviewed-on: http://gerrit.cloudera.org:8080/4317
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f0ffbca2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f0ffbca2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f0ffbca2
Branch: refs/heads/master
Commit: f0ffbca2c36ce3ad8ea5b3629d91e6b579d7dfcf
Parents: 157c800
Author: Alex Behm <al...@cloudera.com>
Authored: Fri Sep 2 10:44:20 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Sep 8 03:25:29 2016 +0000
----------------------------------------------------------------------
.../workloads/tpch/queries/insert_parquet.test | 6 +-
tests/query_test/test_insert_parquet.py | 60 ++++++++------------
2 files changed, 26 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f0ffbca2/testdata/workloads/tpch/queries/insert_parquet.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/tpch/queries/insert_parquet.test b/testdata/workloads/tpch/queries/insert_parquet.test
index 35b7cde..4707b7b 100644
--- a/testdata/workloads/tpch/queries/insert_parquet.test
+++ b/testdata/workloads/tpch/queries/insert_parquet.test
@@ -1,8 +1,8 @@
====
---- QUERY
# Tests using a larger table.
-create table if not exists orders_insert_test like orders location
-'$FILESYSTEM_PREFIX/test-warehouse/orders_insert_table';
+create table if not exists orders_insert_test like tpch_parquet.orders
+location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/orders_insert_table';
insert overwrite table orders_insert_test select * from tpch.orders
---- RESULTS
: 1500000
@@ -56,7 +56,7 @@ bigint
---- QUERY
# Test to verify that huge (larger than 64k) values can be written, see IMPALA-1705
create table if not exists test_insert_huge_vals (s string) stored as parquet
-location '$FILESYSTEM_PREFIX/test-warehouse/test_insert_huge_vals';
+location '$FILESYSTEM_PREFIX/test-warehouse/$DATABASE.db/test_insert_huge_vals';
insert overwrite table test_insert_huge_vals
select cast(l_orderkey as string) from tpch.lineitem
union select group_concat(concat(s_name, s_address, s_phone)) from tpch.supplier
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f0ffbca2/tests/query_test/test_insert_parquet.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert_parquet.py b/tests/query_test/test_insert_parquet.py
index 70d3545..d5b8c17 100644
--- a/tests/query_test/test_insert_parquet.py
+++ b/tests/query_test/test_insert_parquet.py
@@ -26,6 +26,7 @@ from tempfile import mkdtemp as make_tmp_dir
from tests.common.environ import impalad_basedir
from tests.common.impala_test_suite import ImpalaTestSuite
+from tests.common.parametrize import UniqueDatabase
from tests.common.skip import SkipIfIsilon, SkipIfLocal
from tests.common.test_dimensions import create_exec_option_dimension
from tests.common.test_vector import TestDimension
@@ -63,18 +64,14 @@ class TestInsertParquetQueries(ImpalaTestSuite):
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
- @classmethod
- def setup_class(cls):
- super(TestInsertParquetQueries, cls).setup_class()
-
- @pytest.mark.execute_serially
@SkipIfLocal.multiple_impalad
- def test_insert_parquet(self, vector):
+ @UniqueDatabase.parametrize(sync_ddl=True)
+ def test_insert_parquet(self, vector, unique_database):
vector.get_value('exec_option')['PARQUET_FILE_SIZE'] = \
vector.get_value('file_size')
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
vector.get_value('compression_codec')
- self.run_test_case('insert_parquet', vector, multiple_impalad=True)
+ self.run_test_case('insert_parquet', vector, unique_database, multiple_impalad=True)
class TestInsertParquetInvalidCodec(ImpalaTestSuite):
@classmethod
@@ -94,10 +91,6 @@ class TestInsertParquetInvalidCodec(ImpalaTestSuite):
cls.TestMatrix.add_constraint(lambda v:\
v.get_value('table_format').compression_codec == 'none')
- @classmethod
- def setup_class(cls):
- super(TestInsertParquetInvalidCodec, cls).setup_class()
-
@SkipIfLocal.multiple_impalad
def test_insert_parquet_invalid_codec(self, vector):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
@@ -124,40 +117,34 @@ class TestInsertParquetVerifySize(ImpalaTestSuite):
v.get_value('table_format').compression_codec == 'none')
cls.TestMatrix.add_dimension(TestDimension("compression_codec", *PARQUET_CODECS));
- @classmethod
- def setup_class(cls):
- super(TestInsertParquetVerifySize, cls).setup_class()
-
- @pytest.mark.execute_serially
@SkipIfIsilon.hdfs_block_size
@SkipIfLocal.hdfs_client
- def test_insert_parquet_verify_size(self, vector):
- # Test to verify that the result file size is close to what we expect.i
- TBL = "parquet_insert_size"
- DROP = "drop table if exists {0}".format(TBL)
- CREATE = ("create table parquet_insert_size like tpch_parquet.orders"
- " stored as parquet location '{0}/{1}'".format(WAREHOUSE, TBL))
- QUERY = "insert overwrite {0} select * from tpch.orders".format(TBL)
- DIR = get_fs_path("test-warehouse/{0}/".format(TBL))
- BLOCK_SIZE = 40 * 1024 * 1024
-
- self.execute_query(DROP)
- self.execute_query(CREATE)
-
- vector.get_value('exec_option')['PARQUET_FILE_SIZE'] = BLOCK_SIZE
+ def test_insert_parquet_verify_size(self, vector, unique_database):
+ # Test to verify that the result file size is close to what we expect.
+ tbl_name = "parquet_insert_size"
+ fq_tbl_name = unique_database + "." + tbl_name
+ location = get_fs_path("test-warehouse/{0}.db/{1}/"
+ .format(unique_database, tbl_name))
+ create = ("create table {0} like tpch_parquet.orders stored as parquet"
+ .format(fq_tbl_name, location))
+ query = "insert overwrite {0} select * from tpch.orders".format(fq_tbl_name)
+ block_size = 40 * 1024 * 1024
+
+ self.execute_query(create)
+ vector.get_value('exec_option')['PARQUET_FILE_SIZE'] = block_size
vector.get_value('exec_option')['COMPRESSION_CODEC'] =\
vector.get_value('compression_codec')
vector.get_value('exec_option')['num_nodes'] = 1
- self.execute_query(QUERY, vector.get_value('exec_option'))
+ self.execute_query(query, vector.get_value('exec_option'))
# Get the files in hdfs and verify. There can be at most 1 file that is smaller
- # that the BLOCK_SIZE. The rest should be within 80% of it and not over.
+ # that the block_size. The rest should be within 80% of it and not over.
found_small_file = False
- sizes = self.filesystem_client.get_all_file_sizes(DIR)
+ sizes = self.filesystem_client.get_all_file_sizes(location)
for size in sizes:
- assert size < BLOCK_SIZE, "File size greater than expected.\
- Expected: {0}, Got: {1}".format(BLOCK_SIZE, size)
- if size < BLOCK_SIZE * 0.80:
+ assert size < block_size, "File size greater than expected.\
+ Expected: {0}, Got: {1}".format(block_size, size)
+ if size < block_size * 0.80:
assert found_small_file == False
found_small_file = True
@@ -179,7 +166,6 @@ class TestHdfsParquetTableWriter(ImpalaTestSuite):
"""
table_name = "test_hdfs_parquet_table_writer"
qualified_table_name = "%s.%s" % (unique_database, table_name)
- self.execute_query("drop table if exists %s" % qualified_table_name)
self.execute_query("create table %s stored as parquet as select l_linenumber from "
"tpch_parquet.lineitem limit 180000" % qualified_table_name)
[5/7] incubator-impala git commit: IMPALA-3905: Add single-threaded
scan node.
Posted by ab...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scan-node.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scan-node.h b/be/src/exec/hdfs-scan-node.h
index 36e95b5..2efb24e 100644
--- a/be/src/exec/hdfs-scan-node.h
+++ b/be/src/exec/hdfs-scan-node.h
@@ -19,83 +19,29 @@
#ifndef IMPALA_EXEC_HDFS_SCAN_NODE_H_
#define IMPALA_EXEC_HDFS_SCAN_NODE_H_
-#include <vector>
-#include <memory>
+#include <map>
#include <stdint.h>
+#include <vector>
-#include <boost/unordered_map.hpp>
-#include <boost/unordered_set.hpp>
#include <boost/scoped_ptr.hpp>
-#include <boost/thread/condition_variable.hpp>
#include <boost/thread/mutex.hpp>
#include "exec/filter-context.h"
-#include "exec/scan-node.h"
-#include "exec/scanner-context.h"
-#include "runtime/descriptors.h"
+#include "exec/hdfs-scan-node-base.h"
#include "runtime/disk-io-mgr.h"
-#include "util/avro-util.h"
#include "util/counting-barrier.h"
-#include "util/progress-updater.h"
-#include "util/spinlock.h"
#include "util/thread.h"
-#include "gen-cpp/PlanNodes_types.h"
-
namespace impala {
class DescriptorTbl;
-class HdfsScanner;
+class ObjectPool;
+class RuntimeState;
class RowBatch;
-class RuntimeFilter;
-class Status;
-class Tuple;
class TPlanNode;
-class TScanRange;
-
-/// Maintains per file information for files assigned to this scan node. This includes
-/// all the splits for the file. Note that it is not thread-safe.
-struct HdfsFileDesc {
- /// Connection to the filesystem containing the file.
- hdfsFS fs;
-
- /// File name including the path.
- std::string filename;
-
- /// Length of the file. This is not related to which parts of the file have been
- /// assigned to this node.
- int64_t file_length;
-
- /// Last modified time
- int64_t mtime;
-
- THdfsCompression::type file_compression;
-
- /// Splits (i.e. raw byte ranges) for this file, assigned to this scan node.
- std::vector<DiskIoMgr::ScanRange*> splits;
- HdfsFileDesc(const std::string& filename)
- : filename(filename), file_length(0), mtime(0),
- file_compression(THdfsCompression::NONE) {
- }
-};
-
-/// Struct for additional metadata for scan ranges. This contains the partition id
-/// that this scan range is for.
-struct ScanRangeMetadata {
- /// The partition id that this range is part of.
- int64_t partition_id;
-
- /// For parquet scan ranges we initially create a request for the file footer for each
- /// split; we store a pointer to the actual split so that we can recover its information
- /// for the scanner to process.
- const DiskIoMgr::ScanRange* original_split;
-
- ScanRangeMetadata(int64_t partition_id, const DiskIoMgr::ScanRange* original_split)
- : partition_id(partition_id), original_split(original_split) { }
-};
-/// A ScanNode implementation that is used for all tables read directly from
-/// HDFS-serialised data.
+/// Legacy ScanNode implementation used in the non-multi-threaded execution mode
+/// that is used for all tables read directly from HDFS-serialised data.
/// A HdfsScanNode spawns multiple scanner threads to process the bytes in
/// parallel. There is a handshake between the scan node and the scanners
/// to get all the splits queued and bytes processed.
@@ -111,147 +57,36 @@ struct ScanRangeMetadata {
/// 5. The scanner finishes the scan range and informs the scan node so it can track
/// end of stream.
///
-/// An HdfsScanNode may expect to receive runtime filters produced elsewhere in the plan
-/// (even from remote fragments). These filters arrive asynchronously during execution,
-/// and are applied as soon as they arrive. Filters may be applied by the scan node in the
-/// following scopes:
-///
-/// 1. Per-file (all file formats, partition column filters only) - filtering at this
-/// scope saves IO as the filters are applied before scan ranges are issued.
-/// 2. Per-scan-range (all file formats, partition column filters only) - filtering at
-/// this scope saves CPU as filtered scan ranges are never scanned.
-///
-/// Scanners may also use the same filters to eliminate rows at finer granularities
-/// (e.g. per row).
-///
-/// TODO: this class allocates a bunch of small utility objects that should be
+/// TODO: This class allocates a bunch of small utility objects that should be
/// recycled.
-class HdfsScanNode : public ScanNode {
+/// TODO: Remove this class once the fragment-based multi-threaded execution is
+/// fully functional.
+class HdfsScanNode : public HdfsScanNodeBase {
public:
HdfsScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs);
-
~HdfsScanNode();
- /// ExecNode methods
- virtual Status Init(const TPlanNode& tnode, RuntimeState* state);
virtual Status Prepare(RuntimeState* state);
virtual Status Open(RuntimeState* state);
virtual Status GetNext(RuntimeState* state, RowBatch* row_batch, bool* eos);
- virtual Status Reset(RuntimeState* state);
virtual void Close(RuntimeState* state);
- int limit() const { return limit_; }
-
- const std::vector<SlotDescriptor*>& materialized_slots()
- const { return materialized_slots_; }
-
- /// Returns the tuple idx into the row for this scan node to output to.
- /// Currently this is always 0.
- int tuple_idx() const { return 0; }
-
- /// Returns number of partition keys in the table.
- int num_partition_keys() const { return hdfs_table_->num_clustering_cols(); }
-
- /// Returns number of partition key slots.
- int num_materialized_partition_keys() const { return partition_key_slots_.size(); }
-
- const TupleDescriptor* tuple_desc() const { return tuple_desc_; }
-
- const HdfsTableDescriptor* hdfs_table() { return hdfs_table_; }
-
- const AvroSchemaElement& avro_schema() { return *avro_schema_.get(); }
-
- RuntimeState* runtime_state() { return runtime_state_; }
+ virtual bool HasRowBatchQueue() const { return true; }
- int skip_header_line_count() const { return skip_header_line_count_; }
+ bool done() const { return done_; }
- DiskIoRequestContext* reader_context() { return reader_context_; }
-
- typedef std::map<TupleId, std::vector<ExprContext*>> ConjunctsMap;
- const ConjunctsMap& conjuncts_map() const { return conjuncts_map_; }
-
- RuntimeProfile::HighWaterMarkCounter* max_compressed_text_file_length() {
- return max_compressed_text_file_length_;
- }
-
- const static int SKIP_COLUMN = -1;
-
- /// Returns index into materialized_slots with 'path'. Returns SKIP_COLUMN if
- /// that path is not materialized.
- int GetMaterializedSlotIdx(const std::vector<int>& path) const {
- PathToSlotIdxMap::const_iterator result = path_to_materialized_slot_idx_.find(path);
- if (result == path_to_materialized_slot_idx_.end()) return SKIP_COLUMN;
- return result->second;
- }
-
- /// The result array is of length hdfs_table_->num_cols(). The i-th element is true iff
- /// column i should be materialized.
- const bool* is_materialized_col() {
- return reinterpret_cast<const bool*>(&is_materialized_col_[0]);
- }
-
- /// Returns the per format codegen'd function. Scanners call this to get the
- /// codegen'd function to use. Returns NULL if codegen should not be used.
- void* GetCodegenFn(THdfsFileFormat::type);
-
- inline void IncNumScannersCodegenEnabled() {
- num_scanners_codegen_enabled_.Add(1);
- }
-
- inline void IncNumScannersCodegenDisabled() {
- num_scanners_codegen_disabled_.Add(1);
- }
+ /// Adds ranges to the io mgr queue and starts up new scanner threads if possible.
+ virtual Status AddDiskIoRanges(const std::vector<DiskIoMgr::ScanRange*>& ranges,
+ int num_files_queued);
/// Adds a materialized row batch for the scan node. This is called from scanner
/// threads.
/// This function will block if materialized_row_batches_ is full.
void AddMaterializedRowBatch(RowBatch* row_batch);
- /// Allocate a new scan range object, stored in the runtime state's object pool. For
- /// scan ranges that correspond to the original hdfs splits, the partition id must be
- /// set to the range's partition id. For other ranges (e.g. columns in parquet, read
- /// past buffers), the partition_id is unused. expected_local should be true if this
- /// scan range is not expected to require a remote read. The range must fall within
- /// the file bounds. That is, the offset must be >= 0, and offset + len <= file_length.
- /// If not NULL, the 'original_split' pointer is stored for reference in the scan range
- /// metadata of the scan range that is to be allocated.
+ /// Sets the scanner specific metadata for 'filename'.
/// This is thread safe.
- DiskIoMgr::ScanRange* AllocateScanRange(
- hdfsFS fs, const char* file, int64_t len, int64_t offset, int64_t partition_id,
- int disk_id, bool try_cache, bool expected_local, int64_t mtime,
- const DiskIoMgr::ScanRange* original_split = NULL);
-
- /// Adds ranges to the io mgr queue and starts up new scanner threads if possible.
- /// 'num_files_queued' indicates how many file's scan ranges have been added
- /// completely. A file's scan ranges are added completely if no new scanner threads
- /// will be needed to process that file besides the additional threads needed to
- /// process those in 'ranges'.
- Status AddDiskIoRanges(const std::vector<DiskIoMgr::ScanRange*>& ranges,
- int num_files_queued);
-
- /// Adds all splits for file_desc to the io mgr queue and indicates one file has
- /// been added completely.
- inline Status AddDiskIoRanges(const HdfsFileDesc* file_desc) {
- return AddDiskIoRanges(file_desc->splits, 1);
- }
-
- /// Allocates and initialises template_tuple_ with any values from the partition columns
- /// for the current scan range
- /// Returns NULL if there are no partition keys slots.
- /// TODO: cache the tuple template in the partition object.
- Tuple* InitTemplateTuple(RuntimeState* state,
- const std::vector<ExprContext*>& value_ctxs);
-
- /// Allocates and return an empty template tuple (i.e. with no values filled in).
- /// Scanners can use this method to initialize a template tuple even if there are no
- /// partition keys slots (e.g. to hold Avro default values).
- Tuple* InitEmptyTemplateTuple(const TupleDescriptor& tuple_desc);
-
- /// Acquires all allocations from pool into scan_node_pool_. Thread-safe.
- void TransferToScanNodePool(MemPool* pool);
-
- /// Returns the file desc for 'filename'. Returns NULL if filename is invalid.
- HdfsFileDesc* GetFileDesc(const std::string& filename);
+ void SetFileMetadata(const std::string& filename, void* metadata);
/// Gets scanner specific metadata for 'filename'. Scanners can use this to store
/// file header information.
@@ -259,117 +94,18 @@ class HdfsScanNode : public ScanNode {
/// This is thread safe.
void* GetFileMetadata(const std::string& filename);
- /// Sets the scanner specific metadata for 'filename'.
- /// This is thread safe.
- void SetFileMetadata(const std::string& filename, void* metadata);
-
- /// Called by the scanner when a range is complete. Used to trigger done_ and
- /// to log progress. This *must* only be called after the scanner has completely
- /// finished the scan range (i.e. context->Flush()), and has added the final
- /// row batch to the row batch queue. Otherwise the last batch may be
- /// lost due to racing with shutting down the row batch queue.
- void RangeComplete(const THdfsFileFormat::type& file_type,
- const THdfsCompression::type& compression_type);
- /// Same as above except for when multiple compression codecs were used
- /// in the file. The metrics are incremented for each compression_type.
- void RangeComplete(const THdfsFileFormat::type& file_type,
+ /// Called by scanners when a range is complete. Used to record progress and set done_.
+ /// This *must* only be called after a scanner has completely finished its
+ /// scan range (i.e. context->Flush()), and has added the final row batch to the row
+ /// batch queue. Otherwise, we may lose the last batch due to racing with shutting down
+ /// the RowBatch queue.
+ virtual void RangeComplete(const THdfsFileFormat::type& file_type,
const std::vector<THdfsCompression::type>& compression_type);
- /// Utility function to compute the order in which to materialize slots to allow for
- /// computing conjuncts as slots get materialized (on partial tuples).
- /// 'order' will contain for each slot, the first conjunct it is associated with.
- /// e.g. order[2] = 1 indicates materialized_slots[2] must be materialized before
- /// evaluating conjuncts[1]. Slots that are not referenced by any conjuncts will have
- /// order set to conjuncts.size()
- void ComputeSlotMaterializationOrder(std::vector<int>* order) const;
-
- /// Returns true if there are no materialized slots, such as a count(*) over the table.
- inline bool IsZeroSlotTableScan() const {
- return materialized_slots().empty() && tuple_desc()->tuple_path().empty();
- }
-
- /// map from volume id to <number of split, per volume split lengths>
- typedef boost::unordered_map<int32_t, std::pair<int, int64_t>> PerVolumnStats;
-
- /// Update the per volume stats with the given scan range params list
- static void UpdateHdfsSplitStats(
- const std::vector<TScanRangeParams>& scan_range_params_list,
- PerVolumnStats* per_volume_stats);
-
- /// Output the per_volume_stats to stringstream. The output format is a list of:
- /// <volume id>:<# splits>/<per volume split lengths>
- static void PrintHdfsSplitStats(const PerVolumnStats& per_volume_stats,
- std::stringstream* ss);
-
- /// Description string for the per volume stats output.
- static const std::string HDFS_SPLIT_STATS_DESC;
-
- /// Returns true if partition 'partition_id' passes all the filter predicates in
- /// 'filter_ctxs' and should not be filtered out. 'stats_name' is the key of one of the
- /// counter groups in FilterStats, and is used to update the correct statistics.
- ///
- /// 'filter_ctxs' is either an empty list, in which case filtering is disabled and the
- /// function returns true, or a set of filter contexts to evaluate.
- bool PartitionPassesFilters(int32_t partition_id, const std::string& stats_name,
- const std::vector<FilterContext>& filter_ctxs);
-
- const std::vector<FilterContext> filter_ctxs() const { return filter_ctxs_; }
-
- protected:
- friend class ScannerContext;
- friend class HdfsScanner;
-
- RuntimeState* runtime_state_;
-
- // Number of header lines to skip at the beginning of each file of this table. Only set
- // to values > 0 for hdfs text files.
- const int skip_header_line_count_;
-
- /// Tuple id resolved in Prepare() to set tuple_desc_
- const int tuple_id_;
-
- /// RequestContext object to use with the disk-io-mgr for reads.
- DiskIoRequestContext* reader_context_;
-
- /// Descriptor for tuples this scan node constructs
- const TupleDescriptor* tuple_desc_;
-
- /// Map from partition ID to a template tuple (owned by scan_node_pool_) which has only
- /// the partition columns for that partition materialized. Used to filter files and scan
- /// ranges on partition-column filters. Populated in Prepare().
- boost::unordered_map<int64_t, Tuple*> partition_template_tuple_map_;
-
- /// Descriptor for the hdfs table, including partition and format metadata.
- /// Set in Prepare, owned by RuntimeState
- const HdfsTableDescriptor* hdfs_table_;
-
- /// The root of the table's Avro schema, if we're scanning an Avro table.
- ScopedAvroSchemaElement avro_schema_;
-
- /// If true, the warning that some disk ids are unknown was logged. Only log this once
- /// per scan node since it can be noisy.
- bool unknown_disk_id_warned_;
-
- /// Partitions scanned by this scan node.
- boost::unordered_set<int64_t> partition_ids_;
-
- /// File path => file descriptor (which includes the file's splits)
- typedef std::map<std::string, HdfsFileDesc*> FileDescMap;
- FileDescMap file_descs_;
-
- /// File format => file descriptors.
- typedef std::map<THdfsFileFormat::type, std::vector<HdfsFileDesc*>> FileFormatsMap;
- FileFormatsMap per_type_files_;
-
- /// Conjuncts for each materialized tuple (top-level row batch tuples and collection
- /// item tuples). Includes a copy of ExecNode.conjuncts_.
- ConjunctsMap conjuncts_map_;
-
- /// Set to true when the initial scan ranges are issued to the IoMgr. This happens on
- /// the first call to GetNext(). The token manager, in a different thread, will read
- /// this variable.
- bool initial_ranges_issued_;
+ /// Acquires all allocations from pool into scan_node_pool_. Thread-safe.
+ void TransferToScanNodePool(MemPool* pool);
+ private:
/// Released when initial ranges are issued in the first call to GetNext().
CountingBarrier ranges_issued_barrier_;
@@ -378,44 +114,6 @@ class HdfsScanNode : public ScanNode {
/// scanner threads.
int64_t scanner_thread_bytes_required_;
- /// Number of files that have not been issued from the scanners.
- AtomicInt32 num_unqueued_files_;
-
- /// Map of HdfsScanner objects to file types. Only one scanner object will be
- /// created for each file type. Objects stored in runtime_state's pool.
- typedef std::map<THdfsFileFormat::type, HdfsScanner*> ScannerMap;
- ScannerMap scanner_map_;
-
- /// Per scanner type codegen'd fn.
- typedef std::map<THdfsFileFormat::type, void*> CodegendFnMap;
- CodegendFnMap codegend_fn_map_;
-
- /// Maps from a slot's path to its index into materialized_slots_.
- typedef boost::unordered_map<std::vector<int>, int> PathToSlotIdxMap;
- PathToSlotIdxMap path_to_materialized_slot_idx_;
-
- /// List of contexts for expected runtime filters for this scan node. These contexts are
- /// cloned by individual scanners to be used in multi-threaded contexts, passed through
- /// the per-scanner ScannerContext..
- std::vector<FilterContext> filter_ctxs_;
-
- /// is_materialized_col_[i] = <true i-th column should be materialized, false otherwise>
- /// for 0 <= i < total # columns in table
- //
- /// This should be a vector<bool>, but bool vectors are special-cased and not stored
- /// internally as arrays, so instead we store as chars and cast to bools as needed
- std::vector<char> is_materialized_col_;
-
- /// Vector containing slot descriptors for all non-partition key slots. These
- /// descriptors are sorted in order of increasing col_pos.
- std::vector<SlotDescriptor*> materialized_slots_;
-
- /// Vector containing slot descriptors for all partition key slots.
- std::vector<SlotDescriptor*> partition_key_slots_;
-
- /// Keeps track of total splits and the number finished.
- ProgressUpdater progress_;
-
/// Scanner specific per file metadata (e.g. header information) and associated lock.
/// This lock cannot be taken together with any other locks except lock_.
boost::mutex metadata_lock_;
@@ -431,43 +129,15 @@ class HdfsScanNode : public ScanNode {
/// Maximum size of materialized_row_batches_.
int max_materialized_row_batches_;
- /// This is the number of io buffers that are owned by the scan node and the scanners.
- /// This is used just to help debug leaked io buffers to determine if the leak is
- /// happening in the scanners vs other parts of the execution.
- AtomicInt32 num_owned_io_buffers_;
-
- /// Counters which track the number of scanners that have codegen enabled for the
- /// materialize and conjuncts evaluation code paths.
- AtomicInt32 num_scanners_codegen_enabled_;
- AtomicInt32 num_scanners_codegen_disabled_;
-
- /// The size of the largest compressed text file to be scanned. This is used to
- /// estimate scanner thread memory usage.
- RuntimeProfile::HighWaterMarkCounter* max_compressed_text_file_length_;
-
- /// Disk accessed bitmap
- RuntimeProfile::Counter disks_accessed_bitmap_;
-
- /// Total number of bytes read locally
- RuntimeProfile::Counter* bytes_read_local_;
-
- /// Total number of bytes read via short circuit read
- RuntimeProfile::Counter* bytes_read_short_circuit_;
-
- /// Total number of bytes read from data node cache
- RuntimeProfile::Counter* bytes_read_dn_cache_;
-
- /// Total number of remote scan ranges
- RuntimeProfile::Counter* num_remote_ranges_;
-
- /// Total number of bytes read remotely that were expected to be local
- RuntimeProfile::Counter* unexpected_remote_bytes_;
-
/// Lock protects access between scanner thread and main query thread (the one calling
/// GetNext()) for all fields below. If this lock and any other locks needs to be taken
/// together, this lock must be taken first.
boost::mutex lock_;
+ /// Protects file_type_counts_. Cannot be taken together with any other lock
+ /// except lock_, and if so, lock_ must be taken first.
+ SpinLock file_type_counts_;
+
/// Flag signaling that all scanner threads are done. This could be because they
/// are finished, an error/cancellation occurred, or the limit was reached.
/// Setting this to true triggers the scanner threads to clean up.
@@ -478,27 +148,6 @@ class HdfsScanNode : public ScanNode {
/// being processed by scanner threads, but no new ScannerThreads should be started.
bool all_ranges_started_;
- /// Pool for allocating some amounts of memory that is shared between scanners.
- /// e.g. partition key tuple and their string buffers
- boost::scoped_ptr<MemPool> scan_node_pool_;
-
- /// Status of failed operations. This is set in the ScannerThreads
- /// Returned in GetNext() if an error occurred. An non-ok status triggers cleanup
- /// scanner threads.
- Status status_;
-
- /// Mapping of file formats (file type, compression type) to the number of
- /// splits of that type and the lock protecting it.
- /// This lock cannot be taken together with any other lock except lock_.
- SpinLock file_type_counts_lock_;
- typedef std::map<
- std::pair<THdfsFileFormat::type, THdfsCompression::type>, int> FileTypeCountsMap;
- FileTypeCountsMap file_type_counts_;
-
- /// If true, counters are actively running and need to be reported in the runtime
- /// profile.
- bool counters_running_;
-
/// The id of the callback added to the thread resource manager when thread token
/// is available. Used to remove the callback before this scan node is destroyed.
/// -1 if no callback is registered.
@@ -519,11 +168,6 @@ class HdfsScanNode : public ScanNode {
/// (e.g., when adding new ranges) or when threads are available for this scan node.
void ThreadTokenAvailableCb(ThreadResourceMgr::ResourcePool* pool);
- /// Create and open new scanner for this partition type.
- /// If the scanner is successfully created, it is returned in 'scanner'.
- Status CreateAndOpenScanner(HdfsPartitionDescriptor* partition,
- ScannerContext* context, boost::scoped_ptr<HdfsScanner>* scanner);
-
/// Main function for scanner thread. This thread pulls the next range to be
/// processed from the IoMgr and then processes the entire range end to end.
/// This thread terminates when all scan ranges are complete or an error occurred.
@@ -546,42 +190,9 @@ class HdfsScanNode : public ScanNode {
/// Checks for eos conditions and returns batches from materialized_row_batches_.
Status GetNextInternal(RuntimeState* state, RowBatch* row_batch, bool* eos);
- /// sets done_ to true and triggers threads to cleanup. Cannot be calld with
+ /// sets done_ to true and triggers threads to cleanup. Cannot be called with
/// any locks taken. Calling it repeatedly ignores subsequent calls.
void SetDone();
-
- /// Stops periodic counters and aggregates counter values for the entire scan node.
- /// This should be called as soon as the scan node is complete to get the most accurate
- /// counter values.
- /// This can be called multiple times, subsequent calls will be ignored.
- /// This must be called on Close() to unregister counters.
- void StopAndFinalizeCounters();
-
- /// Recursively initializes all NULL collection slots to an empty CollectionValue in
- /// addition to maintaining the null bit. Hack to allow UnnestNode to project out
- /// collection slots. Assumes that the null bit has already been un/set.
- /// TODO: remove this function once the TODOs in UnnestNode regarding projection
- /// have been addressed.
- void InitNullCollectionValues(const TupleDescriptor* tuple_desc, Tuple* tuple) const;
-
- /// Helper to call InitNullCollectionValues() on all tuples produced by this scan
- /// in 'row_batch'.
- void InitNullCollectionValues(RowBatch* row_batch) const;
-
- /// Returns false if, according to filters in 'filter_ctxs', 'file' should be filtered
- /// and therefore not processed. 'file_type' is the the format of 'file', and is used
- /// for bookkeeping. Returns true if all filters pass or are not present.
- bool FilePassesFilterPredicates(const std::vector<FilterContext>& filter_ctxs,
- const THdfsFileFormat::type& file_type, HdfsFileDesc* file);
-
- /// Waits for up to time_ms for runtime filters to arrive, checking every 20ms. Returns
- /// true if all filters arrived within the time limit (as measured from the time of
- /// RuntimeFilterBank::RegisterFilter()), false otherwise.
- bool WaitForRuntimeFilters(int32_t time_ms);
-
- /// Calls ExecDebugAction(). Returns the status based on the debug action specified
- /// for the query.
- Status TriggerDebugAction();
};
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scanner-ir.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner-ir.cc b/be/src/exec/hdfs-scanner-ir.cc
index 2b053ca..e6da220 100644
--- a/be/src/exec/hdfs-scanner-ir.cc
+++ b/be/src/exec/hdfs-scanner-ir.cc
@@ -37,7 +37,7 @@ int HdfsScanner::WriteAlignedTuples(MemPool* pool, TupleRow* tuple_row, int row_
FieldLocation* fields, int num_tuples, int max_added_tuples,
int slots_per_tuple, int row_idx_start) {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
DCHECK(tuple_ != NULL);
uint8_t* tuple_row_mem = reinterpret_cast<uint8_t*>(tuple_row);
uint8_t* tuple_mem = reinterpret_cast<uint8_t*>(tuple_);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner.cc b/be/src/exec/hdfs-scanner.cc
index ad3d834..841d1e4 100644
--- a/be/src/exec/hdfs-scanner.cc
+++ b/be/src/exec/hdfs-scanner.cc
@@ -26,6 +26,7 @@
#include "common/object-pool.h"
#include "exec/text-converter.h"
#include "exec/hdfs-scan-node.h"
+#include "exec/hdfs-scan-node-mt.h"
#include "exec/read-write-util.h"
#include "exec/text-converter.inline.h"
#include "exprs/expr-context.h"
@@ -56,20 +57,20 @@ using namespace strings;
const char* FieldLocation::LLVM_CLASS_NAME = "struct.impala::FieldLocation";
const char* HdfsScanner::LLVM_CLASS_NAME = "class.impala::HdfsScanner";
-HdfsScanner::HdfsScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue)
+HdfsScanner::HdfsScanner(HdfsScanNodeBase* scan_node, RuntimeState* state)
: scan_node_(scan_node),
state_(state),
- add_batches_to_queue_(add_batches_to_queue),
context_(NULL),
stream_(NULL),
+ eos_(false),
scanner_conjunct_ctxs_(NULL),
+ template_tuple_pool_(new MemPool(scan_node->mem_tracker())),
template_tuple_(NULL),
tuple_byte_size_(scan_node->tuple_desc()->byte_size()),
+ num_null_bytes_(scan_node->tuple_desc()->num_null_bytes()),
tuple_(NULL),
batch_(NULL),
tuple_mem_(NULL),
- num_null_bytes_(scan_node->tuple_desc()->num_null_bytes()),
parse_status_(Status::OK()),
decompression_type_(THdfsCompression::NONE),
data_buffer_pool_(new MemPool(scan_node->mem_tracker())),
@@ -80,16 +81,17 @@ HdfsScanner::HdfsScanner(HdfsScanNode* scan_node, RuntimeState* state,
HdfsScanner::HdfsScanner()
: scan_node_(NULL),
state_(NULL),
- add_batches_to_queue_(true),
context_(NULL),
stream_(NULL),
+ eos_(false),
scanner_conjunct_ctxs_(NULL),
+ template_tuple_pool_(NULL),
template_tuple_(NULL),
tuple_byte_size_(-1),
+ num_null_bytes_(-1),
tuple_(NULL),
batch_(NULL),
tuple_mem_(NULL),
- num_null_bytes_(-1),
parse_status_(Status::OK()),
decompression_type_(THdfsCompression::NONE),
data_buffer_pool_(NULL),
@@ -107,19 +109,23 @@ Status HdfsScanner::Open(ScannerContext* context) {
// Clone the scan node's conjuncts map. The cloned contexts must be closed by the
// caller.
- HdfsScanNode::ConjunctsMap::const_iterator iter = scan_node_->conjuncts_map().begin();
- for (; iter != scan_node_->conjuncts_map().end(); ++iter) {
- RETURN_IF_ERROR(Expr::CloneIfNotExists(iter->second,
- scan_node_->runtime_state(), &scanner_conjuncts_map_[iter->first]));
+ for (const auto& entry: scan_node_->conjuncts_map()) {
+ RETURN_IF_ERROR(Expr::CloneIfNotExists(entry.second,
+ scan_node_->runtime_state(), &scanner_conjuncts_map_[entry.first]));
}
DCHECK(scanner_conjuncts_map_.find(scan_node_->tuple_desc()->id()) !=
scanner_conjuncts_map_.end());
scanner_conjunct_ctxs_ = &scanner_conjuncts_map_[scan_node_->tuple_desc()->id()];
// Initialize the template_tuple_.
- template_tuple_ = scan_node_->InitTemplateTuple(
- state_, context_->partition_descriptor()->partition_key_value_ctxs());
+ vector<ExprContext*> partition_key_value_ctxs;
+ RETURN_IF_ERROR(Expr::CloneIfNotExists(
+ context_->partition_descriptor()->partition_key_value_ctxs(), state_,
+ &partition_key_value_ctxs));
+ template_tuple_ = scan_node_->InitTemplateTuple(partition_key_value_ctxs,
+ template_tuple_pool_.get(), state_);
template_tuple_map_[scan_node_->tuple_desc()] = template_tuple_;
+ Expr::Close(partition_key_value_ctxs, state_);
decompress_timer_ = ADD_TIMER(scan_node_->runtime_profile(), "DecompressionTime");
return Status::OK();
@@ -127,10 +133,7 @@ Status HdfsScanner::Open(ScannerContext* context) {
void HdfsScanner::Close(RowBatch* row_batch) {
if (decompressor_.get() != NULL) decompressor_->Close();
- HdfsScanNode::ConjunctsMap::const_iterator iter = scanner_conjuncts_map_.begin();
- for (; iter != scanner_conjuncts_map_.end(); ++iter) {
- Expr::Close(iter->second, state_);
- }
+ for (const auto& entry: scanner_conjuncts_map_) Expr::Close(entry.second, state_);
obj_pool_.Clear();
stream_ = NULL;
context_->ClearStreams();
@@ -158,7 +161,7 @@ Status HdfsScanner::InitializeWriteTuplesFn(HdfsPartitionDescriptor* partition,
}
Status HdfsScanner::StartNewRowBatch() {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
batch_ = new RowBatch(scan_node_->row_desc(), state_->batch_size(),
scan_node_->mem_tracker());
int64_t tuple_buffer_size;
@@ -168,7 +171,7 @@ Status HdfsScanner::StartNewRowBatch() {
}
int HdfsScanner::GetMemory(MemPool** pool, Tuple** tuple_mem, TupleRow** tuple_row_mem) {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
DCHECK(batch_ != NULL);
DCHECK_GT(batch_->capacity(), batch_->num_rows());
*pool = batch_->tuple_data_pool();
@@ -190,7 +193,7 @@ Status HdfsScanner::GetCollectionMemory(CollectionValueBuilder* builder, MemPool
// TODO(skye): have this check scan_node_->ReachedLimit() and get rid of manual check?
Status HdfsScanner::CommitRows(int num_rows) {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
DCHECK(batch_ != NULL);
DCHECK_LE(num_rows, batch_->capacity() - batch_->num_rows());
batch_->CommitRows(num_rows);
@@ -201,16 +204,15 @@ Status HdfsScanner::CommitRows(int num_rows) {
// if no rows passed predicates.
if (batch_->AtCapacity() || context_->num_completed_io_buffers() > 0) {
context_->ReleaseCompletedResources(batch_, /* done */ false);
- scan_node_->AddMaterializedRowBatch(batch_);
+ static_cast<HdfsScanNode*>(scan_node_)->AddMaterializedRowBatch(batch_);
RETURN_IF_ERROR(StartNewRowBatch());
}
if (context_->cancelled()) return Status::CANCELLED;
// Check for UDF errors.
RETURN_IF_ERROR(state_->GetQueryStatus());
// Free local expr allocations for this thread
- HdfsScanNode::ConjunctsMap::const_iterator iter = scanner_conjuncts_map_.begin();
- for (; iter != scanner_conjuncts_map_.end(); ++iter) {
- ExprContext::FreeLocalAllocations(iter->second);
+ for (const auto& entry: scanner_conjuncts_map_) {
+ ExprContext::FreeLocalAllocations(entry.second);
}
return Status::OK();
}
@@ -357,8 +359,9 @@ bool HdfsScanner::WriteCompleteTuple(MemPool* pool, FieldLocation* fields,
// eval_fail: ; preds = %parse
// ret i1 false
// }
-Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNode* node, LlvmCodeGen* codegen,
- const vector<ExprContext*>& conjunct_ctxs, Function** write_complete_tuple_fn) {
+Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNodeBase* node,
+ LlvmCodeGen* codegen, const vector<ExprContext*>& conjunct_ctxs,
+ Function** write_complete_tuple_fn) {
*write_complete_tuple_fn = NULL;
SCOPED_TIMER(codegen->codegen_timer());
RuntimeState* state = node->runtime_state();
@@ -565,8 +568,9 @@ Status HdfsScanner::CodegenWriteCompleteTuple(HdfsScanNode* node, LlvmCodeGen* c
return Status::OK();
}
-Status HdfsScanner::CodegenWriteAlignedTuples(HdfsScanNode* node, LlvmCodeGen* codegen,
- Function* write_complete_tuple_fn, Function** write_aligned_tuples_fn) {
+Status HdfsScanner::CodegenWriteAlignedTuples(HdfsScanNodeBase* node,
+ LlvmCodeGen* codegen, Function* write_complete_tuple_fn,
+ Function** write_aligned_tuples_fn) {
*write_aligned_tuples_fn = NULL;
SCOPED_TIMER(codegen->codegen_timer());
DCHECK(write_complete_tuple_fn != NULL);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-scanner.h b/be/src/exec/hdfs-scanner.h
index 6245e5f..d9fa424 100644
--- a/be/src/exec/hdfs-scanner.h
+++ b/be/src/exec/hdfs-scanner.h
@@ -27,7 +27,7 @@
#include "codegen/impala-ir.h"
#include "common/object-pool.h"
-#include "exec/hdfs-scan-node.h"
+#include "exec/hdfs-scan-node-base.h"
#include "exec/scan-node.h"
#include "exec/scanner-context.h"
#include "runtime/disk-io-mgr.h"
@@ -75,8 +75,10 @@ struct FieldLocation {
/// 1. Open()
/// 2. ProcessSplit() or GetNext()*
/// 3. Close()
-/// The scanner can be used in either of two modes, indicated via the add_batches_to_queue
-/// c'tor parameter.
+/// The scanner can be used in either of two modes. Which mode is expected to be used
+/// depends on the type of parent scan node. Parent scan nodes with a row batch queue
+/// are expected to call ProcessSplit() and not GetNext(). Row batches will be added to
+/// the scan node's row batch queue, including the final one in Close().
/// ProcessSplit() scans the split and adds materialized row batches to the scan node's
/// row batch queue until the split is complete or an error occurred.
/// GetNext() provides an iterator-like interface where the caller can request
@@ -103,16 +105,15 @@ struct FieldLocation {
/// resources (IO buffers and mem pools) to the current row batch, and passing row batches
/// up to the scan node. Subclasses can also use GetMemory() to help with per-row memory
/// management.
+/// TODO: Have a pass over all members and move them out of the base class if sensible
+/// to clarify which state each concrete scanner type actually has.
class HdfsScanner {
public:
/// Assumed size of an OS file block. Used mostly when reading file format headers, etc.
/// This probably ought to be a derived number from the environment.
const static int FILE_BLOCK_SIZE = 4096;
- /// If 'add_batches_to_queue' is true the caller must call ProcessSplit() and not
- /// GetNext(). Row batches will be added to the scan node's row batch queue, including
- /// the final one in Close().
- HdfsScanner(HdfsScanNode* scan_node, RuntimeState* state, bool add_batches_to_queue);
+ HdfsScanner(HdfsScanNodeBase* scan_node, RuntimeState* state);
virtual ~HdfsScanner();
@@ -125,16 +126,16 @@ class HdfsScanner {
/// returned, 'parse_status_' is guaranteed to be OK as well.
/// The memory referenced by the tuples is valid until this or any subsequently
/// returned batch is reset or destroyed.
- /// Only valid to call if 'add_batches_to_queue_' is false.
- Status GetNext(RowBatch* row_batch, bool* eos) {
- DCHECK(!add_batches_to_queue_);
- return GetNextInternal(row_batch, eos);
+ /// Only valid to call if the parent scan node is single-threaded.
+ Status GetNext(RowBatch* row_batch) {
+ DCHECK(!scan_node_->HasRowBatchQueue());
+ return GetNextInternal(row_batch);
}
/// Process an entire split, reading bytes from the context's streams. Context is
/// initialized with the split data (e.g. template tuple, partition descriptor, etc).
/// This function should only return on error or end of scan range.
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is multi-threaded.
virtual Status ProcessSplit() = 0;
/// Transfers the ownership of memory backing returned tuples such as IO buffers
@@ -143,9 +144,15 @@ class HdfsScanner {
/// that are not backing returned rows (e.g. temporary decompression buffers).
virtual void Close(RowBatch* row_batch);
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is single-threaded.
+ bool eos() const {
+ DCHECK(!scan_node_->HasRowBatchQueue());
+ return eos_;
+ }
+
+ /// Only valid to call if the parent scan node is multi-threaded.
RowBatch* batch() const {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
return batch_;
}
@@ -177,16 +184,11 @@ class HdfsScanner {
protected:
/// The scan node that started this scanner
- HdfsScanNode* scan_node_;
+ HdfsScanNodeBase* scan_node_;
/// RuntimeState for error reporting
RuntimeState* state_;
- /// True if the creator of this scanner intends to use ProcessSplit() and not GetNext).
- /// Row batches will be added to the scan node's row batch queue, including the final
- /// one in Close().
- const bool add_batches_to_queue_;
-
/// Context for this scanner
ScannerContext* context_;
@@ -196,14 +198,24 @@ class HdfsScanner {
/// The first stream for context_
ScannerContext::Stream* stream_;
+ /// Set if this scanner has processed all ranges and will not produce more rows.
+ /// Only relevant when calling the GetNext() interface.
+ bool eos_;
+
/// Clones of the conjuncts ExprContexts in scan_node_->conjuncts_map(). Each scanner
/// has its own ExprContexts so the conjuncts can be safely evaluated in parallel.
- HdfsScanNode::ConjunctsMap scanner_conjuncts_map_;
+ HdfsScanNodeBase::ConjunctsMap scanner_conjuncts_map_;
// Convenience reference to scanner_conjuncts_map_[scan_node_->tuple_idx()] for scanners
// that do not support nested types.
const std::vector<ExprContext*>* scanner_conjunct_ctxs_;
+ /// Holds memory for template tuples. The memory in this pool must remain valid as long
+ /// as the row batches produced by this scanner. This typically means that the
+ /// ownership is transferred to the last row batch in Close(). Some scanners transfer
+ /// the ownership to the parent scan node instead due being closed multiple times.
+ boost::scoped_ptr<MemPool> template_tuple_pool_;
+
/// A template tuple is a partially-materialized tuple with only partition key slots set
/// (or other default values, such as NULL for columns missing in a file). The other
/// slots are set to NULL. The template tuple must be copied into output tuples before
@@ -211,9 +223,8 @@ class HdfsScanner {
///
/// Each tuple descriptor (i.e. scan_node_->tuple_desc() and any collection item tuple
/// descs) has a template tuple, or NULL if there are no partition key or default slots.
- /// Template tuples are computed once for each file and valid for the duration of that
- /// file. They are owned by the HDFS scan node, although each scanner has its own
- /// template tuples.
+ /// Template tuples are computed once for each file and are allocated from
+ /// template_tuple_pool_.
std::map<const TupleDescriptor*, Tuple*> template_tuple_map_;
/// Convenience variable set to the top-level template tuple
@@ -221,7 +232,10 @@ class HdfsScanner {
Tuple* template_tuple_;
/// Fixed size of each top-level tuple, in bytes
- int tuple_byte_size_;
+ const int32_t tuple_byte_size_;
+
+ /// Number of null bytes in the top-level tuple.
+ const int32_t num_null_bytes_;
/// Current tuple pointer into tuple_mem_.
Tuple* tuple_;
@@ -239,9 +253,6 @@ class HdfsScanner {
/// Helper class for converting text to other types;
boost::scoped_ptr<TextConverter> text_converter_;
- /// Number of null bytes in the top-level tuple.
- int32_t num_null_bytes_;
-
/// Contains current parse status to minimize the number of Status objects returned.
/// This significantly minimizes the cross compile dependencies for llvm since status
/// objects inline a bunch of string functions. Also, status objects aren't extremely
@@ -269,8 +280,8 @@ class HdfsScanner {
WriteTuplesFn write_tuples_fn_;
/// Implements GetNext(). Should be overridden by subclasses.
- /// May be called even if 'add_batches_to_queue_' is true.
- virtual Status GetNextInternal(RowBatch* row_batch, bool* eos) {
+ /// Only valid to call if the parent scan node is multi-threaded.
+ virtual Status GetNextInternal(RowBatch* row_batch) {
DCHECK(false) << "GetNextInternal() not implemented for this scanner type.";
return Status::OK();
}
@@ -283,7 +294,7 @@ class HdfsScanner {
THdfsFileFormat::type type, const std::string& scanner_name);
/// Set 'batch_' to a new row batch and update 'tuple_mem_' accordingly.
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is multi-threaded.
Status StartNewRowBatch();
/// Reset internal state for a new scan range.
@@ -297,7 +308,7 @@ class HdfsScanner {
/// current row batch is complete and a new one is allocated).
/// Memory returned from this call is invalidated after calling CommitRows().
/// Callers must call GetMemory() again after calling this function.
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is multi-threaded.
int GetMemory(MemPool** pool, Tuple** tuple_mem, TupleRow** tuple_row_mem);
/// Gets memory for outputting tuples into the CollectionValue being constructed via
@@ -316,15 +327,15 @@ class HdfsScanner {
/// Returns Status::OK if the query is not cancelled and hasn't exceeded any mem limits.
/// Scanner can call this with 0 rows to flush any pending resources (attached pools
/// and io buffers) to minimize memory consumption.
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is multi-threaded.
Status CommitRows(int num_rows);
/// Release all memory in 'pool' to batch_. If commit_batch is true, the row batch
/// will be committed. commit_batch should be true if the attached pool is expected
/// to be non-trivial (i.e. a decompression buffer) to minimize scanner mem usage.
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is multi-threaded.
void AttachPool(MemPool* pool, bool commit_batch) {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
DCHECK(batch_ != NULL);
DCHECK(pool != NULL);
batch_->tuple_data_pool()->AcquireData(pool, false);
@@ -359,7 +370,7 @@ class HdfsScanner {
/// Returns the number of tuples added to the row batch. This can be less than
/// num_tuples/tuples_till_limit because of failed conjuncts.
/// Returns -1 if parsing should be aborted due to parse errors.
- /// Only valid to call if 'add_batches_to_queue_' is true.
+ /// Only valid to call if the parent scan node is multi-threaded.
int WriteAlignedTuples(MemPool* pool, TupleRow* tuple_row_mem, int row_size,
FieldLocation* fields, int num_tuples,
int max_added_tuples, int slots_per_tuple, int row_start_indx);
@@ -410,7 +421,7 @@ class HdfsScanner {
/// Codegen function to replace WriteCompleteTuple. Should behave identically
/// to WriteCompleteTuple. Stores the resulting function in 'write_complete_tuple_fn'
/// if codegen was successful or NULL otherwise.
- static Status CodegenWriteCompleteTuple(HdfsScanNode*, LlvmCodeGen*,
+ static Status CodegenWriteCompleteTuple(HdfsScanNodeBase* node, LlvmCodeGen* codegen,
const std::vector<ExprContext*>& conjunct_ctxs,
llvm::Function** write_complete_tuple_fn);
@@ -418,7 +429,7 @@ class HdfsScanner {
/// compiled to IR. This function loads the precompiled IR function, modifies it,
/// and stores the resulting function in 'write_aligned_tuples_fn' if codegen was
/// successful or NULL otherwise.
- static Status CodegenWriteAlignedTuples(HdfsScanNode*, LlvmCodeGen*,
+ static Status CodegenWriteAlignedTuples(HdfsScanNodeBase*, LlvmCodeGen*,
llvm::Function* write_tuple_fn, llvm::Function** write_aligned_tuples_fn);
/// Report parse error for column @ desc. If abort_on_error is true, sets
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-sequence-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-sequence-scanner.cc b/be/src/exec/hdfs-sequence-scanner.cc
index ec52901..03a07ce 100644
--- a/be/src/exec/hdfs-sequence-scanner.cc
+++ b/be/src/exec/hdfs-sequence-scanner.cc
@@ -41,9 +41,9 @@ const uint8_t HdfsSequenceScanner::SEQFILE_VERSION_HEADER[4] = {'S', 'E', 'Q', 6
#define RETURN_IF_FALSE(x) if (UNLIKELY(!(x))) return parse_status_
-HdfsSequenceScanner::HdfsSequenceScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue)
- : BaseSequenceScanner(scan_node, state, add_batches_to_queue),
+HdfsSequenceScanner::HdfsSequenceScanner(HdfsScanNodeBase* scan_node,
+ RuntimeState* state)
+ : BaseSequenceScanner(scan_node, state),
unparsed_data_buffer_(NULL),
num_buffered_records_in_compressed_block_(0) {
}
@@ -52,7 +52,7 @@ HdfsSequenceScanner::~HdfsSequenceScanner() {
}
// Codegen for materialized parsed data into tuples.
-Status HdfsSequenceScanner::Codegen(HdfsScanNode* node,
+Status HdfsSequenceScanner::Codegen(HdfsScanNodeBase* node,
const vector<ExprContext*>& conjunct_ctxs, Function** write_aligned_tuples_fn) {
*write_aligned_tuples_fn = NULL;
if (!node->runtime_state()->codegen_enabled()) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-sequence-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-sequence-scanner.h b/be/src/exec/hdfs-sequence-scanner.h
index e2dcee9..5d6074f 100644
--- a/be/src/exec/hdfs-sequence-scanner.h
+++ b/be/src/exec/hdfs-sequence-scanner.h
@@ -162,8 +162,7 @@ class HdfsSequenceScanner : public BaseSequenceScanner {
/// SeqFile file: {'S', 'E', 'Q', 6}
static const uint8_t SEQFILE_VERSION_HEADER[4];
- HdfsSequenceScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue);
+ HdfsSequenceScanner(HdfsScanNodeBase* scan_node, RuntimeState* state);
virtual ~HdfsSequenceScanner();
@@ -172,7 +171,8 @@ class HdfsSequenceScanner : public BaseSequenceScanner {
/// Codegen WriteAlignedTuples(). Stores the resulting function in
/// 'write_aligned_tuples_fn' if codegen was successful or NULL otherwise.
- static Status Codegen(HdfsScanNode*, const std::vector<ExprContext*>& conjunct_ctxs,
+ static Status Codegen(HdfsScanNodeBase* node,
+ const std::vector<ExprContext*>& conjunct_ctxs,
llvm::Function** write_aligned_tuples_fn);
protected:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-text-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-text-scanner.cc b/be/src/exec/hdfs-text-scanner.cc
index f1c80b0..f3f0a7c 100644
--- a/be/src/exec/hdfs-text-scanner.cc
+++ b/be/src/exec/hdfs-text-scanner.cc
@@ -50,9 +50,8 @@ const string HdfsTextScanner::LZO_INDEX_SUFFIX = ".index";
// progress.
const int64_t COMPRESSED_DATA_FIXED_READ_SIZE = 1 * 1024 * 1024;
-HdfsTextScanner::HdfsTextScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue)
- : HdfsScanner(scan_node, state, add_batches_to_queue),
+HdfsTextScanner::HdfsTextScanner(HdfsScanNodeBase* scan_node, RuntimeState* state)
+ : HdfsScanner(scan_node, state),
byte_buffer_ptr_(NULL),
byte_buffer_end_(NULL),
byte_buffer_read_size_(0),
@@ -67,7 +66,7 @@ HdfsTextScanner::HdfsTextScanner(HdfsScanNode* scan_node, RuntimeState* state,
HdfsTextScanner::~HdfsTextScanner() {
}
-Status HdfsTextScanner::IssueInitialRanges(HdfsScanNode* scan_node,
+Status HdfsTextScanner::IssueInitialRanges(HdfsScanNodeBase* scan_node,
const vector<HdfsFileDesc*>& files) {
vector<DiskIoMgr::ScanRange*> compressed_text_scan_ranges;
int compressed_text_files = 0;
@@ -108,7 +107,7 @@ Status HdfsTextScanner::IssueInitialRanges(HdfsScanNode* scan_node,
// Populate the list of compressed text scan ranges.
DCHECK_GT(files[i]->file_length, 0);
ScanRangeMetadata* metadata =
- reinterpret_cast<ScanRangeMetadata*>(split->meta_data());
+ static_cast<ScanRangeMetadata*>(split->meta_data());
DiskIoMgr::ScanRange* file_range = scan_node->AllocateScanRange(
files[i]->fs, files[i]->filename.c_str(), files[i]->file_length, 0,
metadata->partition_id, split->disk_id(), split->try_cache(),
@@ -149,7 +148,7 @@ Status HdfsTextScanner::IssueInitialRanges(HdfsScanNode* scan_node,
}
Status HdfsTextScanner::ProcessSplit() {
- DCHECK(add_batches_to_queue_);
+ DCHECK(scan_node_->HasRowBatchQueue());
// Reset state for new scan range
RETURN_IF_ERROR(InitNewRange());
@@ -187,12 +186,19 @@ void HdfsTextScanner::Close(RowBatch* row_batch) {
decompressor_.reset(NULL);
}
if (row_batch != NULL) {
+ row_batch->tuple_data_pool()->AcquireData(template_tuple_pool_.get(), false);
row_batch->tuple_data_pool()->AcquireData(data_buffer_pool_.get(), false);
row_batch->tuple_data_pool()->AcquireData(boundary_pool_.get(), false);
context_->ReleaseCompletedResources(row_batch, true);
- if (add_batches_to_queue_) scan_node_->AddMaterializedRowBatch(row_batch);
+ if (scan_node_->HasRowBatchQueue()) {
+ static_cast<HdfsScanNode*>(scan_node_)->AddMaterializedRowBatch(row_batch);
+ }
+ } else {
+ if (template_tuple_pool_.get() != NULL) template_tuple_pool_->FreeAll();
}
+
// Verify all resources (if any) have been transferred.
+ DCHECK_EQ(template_tuple_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(data_buffer_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(boundary_pool_.get()->total_allocated_bytes(), 0);
DCHECK_EQ(context_->num_completed_io_buffers(), 0);
@@ -687,7 +693,7 @@ Status HdfsTextScanner::CheckForSplitDelimiter(bool* split_delimiter) {
// Codegen for materializing parsed data into tuples. The function WriteCompleteTuple is
// codegen'd using the IRBuilder for the specific tuple description. This function
// is then injected into the cross-compiled driving function, WriteAlignedTuples().
-Status HdfsTextScanner::Codegen(HdfsScanNode* node,
+Status HdfsTextScanner::Codegen(HdfsScanNodeBase* node,
const vector<ExprContext*>& conjunct_ctxs, Function** write_aligned_tuples_fn) {
*write_aligned_tuples_fn = NULL;
if (!node->runtime_state()->codegen_enabled()) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/hdfs-text-scanner.h
----------------------------------------------------------------------
diff --git a/be/src/exec/hdfs-text-scanner.h b/be/src/exec/hdfs-text-scanner.h
index 937626f..e68d45f 100644
--- a/be/src/exec/hdfs-text-scanner.h
+++ b/be/src/exec/hdfs-text-scanner.h
@@ -47,8 +47,7 @@ struct HdfsFileDesc;
/// scanner for the tuple directly after it.
class HdfsTextScanner : public HdfsScanner {
public:
- HdfsTextScanner(HdfsScanNode* scan_node, RuntimeState* state,
- bool add_batches_to_queue);
+ HdfsTextScanner(HdfsScanNodeBase* scan_node, RuntimeState* state);
virtual ~HdfsTextScanner();
/// Implementation of HdfsScanner interface.
@@ -57,12 +56,13 @@ class HdfsTextScanner : public HdfsScanner {
virtual void Close(RowBatch* row_batch);
/// Issue io manager byte ranges for 'files'.
- static Status IssueInitialRanges(HdfsScanNode* scan_node,
+ static Status IssueInitialRanges(HdfsScanNodeBase* scan_node,
const std::vector<HdfsFileDesc*>& files);
/// Codegen WriteAlignedTuples(). Stores the resulting function in
/// 'write_aligned_tuples_fn' if codegen was successful or NULL otherwise.
- static Status Codegen(HdfsScanNode*, const std::vector<ExprContext*>& conjunct_ctxs,
+ static Status Codegen(HdfsScanNodeBase* node,
+ const std::vector<ExprContext*>& conjunct_ctxs,
llvm::Function** write_aligned_tuples_fn);
/// Suffix for lzo index files.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/scanner-context.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.cc b/be/src/exec/scanner-context.cc
index 69d682e..033ab82 100644
--- a/be/src/exec/scanner-context.cc
+++ b/be/src/exec/scanner-context.cc
@@ -19,6 +19,7 @@
#include <gutil/strings/substitute.h>
+#include "exec/hdfs-scan-node-base.h"
#include "exec/hdfs-scan-node.h"
#include "runtime/row-batch.h"
#include "runtime/mem-pool.h"
@@ -40,7 +41,7 @@ static const int64_t INIT_READ_PAST_SIZE_BYTES = 64 * 1024;
// output_buffer_bytes_left_ will be set to something else.
static const int64_t OUTPUT_BUFFER_BYTES_LEFT_INIT = 0;
-ScannerContext::ScannerContext(RuntimeState* state, HdfsScanNode* scan_node,
+ScannerContext::ScannerContext(RuntimeState* state, HdfsScanNodeBase* scan_node,
HdfsPartitionDescriptor* partition_desc, DiskIoMgr::ScanRange* scan_range,
const vector<FilterContext>& filter_ctxs)
: state_(state),
@@ -311,7 +312,8 @@ Status ScannerContext::Stream::GetBytesInternal(int64_t requested_len,
}
bool ScannerContext::cancelled() const {
- return scan_node_->done_;
+ if (!scan_node_->HasRowBatchQueue()) return false;
+ return static_cast<HdfsScanNode*>(scan_node_)->done();
}
Status ScannerContext::Stream::ReportIncompleteRead(int64_t length, int64_t bytes_read) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exec/scanner-context.h
----------------------------------------------------------------------
diff --git a/be/src/exec/scanner-context.h b/be/src/exec/scanner-context.h
index 9178324..1f1bc0d 100644
--- a/be/src/exec/scanner-context.h
+++ b/be/src/exec/scanner-context.h
@@ -31,7 +31,7 @@ namespace impala {
struct HdfsFileDesc;
class HdfsPartitionDescriptor;
-class HdfsScanNode;
+class HdfsScanNodeBase;
class MemPool;
class RowBatch;
class RuntimeState;
@@ -55,12 +55,14 @@ class TupleRow;
/// from processing the bytes. This is the consumer.
/// 3. The scan node/main thread which calls into the context to trigger cancellation
/// or other end of stream conditions.
+/// TODO: Some of the synchronization mechanisms such as cancelled() can be removed
+/// once the legacy hdfs scan node has been removed.
class ScannerContext {
public:
/// Create a scanner context with the parent scan_node (where materialized row batches
/// get pushed to) and the scan range to process.
/// This context starts with 1 stream.
- ScannerContext(RuntimeState*, HdfsScanNode*, HdfsPartitionDescriptor*,
+ ScannerContext(RuntimeState*, HdfsScanNodeBase*, HdfsPartitionDescriptor*,
DiskIoMgr::ScanRange* scan_range, const std::vector<FilterContext>& filter_ctxs);
/// Destructor verifies that all stream objects have been released.
@@ -295,7 +297,8 @@ class ScannerContext {
/// The stream is created in the runtime state's object pool
Stream* AddStream(DiskIoMgr::ScanRange* range);
- /// If true, the ScanNode has been cancelled and the scanner thread should finish up
+ /// Returns false it scan_node_ is multi-threaded and has been cancelled.
+ /// Always returns false if the scan_node_ is not multi-threaded.
bool cancelled() const;
int num_completed_io_buffers() const { return num_completed_io_buffers_; }
@@ -306,7 +309,7 @@ class ScannerContext {
friend class Stream;
RuntimeState* state_;
- HdfsScanNode* scan_node_;
+ HdfsScanNodeBase* scan_node_;
HdfsPartitionDescriptor* partition_desc_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/exprs/expr-context.h
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-context.h b/be/src/exprs/expr-context.h
index 3c5d5af..09de6a5 100644
--- a/be/src/exprs/expr-context.h
+++ b/be/src/exprs/expr-context.h
@@ -110,8 +110,9 @@ class ExprContext {
return fn_contexts_[i];
}
- Expr* root() { return root_; }
- bool closed() { return closed_; }
+ Expr* root() const { return root_; }
+ bool closed() const { return closed_; }
+ bool is_clone() const { return is_clone_; }
/// Calls Get*Val on root_
BooleanVal GetBooleanVal(TupleRow* row);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/be/src/runtime/tuple.h
----------------------------------------------------------------------
diff --git a/be/src/runtime/tuple.h b/be/src/runtime/tuple.h
index 10811ea..3a3e399 100644
--- a/be/src/runtime/tuple.h
+++ b/be/src/runtime/tuple.h
@@ -19,7 +19,6 @@
#ifndef IMPALA_RUNTIME_TUPLE_H
#define IMPALA_RUNTIME_TUPLE_H
-#include <cstring>
#include "codegen/impala-ir.h"
#include "common/logging.h"
#include "gutil/macros.h"
@@ -70,9 +69,7 @@ class Tuple {
return result;
}
- void Init(int size) {
- bzero(this, size);
- }
+ void Init(int size) { memset(this, 0, size); }
/// The total size of all data represented in this tuple (tuple data and referenced
/// string and collection data).
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index f4b070f..bf03d98 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -183,10 +183,9 @@ struct TQueryOptions {
43: optional TParquetFallbackSchemaResolution parquet_fallback_schema_resolution = 0
// Multi-threaded execution: number of cores per query per node.
- // > 1: multi-threaded execution mode, with given number of cores
- // 1: single-threaded execution mode
- // 0: multi-threaded execution mode, number of cores is the pool default
- 44: optional i32 mt_num_cores = 1
+ // > 0: multi-threaded execution mode, with given number of cores
+ // 0: single-threaded execution mode
+ 44: optional i32 mt_num_cores = 0
// If true, INSERT writes to S3 go directly to their final location rather than being
// copied there by the coordinator. We cannot do this for INSERT OVERWRITES because for
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/faebfebd/fe/src/main/java/com/cloudera/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/service/Frontend.java b/fe/src/main/java/com/cloudera/impala/service/Frontend.java
index 83af818..2d38396 100644
--- a/fe/src/main/java/com/cloudera/impala/service/Frontend.java
+++ b/fe/src/main/java/com/cloudera/impala/service/Frontend.java
@@ -956,7 +956,7 @@ public class Frontend {
LOG.debug("create plan");
Planner planner = new Planner(analysisResult, queryCtx);
if (RuntimeEnv.INSTANCE.isTestEnv()
- && queryCtx.request.query_options.mt_num_cores != 1) {
+ && queryCtx.request.query_options.mt_num_cores > 0) {
// TODO: this is just to be able to run tests; implement this
List<PlanFragment> planRoots = planner.createParallelPlans();
for (PlanFragment planRoot: planRoots) {