You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by kw...@apache.org on 2016/09/30 02:14:33 UTC
[16/61] [partial] incubator-impala git commit: IMPALA-3786: Replace
"cloudera" with "apache" (part 1)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/planner/SortNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/SortNode.java b/fe/src/main/java/com/cloudera/impala/planner/SortNode.java
deleted file mode 100644
index 02f0df6..0000000
--- a/fe/src/main/java/com/cloudera/impala/planner/SortNode.java
+++ /dev/null
@@ -1,242 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.impala.analysis.Analyzer;
-import com.cloudera.impala.analysis.Expr;
-import com.cloudera.impala.analysis.ExprSubstitutionMap;
-import com.cloudera.impala.analysis.SlotDescriptor;
-import com.cloudera.impala.analysis.SlotRef;
-import com.cloudera.impala.analysis.SortInfo;
-import com.cloudera.impala.common.InternalException;
-import com.cloudera.impala.service.BackendConfig;
-import com.cloudera.impala.thrift.TExplainLevel;
-import com.cloudera.impala.thrift.TPlanNode;
-import com.cloudera.impala.thrift.TPlanNodeType;
-import com.cloudera.impala.thrift.TQueryOptions;
-import com.cloudera.impala.thrift.TSortInfo;
-import com.cloudera.impala.thrift.TSortNode;
-import com.google.common.base.Joiner;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * Node that implements a sort with or without a limit. useTopN_ is true for sorts
- * with limits that are implemented by a TopNNode in the backend. SortNode is used
- * otherwise.
- * Will always materialize the new tuple info_.sortTupleDesc_.
- */
-public class SortNode extends PlanNode {
- private final static Logger LOG = LoggerFactory.getLogger(SortNode.class);
-
- private final SortInfo info_;
-
- // if set, this SortNode requires its input to have this data partition
- private DataPartition inputPartition_;
-
- // if true, the output of this node feeds an AnalyticNode
- private boolean isAnalyticSort_;
-
- // info_.sortTupleSlotExprs_ substituted with the outputSmap_ for materialized slots
- // in init().
- private List<Expr> resolvedTupleExprs_;
- private final boolean useTopN_;
- // The offset of the first row to return.
- protected long offset_;
-
- public SortNode(PlanNodeId id, PlanNode input, SortInfo info, boolean useTopN,
- long offset) {
- super(id, info.getSortTupleDescriptor().getId().asList(),
- getDisplayName(useTopN, false));
- info_ = info;
- useTopN_ = useTopN;
- children_.add(input);
- offset_ = offset;
- }
-
- public long getOffset() { return offset_; }
- public void setOffset(long offset) { offset_ = offset; }
- public boolean hasOffset() { return offset_ > 0; }
- public boolean useTopN() { return useTopN_; }
- public SortInfo getSortInfo() { return info_; }
- public void setInputPartition(DataPartition inputPartition) {
- inputPartition_ = inputPartition;
- }
- public DataPartition getInputPartition() { return inputPartition_; }
- public boolean isAnalyticSort() { return isAnalyticSort_; }
- public void setIsAnalyticSort(boolean v) { isAnalyticSort_ = v; }
-
- @Override
- public boolean isBlockingNode() { return true; }
-
- @Override
- public void init(Analyzer analyzer) throws InternalException {
- // Do not assignConjuncts() here, so that conjuncts bound by this SortNode's tuple id
- // can be placed in a downstream SelectNode. A SortNode cannot evaluate conjuncts.
- Preconditions.checkState(conjuncts_.isEmpty());
- // Compute the memory layout for the generated tuple.
- computeMemLayout(analyzer);
- computeStats(analyzer);
-
- // populate resolvedTupleExprs_ and outputSmap_
- List<SlotDescriptor> sortTupleSlots = info_.getSortTupleDescriptor().getSlots();
- List<Expr> slotExprs = info_.getSortTupleSlotExprs();
- Preconditions.checkState(sortTupleSlots.size() == slotExprs.size());
- resolvedTupleExprs_ = Lists.newArrayList();
- outputSmap_ = new ExprSubstitutionMap();
- for (int i = 0; i < slotExprs.size(); ++i) {
- if (!sortTupleSlots.get(i).isMaterialized()) continue;
- resolvedTupleExprs_.add(slotExprs.get(i));
- outputSmap_.put(slotExprs.get(i), new SlotRef(sortTupleSlots.get(i)));
- }
- ExprSubstitutionMap childSmap = getCombinedChildSmap();
- resolvedTupleExprs_ =
- Expr.substituteList(resolvedTupleExprs_, childSmap, analyzer, false);
-
- // Remap the ordering exprs to the tuple materialized by this sort node. The mapping
- // is a composition of the childSmap and the outputSmap_ because the child node may
- // have also remapped its input (e.g., as in a a series of (sort->analytic)* nodes).
- // Parent nodes have have to do the same so set the composition as the outputSmap_.
- outputSmap_ = ExprSubstitutionMap.compose(childSmap, outputSmap_, analyzer);
-
- info_.substituteOrderingExprs(outputSmap_, analyzer);
- info_.checkConsistency();
-
- LOG.trace("sort id " + tupleIds_.get(0).toString() + " smap: "
- + outputSmap_.debugString());
- LOG.trace("sort input exprs: " + Expr.debugString(resolvedTupleExprs_));
- }
-
- @Override
- protected void computeStats(Analyzer analyzer) {
- super.computeStats(analyzer);
- cardinality_ = capAtLimit(getChild(0).cardinality_);
- LOG.debug("stats Sort: cardinality=" + Long.toString(cardinality_));
- }
-
- @Override
- protected String debugString() {
- List<String> strings = Lists.newArrayList();
- for (Boolean isAsc : info_.getIsAscOrder()) {
- strings.add(isAsc ? "a" : "d");
- }
- return Objects.toStringHelper(this)
- .add("ordering_exprs", Expr.debugString(info_.getOrderingExprs()))
- .add("is_asc", "[" + Joiner.on(" ").join(strings) + "]")
- .add("nulls_first", "[" + Joiner.on(" ").join(info_.getNullsFirst()) + "]")
- .add("offset_", offset_)
- .addValue(super.debugString())
- .toString();
- }
-
- @Override
- protected void toThrift(TPlanNode msg) {
- msg.node_type = TPlanNodeType.SORT_NODE;
- TSortInfo sort_info = new TSortInfo(Expr.treesToThrift(info_.getOrderingExprs()),
- info_.getIsAscOrder(), info_.getNullsFirst());
- Preconditions.checkState(tupleIds_.size() == 1,
- "Incorrect size for tupleIds_ in SortNode");
- sort_info.sort_tuple_slot_exprs = Expr.treesToThrift(resolvedTupleExprs_);
- TSortNode sort_node = new TSortNode(sort_info, useTopN_);
- sort_node.setOffset(offset_);
- msg.sort_node = sort_node;
- }
-
- @Override
- protected String getNodeExplainString(String prefix, String detailPrefix,
- TExplainLevel detailLevel) {
- StringBuilder output = new StringBuilder();
- output.append(String.format("%s%s:%s%s\n", prefix, id_.toString(),
- displayName_, getNodeExplainDetail(detailLevel)));
- if (detailLevel.ordinal() >= TExplainLevel.STANDARD.ordinal()) {
- output.append(detailPrefix + "order by: ");
- for (int i = 0; i < info_.getOrderingExprs().size(); ++i) {
- if (i > 0) output.append(", ");
- output.append(info_.getOrderingExprs().get(i).toSql() + " ");
- output.append(info_.getIsAscOrder().get(i) ? "ASC" : "DESC");
-
- Boolean nullsFirstParam = info_.getNullsFirstParams().get(i);
- if (nullsFirstParam != null) {
- output.append(nullsFirstParam ? " NULLS FIRST" : " NULLS LAST");
- }
- }
- output.append("\n");
- }
- return output.toString();
- }
-
- private String getNodeExplainDetail(TExplainLevel detailLevel) {
- if (!hasLimit()) return "";
- if (hasOffset()) {
- return String.format(" [LIMIT=%s OFFSET=%s]", limit_, offset_);
- } else {
- return String.format(" [LIMIT=%s]", limit_);
- }
- }
-
- @Override
- protected String getOffsetExplainString(String prefix) {
- return offset_ != 0 ? prefix + "offset: " + Long.toString(offset_) + "\n" : "";
- }
-
- @Override
- public void computeCosts(TQueryOptions queryOptions) {
- Preconditions.checkState(hasValidStats());
- if (useTopN_) {
- perHostMemCost_ = (long) Math.ceil((cardinality_ + offset_) * avgRowSize_);
- return;
- }
-
- // For an external sort, set the memory cost to be what is required for a 2-phase
- // sort. If the input to be sorted would take up N blocks in memory, then the
- // memory required for a 2-phase sort is sqrt(N) blocks. A single run would be of
- // size sqrt(N) blocks, and we could merge sqrt(N) such runs with sqrt(N) blocks
- // of memory.
- double fullInputSize = getChild(0).cardinality_ * avgRowSize_;
- boolean hasVarLenSlots = false;
- for (SlotDescriptor slotDesc: info_.getSortTupleDescriptor().getSlots()) {
- if (slotDesc.isMaterialized() && !slotDesc.getType().isFixedLengthType()) {
- hasVarLenSlots = true;
- break;
- }
- }
-
- // The block size used by the sorter is the same as the configured I/O read size.
- long blockSize = BackendConfig.INSTANCE.getReadSize();
- // The external sorter writes fixed-len and var-len data in separate sequences of
- // blocks on disk and reads from both sequences when merging. This effectively
- // doubles the block size when there are var-len columns present.
- if (hasVarLenSlots) blockSize *= 2;
- double numInputBlocks = Math.ceil(fullInputSize / blockSize);
- perHostMemCost_ = blockSize * (long) Math.ceil(Math.sqrt(numInputBlocks));
- }
-
- private static String getDisplayName(boolean isTopN, boolean isMergeOnly) {
- if (isTopN) {
- return "TOP-N";
- } else {
- return "SORT";
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/planner/SubplanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/SubplanNode.java b/fe/src/main/java/com/cloudera/impala/planner/SubplanNode.java
deleted file mode 100644
index 4512268..0000000
--- a/fe/src/main/java/com/cloudera/impala/planner/SubplanNode.java
+++ /dev/null
@@ -1,109 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import com.cloudera.impala.analysis.Analyzer;
-import com.cloudera.impala.common.InternalException;
-import com.cloudera.impala.thrift.TExplainLevel;
-import com.cloudera.impala.thrift.TPlanNode;
-import com.cloudera.impala.thrift.TPlanNodeType;
-import com.google.common.base.Preconditions;
-
-/**
- * A SubplanNode evaluates its right child plan tree for every row from its left child,
- * and returns those rows produced by the right child. The right child is called the
- * 'subplan tree' and the left child the 'input'. A SubplanNode is similar to a join,
- * but different in the following respects. First, a SubplanNode does not do any real
- * work itself. It only returns rows produced by the right child plan tree, which
- * typically has a dependency on the current input row (see SingularRowSrcNode and
- * UnnestNode). Second, no join predicates are required. A SubplanNode does not
- * evaluate any conjuncts.
- */
-public class SubplanNode extends PlanNode {
- private PlanNode subplan_;
-
- public SubplanNode(PlanNode input) {
- super("SUBPLAN");
- children_.add(input);
- }
-
- /**
- * Sets the subplan of this SubplanNode. Dependent plan nodes such as UnnestNodes
- * and SingularRowSrcNodes need to know their SubplanNode parent, therefore, setting
- * the subplan in this SubplanNode is deferred until the subplan tree has been
- * constructed (which requires the parent SubplanNode to have been constructed).
- */
- public void setSubplan(PlanNode subplan) {
- Preconditions.checkState(children_.size() == 1);
- subplan_ = subplan;
- children_.add(subplan);
- computeTupleIds();
- }
-
- @Override
- public void computeTupleIds() {
- Preconditions.checkNotNull(subplan_);
- clearTupleIds();
- tblRefIds_.addAll(subplan_.getTblRefIds());
- tupleIds_.addAll(subplan_.getTupleIds());
- nullableTupleIds_.addAll(subplan_.getNullableTupleIds());
- }
-
- @Override
- public void init(Analyzer analyzer) throws InternalException {
- // Subplan root must have been set.
- Preconditions.checkState(children_.size() == 2);
- // Check that there are no unassigned conjuncts that can be evaluated by this node.
- // All such conjuncts should have already been assigned in the right child.
- assignConjuncts(analyzer);
- Preconditions.checkState(conjuncts_.isEmpty());
- computeStats(analyzer);
- outputSmap_ = getChild(1).getOutputSmap();
- // Save state of assigned conjuncts for join-ordering attempts (see member comment).
- assignedConjuncts_ = analyzer.getAssignedConjuncts();
- }
-
- @Override
- protected void computeStats(Analyzer analyzer) {
- super.computeStats(analyzer);
- if (getChild(0).cardinality_ != -1 && getChild(1).cardinality_ != -1) {
- cardinality_ =
- multiplyCardinalities(getChild(0).cardinality_, getChild(1).cardinality_);
- } else {
- cardinality_ = -1;
- }
- cardinality_ = capAtLimit(cardinality_);
- }
-
- @Override
- protected String getNodeExplainString(String prefix, String detailPrefix,
- TExplainLevel detailLevel) {
- StringBuilder output = new StringBuilder();
- output.append(String.format("%s%s\n", prefix, getDisplayLabel()));
- if (detailLevel.ordinal() >= TExplainLevel.STANDARD.ordinal()) {
- if (!conjuncts_.isEmpty()) {
- output.append(detailPrefix + "predicates: " +
- getExplainString(conjuncts_) + "\n");
- }
- }
- return output.toString();
- }
-
- @Override
- protected void toThrift(TPlanNode msg) { msg.node_type = TPlanNodeType.SUBPLAN_NODE; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/planner/TableSink.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/TableSink.java b/fe/src/main/java/com/cloudera/impala/planner/TableSink.java
deleted file mode 100644
index ae73b54..0000000
--- a/fe/src/main/java/com/cloudera/impala/planner/TableSink.java
+++ /dev/null
@@ -1,114 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import com.cloudera.impala.analysis.Expr;
-import com.cloudera.impala.catalog.HBaseTable;
-import com.cloudera.impala.catalog.HdfsTable;
-import com.cloudera.impala.catalog.KuduTable;
-import com.cloudera.impala.catalog.Table;
-import com.cloudera.impala.thrift.TSinkAction;
-import com.google.common.base.Preconditions;
-
-import java.util.List;
-
-/**
- * A DataSink that writes into a table.
- *
- */
-public abstract class TableSink extends DataSink {
-
- /**
- * Enum to specify the sink operation type.
- */
- public enum Op {
- INSERT {
- @Override
- public String toExplainString() { return "INSERT INTO"; }
-
- @Override
- public TSinkAction toThrift() { return TSinkAction.INSERT; }
- },
- UPDATE {
- @Override
- public String toExplainString() { return "UPDATE"; }
-
- @Override
- public TSinkAction toThrift() { return TSinkAction.UPDATE; }
- },
- DELETE {
- @Override
- public String toExplainString() { return "DELETE FROM"; }
-
- @Override
- public TSinkAction toThrift() { return TSinkAction.DELETE; }
- };
-
- public abstract String toExplainString();
-
- public abstract TSinkAction toThrift();
- }
-
- // Table which is to be populated by this sink.
- protected final Table targetTable_;
- // The type of operation to be performed by this sink.
- protected final Op sinkOp_;
-
- public TableSink(Table targetTable, Op sinkAction) {
- targetTable_ = targetTable;
- sinkOp_ = sinkAction;
- }
-
- /**
- * Returns an output sink appropriate for writing to the given table.
- * Not all Ops are supported for all tables.
- * All parameters must be non-null, the lists in particular need to be empty if they
- * don't make sense for a certain table type.
- */
- public static TableSink create(Table table, Op sinkAction,
- List<Expr> partitionKeyExprs, List<Integer> referencedColumns,
- boolean overwrite, boolean ignoreDuplicates) {
- if (table instanceof HdfsTable) {
- // Hdfs only supports inserts.
- Preconditions.checkState(sinkAction == Op.INSERT);
- // Referenced columns don't make sense for an Hdfs table.
- Preconditions.checkState(referencedColumns.isEmpty());
- return new HdfsTableSink(table, partitionKeyExprs, overwrite);
- } else if (table instanceof HBaseTable) {
- // HBase only supports inserts.
- Preconditions.checkState(sinkAction == Op.INSERT);
- // Partition clause doesn't make sense for an HBase table.
- Preconditions.checkState(partitionKeyExprs.isEmpty());
- // HBase doesn't have a way to perform INSERT OVERWRITE
- Preconditions.checkState(overwrite == false);
- // Referenced columns don't make sense for an HBase table.
- Preconditions.checkState(referencedColumns.isEmpty());
- // Create the HBaseTableSink and return it.
- return new HBaseTableSink(table);
- } else if (table instanceof KuduTable) {
- // Kudu doesn't have a way to perform INSERT OVERWRITE.
- Preconditions.checkState(overwrite == false);
- // Partition clauses don't make sense for Kudu inserts.
- Preconditions.checkState(partitionKeyExprs.isEmpty());
- return new KuduTableSink(table, sinkAction, referencedColumns, ignoreDuplicates);
- } else {
- throw new UnsupportedOperationException(
- "Cannot create data sink into table of type: " + table.getClass().getName());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/planner/UnionNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/UnionNode.java b/fe/src/main/java/com/cloudera/impala/planner/UnionNode.java
deleted file mode 100644
index ef67277..0000000
--- a/fe/src/main/java/com/cloudera/impala/planner/UnionNode.java
+++ /dev/null
@@ -1,231 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.impala.analysis.Analyzer;
-import com.cloudera.impala.analysis.Expr;
-import com.cloudera.impala.analysis.SlotDescriptor;
-import com.cloudera.impala.analysis.TupleId;
-import com.cloudera.impala.common.Pair;
-import com.cloudera.impala.thrift.TExplainLevel;
-import com.cloudera.impala.thrift.TExpr;
-import com.cloudera.impala.thrift.TPlanNode;
-import com.cloudera.impala.thrift.TPlanNodeType;
-import com.cloudera.impala.thrift.TUnionNode;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * Node that merges the results of its child plans by materializing
- * the corresponding result exprs into a new tuple.
- */
-public class UnionNode extends PlanNode {
- private final static Logger LOG = LoggerFactory.getLogger(UnionNode.class);
-
- // Expr lists corresponding to the input query stmts.
- // The ith resultExprList belongs to the ith child.
- // All exprs are resolved to base tables.
- protected List<List<Expr>> resultExprLists_ = Lists.newArrayList();
-
- // Expr lists that originate from constant select stmts.
- // We keep them separate from the regular expr lists to avoid null children.
- protected List<List<Expr>> constExprLists_ = Lists.newArrayList();
-
- // Materialized result/const exprs corresponding to materialized slots.
- // Set in init() and substituted against the corresponding child's output smap.
- protected List<List<Expr>> materializedResultExprLists_ = Lists.newArrayList();
- protected List<List<Expr>> materializedConstExprLists_ = Lists.newArrayList();
-
- protected final TupleId tupleId_;
-
- protected UnionNode(PlanNodeId id, TupleId tupleId) {
- super(id, tupleId.asList(), "UNION");
- tupleId_ = tupleId;
- }
-
- public void addConstExprList(List<Expr> exprs) { constExprLists_.add(exprs); }
-
- /**
- * Returns true if this UnionNode has only constant exprs.
- */
- public boolean isConstantUnion() { return resultExprLists_.isEmpty(); }
-
- /**
- * Add a child tree plus its corresponding resolved resultExprs.
- */
- public void addChild(PlanNode node, List<Expr> baseTblResultExprs) {
- super.addChild(node);
- resultExprLists_.add(baseTblResultExprs);
- if (baseTblResultExprs != null) {
- // if we're materializing output, we can only do that into a single
- // output tuple
- Preconditions.checkState(tupleIds_.size() == 1, tupleIds_.size());
- }
- }
-
- @Override
- public void computeStats(Analyzer analyzer) {
- super.computeStats(analyzer);
- cardinality_ = constExprLists_.size();
- for (PlanNode child: children_) {
- // ignore missing child cardinality info in the hope it won't matter enough
- // to change the planning outcome
- if (child.cardinality_ > 0) {
- cardinality_ = addCardinalities(cardinality_, child.cardinality_);
- }
- }
- // The number of nodes of a union node is -1 (invalid) if all the referenced tables
- // are inline views (e.g. select 1 FROM (VALUES(1 x, 1 y)) a FULL OUTER JOIN
- // (VALUES(1 x, 1 y)) b ON (a.x = b.y)). We need to set the correct value.
- if (numNodes_ == -1) numNodes_ = 1;
- cardinality_ = capAtLimit(cardinality_);
- LOG.debug("stats Union: cardinality=" + Long.toString(cardinality_));
- }
-
- /**
- * Re-order the union's operands descending by their estimated per-host memory,
- * such that parent nodes can gauge the peak memory consumption of this MergeNode after
- * opening it during execution (a MergeNode opens its first operand in Open()).
- * Scan nodes are always ordered last because they can dynamically scale down their
- * memory usage, whereas many other nodes cannot (e.g., joins, aggregations).
- * One goal is to decrease the likelihood of a SortNode parent claiming too much
- * memory in its Open(), possibly causing the mem limit to be hit when subsequent
- * union operands are executed.
- * Can only be called on a fragmented plan because this function calls computeCosts()
- * on this node's children.
- * TODO: Come up with a good way of handing memory out to individual operators so that
- * they don't trip each other up. Then remove this function.
- */
- public void reorderOperands(Analyzer analyzer) {
- Preconditions.checkNotNull(fragment_,
- "Operands can only be reordered on the fragmented plan.");
-
- // List of estimated per-host memory consumption (first) by child index (second).
- List<Pair<Long, Integer>> memByChildIdx = Lists.newArrayList();
- for (int i = 0; i < children_.size(); ++i) {
- PlanNode child = children_.get(i);
- child.computeCosts(analyzer.getQueryCtx().request.getQuery_options());
- memByChildIdx.add(new Pair<Long, Integer>(child.getPerHostMemCost(), i));
- }
-
- Collections.sort(memByChildIdx,
- new Comparator<Pair<Long, Integer>>() {
- public int compare(Pair<Long, Integer> a, Pair<Long, Integer> b) {
- PlanNode aNode = children_.get(a.second);
- PlanNode bNode = children_.get(b.second);
- // Order scan nodes last because they can dynamically scale down their mem.
- if (bNode instanceof ScanNode && !(aNode instanceof ScanNode)) return -1;
- if (aNode instanceof ScanNode && !(bNode instanceof ScanNode)) return 1;
- long diff = b.first - a.first;
- return (diff < 0 ? -1 : (diff > 0 ? 1 : 0));
- }
- });
-
- List<List<Expr>> newResultExprLists = Lists.newArrayList();
- ArrayList<PlanNode> newChildren = Lists.newArrayList();
- for (Pair<Long, Integer> p: memByChildIdx) {
- newResultExprLists.add(resultExprLists_.get(p.second));
- newChildren.add(children_.get(p.second));
- }
- resultExprLists_ = newResultExprLists;
- children_ = newChildren;
- }
-
- /**
- * Must be called after addChild()/addConstExprList(). Computes the materialized
- * result/const expr lists based on the materialized slots of this UnionNode's
- * produced tuple. The UnionNode doesn't need an smap: like a ScanNode, it
- * materializes an original tuple.
- * There is no need to call assignConjuncts() because all non-constant conjuncts
- * have already been assigned to the union operands, and all constant conjuncts have
- * been evaluated during registration to set analyzer.hasEmptyResultSet_.
- */
- @Override
- public void init(Analyzer analyzer) {
- Preconditions.checkState(conjuncts_.isEmpty());
- computeMemLayout(analyzer);
- computeStats(analyzer);
-
- // drop resultExprs/constExprs that aren't getting materialized (= where the
- // corresponding output slot isn't being materialized)
- materializedResultExprLists_.clear();
- Preconditions.checkState(resultExprLists_.size() == children_.size());
- List<SlotDescriptor> slots = analyzer.getDescTbl().getTupleDesc(tupleId_).getSlots();
- for (int i = 0; i < resultExprLists_.size(); ++i) {
- List<Expr> exprList = resultExprLists_.get(i);
- List<Expr> newExprList = Lists.newArrayList();
- Preconditions.checkState(exprList.size() == slots.size());
- for (int j = 0; j < exprList.size(); ++j) {
- if (slots.get(j).isMaterialized()) newExprList.add(exprList.get(j));
- }
- materializedResultExprLists_.add(
- Expr.substituteList(newExprList, getChild(i).getOutputSmap(), analyzer, true));
- }
- Preconditions.checkState(
- materializedResultExprLists_.size() == getChildren().size());
-
- materializedConstExprLists_.clear();
- for (List<Expr> exprList: constExprLists_) {
- Preconditions.checkState(exprList.size() == slots.size());
- List<Expr> newExprList = Lists.newArrayList();
- for (int i = 0; i < exprList.size(); ++i) {
- if (slots.get(i).isMaterialized()) newExprList.add(exprList.get(i));
- }
- materializedConstExprLists_.add(newExprList);
- }
- }
-
- @Override
- protected void toThrift(TPlanNode msg) {
- Preconditions.checkState(materializedResultExprLists_.size() == children_.size());
- List<List<TExpr>> texprLists = Lists.newArrayList();
- for (List<Expr> exprList: materializedResultExprLists_) {
- texprLists.add(Expr.treesToThrift(exprList));
- }
- List<List<TExpr>> constTexprLists = Lists.newArrayList();
- for (List<Expr> constTexprList: materializedConstExprLists_) {
- constTexprLists.add(Expr.treesToThrift(constTexprList));
- }
- msg.union_node = new TUnionNode(tupleId_.asInt(), texprLists, constTexprLists);
- msg.node_type = TPlanNodeType.UNION_NODE;
- }
-
- @Override
- protected String getNodeExplainString(String prefix, String detailPrefix,
- TExplainLevel detailLevel) {
- StringBuilder output = new StringBuilder();
- output.append(String.format("%s%s:%s\n", prefix, id_.toString(), displayName_));
- // A UnionNode may have predicates if a union is used inside an inline view,
- // and the enclosing select stmt has predicates referring to the inline view.
- if (!conjuncts_.isEmpty()) {
- output.append(detailPrefix + "predicates: " + getExplainString(conjuncts_) + "\n");
- }
- if (!constExprLists_.isEmpty()) {
- output.append(detailPrefix + "constant-operands=" + constExprLists_.size() + "\n");
- }
- return output.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/planner/UnnestNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/UnnestNode.java b/fe/src/main/java/com/cloudera/impala/planner/UnnestNode.java
deleted file mode 100644
index 3a45882..0000000
--- a/fe/src/main/java/com/cloudera/impala/planner/UnnestNode.java
+++ /dev/null
@@ -1,107 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import com.cloudera.impala.analysis.Analyzer;
-import com.cloudera.impala.analysis.CollectionTableRef;
-import com.cloudera.impala.analysis.Expr;
-import com.cloudera.impala.common.ImpalaException;
-import com.cloudera.impala.thrift.TExplainLevel;
-import com.cloudera.impala.thrift.TPlanNode;
-import com.cloudera.impala.thrift.TPlanNodeType;
-import com.cloudera.impala.thrift.TUnnestNode;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-
-/**
- * An UnnestNode scans over a collection materialized in memory, and returns
- * one row per item in the collection.
- * An UnnestNode can only appear in the plan tree of a SubplanNode.
- */
-public class UnnestNode extends PlanNode {
- private final SubplanNode containingSubplanNode_;
- private final CollectionTableRef tblRef_;
- private final Expr collectionExpr_;
-
- public UnnestNode(PlanNodeId id, SubplanNode containingSubplanNode,
- CollectionTableRef tblRef) {
- super(id, tblRef.getDesc().getId().asList(), "UNNEST");
- containingSubplanNode_ = containingSubplanNode;
- tblRef_ = tblRef;
- collectionExpr_ = tblRef_.getCollectionExpr();
- // Assume the collection expr has been fully resolved in analysis.
- Preconditions.checkState(
- collectionExpr_.isBoundByTupleIds(containingSubplanNode.getChild(0).tupleIds_));
- }
-
- @Override
- public void init(Analyzer analyzer) throws ImpalaException {
- // Do not assign binding predicates or predicates for enforcing slot equivalences
- // because they must have been assigned in the scan node materializing the
- // collection-typed slot.
- super.init(analyzer);
- conjuncts_ = orderConjunctsByCost(conjuncts_);
-
- // Unnest is like a scan and must materialize the slots of its conjuncts.
- analyzer.materializeSlots(conjuncts_);
- computeMemLayout(analyzer);
- }
-
- @Override
- public void computeStats(Analyzer analyzer) {
- super.computeStats(analyzer);
- cardinality_ = PlannerContext.AVG_COLLECTION_SIZE;
- // The containing SubplanNode has not yet been initialized, so get the number
- // of nodes from the SubplanNode's input.
- numNodes_ = containingSubplanNode_.getChild(0).getNumNodes();
- cardinality_ = capAtLimit(cardinality_);
- }
-
- @Override
- protected String getNodeExplainString(String prefix, String detailPrefix,
- TExplainLevel detailLevel) {
- StringBuilder output = new StringBuilder();
- output.append(String.format("%s%s [%s]\n", prefix, getDisplayLabel(),
- getDisplayLabelDetail()));
- if (detailLevel.ordinal() >= TExplainLevel.EXTENDED.ordinal()) {
- output.append(String.format(
- "%sparent-subplan=%s\n", detailPrefix, containingSubplanNode_.getId()));
- }
- if (detailLevel.ordinal() >= TExplainLevel.STANDARD.ordinal()) {
- if (!conjuncts_.isEmpty()) {
- output.append(
- detailPrefix + "predicates: " + getExplainString(conjuncts_) + "\n");
- }
- }
- return output.toString();
- }
-
- @Override
- protected String getDisplayLabelDetail() {
- StringBuilder strBuilder = new StringBuilder();
- strBuilder.append(Joiner.on(".").join(tblRef_.getPath()));
- if (tblRef_.hasExplicitAlias()) strBuilder.append(" " + tblRef_.getExplicitAlias());
- return strBuilder.toString();
- }
-
- @Override
- protected void toThrift(TPlanNode msg) {
- msg.node_type = TPlanNodeType.UNNEST_NODE;
- msg.setUnnest_node(new TUnnestNode(collectionExpr_.treeToThrift()));
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/planner/ValueRange.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/planner/ValueRange.java b/fe/src/main/java/com/cloudera/impala/planner/ValueRange.java
deleted file mode 100644
index e76da9d..0000000
--- a/fe/src/main/java/com/cloudera/impala/planner/ValueRange.java
+++ /dev/null
@@ -1,121 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.planner;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.cloudera.impala.analysis.Analyzer;
-import com.cloudera.impala.analysis.BinaryPredicate;
-import com.cloudera.impala.analysis.CompoundPredicate;
-import com.cloudera.impala.analysis.Expr;
-import com.cloudera.impala.analysis.Predicate;
-import com.cloudera.impala.common.AnalysisException;
-import com.cloudera.impala.common.InternalException;
-import com.cloudera.impala.service.FeSupport;
-import com.google.common.base.Preconditions;
-
-/**
- * Representation of a two-sided interval of values. Either one of the sides
- * is optional, and can be exclusive or inclusive. For a range representing a single
- * value, both bounds are set.
- */
-public class ValueRange {
- private final static Logger LOG = LoggerFactory.getLogger(ValueRange.class);
-
- private Expr lowerBound_;
- private boolean lowerBoundInclusive_;
- private Expr upperBound_;
- private boolean upperBoundInclusive_;
-
- Expr getLowerBound() { return lowerBound_; }
- void setLowerBound(Expr e) { lowerBound_ = e; }
- boolean getLowerBoundInclusive() { return lowerBoundInclusive_; }
- void setLowerBoundInclusive(boolean b) { lowerBoundInclusive_ = b; }
- Expr getUpperBound() { return upperBound_; }
- void setUpperBound(Expr e) { upperBound_ = e; }
- boolean getUpperBoundInclusive() { return upperBoundInclusive_; }
- void setUpperBoundInclusive(boolean b) { upperBoundInclusive_ = b; }
-
- static public ValueRange createEqRange(Expr valueExpr) {
- ValueRange result = new ValueRange();
- result.lowerBound_ = valueExpr;
- result.lowerBoundInclusive_ = true;
- result.upperBound_ = valueExpr;
- result.upperBoundInclusive_ = true;
- return result;
- }
-
- public boolean isEqRange() {
- return lowerBound_ == upperBound_ && lowerBoundInclusive_ && upperBoundInclusive_;
- }
-
- /**
- * Determines whether a given constant expr is within the range.
- * Does this by constructing predicate that represents the range,
- * with the valueExpr inserted appropriately, and then calls the
- * backend for evaluation.
- */
- public boolean isInRange(Analyzer analyzer, Expr valueExpr) throws
- InternalException {
- Preconditions.checkState(valueExpr.isConstant());
- Preconditions.checkState(lowerBound_ != null || upperBound_ != null);
-
- // construct predicate
- Predicate p = null;
- if (lowerBound_ != null && upperBound_ != null
- && lowerBoundInclusive_ && upperBoundInclusive_
- && lowerBound_ == upperBound_) {
- // construct "=" predicate
- p = new BinaryPredicate(BinaryPredicate.Operator.EQ, valueExpr, lowerBound_);
- } else {
- // construct range predicate
- if (lowerBound_ != null) {
- p = new BinaryPredicate(
- lowerBoundInclusive_
- ? BinaryPredicate.Operator.GE : BinaryPredicate.Operator.GT,
- valueExpr, lowerBound_);
- }
- if (upperBound_ != null) {
- Predicate p2 = new BinaryPredicate(
- upperBoundInclusive_
- ? BinaryPredicate.Operator.GE : BinaryPredicate.Operator.GT,
- upperBound_, valueExpr);
- if (p != null) {
- p = new CompoundPredicate(CompoundPredicate.Operator.AND, p, p2);
- } else {
- p = p2;
- }
- }
- }
-
- Preconditions.checkState(p.isConstant());
- // analyze to insert casts, etc.
- try {
- p.analyze(analyzer);
- } catch (AnalysisException e) {
- // this should never happen
- throw new InternalException(
- "couldn't analyze predicate " + p.toSql() + "\n" + e.toString());
- }
-
- // call backend
- return FeSupport.EvalPredicate(p, analyzer.getQueryCtx());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b544f019/fe/src/main/java/com/cloudera/impala/service/BackendConfig.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/com/cloudera/impala/service/BackendConfig.java b/fe/src/main/java/com/cloudera/impala/service/BackendConfig.java
deleted file mode 100644
index e98ab571..0000000
--- a/fe/src/main/java/com/cloudera/impala/service/BackendConfig.java
+++ /dev/null
@@ -1,49 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package com.cloudera.impala.service;
-
-/**
- * This class is meant to provide the FE with impalad backend configuration parameters,
- * including command line arguments.
- * TODO: Remove this class and either
- * a) Figure out if there's a standard way to access flags from java
- * b) Create a util/gflags.java that let's us access the be flags
- */
-public class BackendConfig {
- public static BackendConfig INSTANCE = new BackendConfig();
-
- // Default read block size (in bytes). This is the same as
- // the default FLAGS_read_size used by the IO manager in the backend.
- private final long READ_SIZE;
-
- // This is overriden by JniFrontend/JniCatalog classes with user set configuration.
- // TODO: Read this from backend instead of using static variables.
- private static boolean allowAuthToLocalRules_ = false;
-
- private BackendConfig() {
- // TODO: Populate these by making calls to the backend instead of default constants.
- READ_SIZE = 8 * 1024 * 1024L;
- }
-
- public long getReadSize() { return READ_SIZE; }
-
- public static boolean isAuthToLocalEnabled() { return allowAuthToLocalRules_; }
- public static void setAuthToLocal(boolean authToLocal) {
- allowAuthToLocalRules_ = authToLocal;
- }
-}