You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/03/22 14:44:56 UTC
[2/2] flink git commit: [FLINK-3179] [dataSet][optimizer] Log a WARN
message if combiner is not added in front of PartitionOperator
[FLINK-3179] [dataSet][optimizer] Log a WARN message if combiner is not added in front of PartitionOperator
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/cd5773ed
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/cd5773ed
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/cd5773ed
Branch: refs/heads/release-1.0
Commit: cd5773ed302cc3c287b00cef6440cc2d740fc469
Parents: 0880c59
Author: ramkrishna <ra...@gmail.com>
Authored: Mon Mar 21 17:03:42 2016 +0530
Committer: Fabian Hueske <fh...@apache.org>
Committed: Tue Mar 22 13:58:04 2016 +0100
----------------------------------------------------------------------
.../optimizer/operators/GroupReduceWithCombineProperties.java | 7 +++++++
.../apache/flink/optimizer/operators/ReduceProperties.java | 7 +++++++
2 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/cd5773ed/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
index a4f295d..888b670 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/GroupReduceWithCombineProperties.java
@@ -27,6 +27,7 @@ import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.util.FieldSet;
import org.apache.flink.optimizer.costs.Costs;
import org.apache.flink.optimizer.dag.GroupReduceNode;
+import org.apache.flink.optimizer.dag.PartitionNode;
import org.apache.flink.optimizer.dag.SingleInputNode;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.LocalProperties;
@@ -39,8 +40,11 @@ import org.apache.flink.runtime.io.network.DataExchangeMode;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public final class GroupReduceWithCombineProperties extends OperatorDescriptorSingle {
+ private static final Logger LOG = LoggerFactory.getLogger(GroupReduceWithCombineProperties.class);
private final Ordering ordering; // ordering that we need to use if an additional ordering is requested
@@ -90,6 +94,9 @@ public final class GroupReduceWithCombineProperties extends OperatorDescriptorSi
@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
+ if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
+ LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
+ }
// adjust a sort (changes grouping, so it must be for this driver to combining sort
if (in.getLocalStrategy() == LocalStrategy.SORT) {
if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
http://git-wip-us.apache.org/repos/asf/flink/blob/cd5773ed/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
index 81afe1e..78007fe 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.flink.api.common.functions.Partitioner;
import org.apache.flink.api.common.operators.util.FieldSet;
import org.apache.flink.optimizer.costs.Costs;
+import org.apache.flink.optimizer.dag.PartitionNode;
import org.apache.flink.optimizer.dag.ReduceNode;
import org.apache.flink.optimizer.dag.SingleInputNode;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
@@ -37,8 +38,11 @@ import org.apache.flink.runtime.io.network.DataExchangeMode;
import org.apache.flink.runtime.operators.DriverStrategy;
import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
import org.apache.flink.runtime.operators.util.LocalStrategy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public final class ReduceProperties extends OperatorDescriptorSingle {
+ private static final Logger LOG = LoggerFactory.getLogger(ReduceProperties.class);
private final Partitioner<?> customPartitioner;
@@ -61,6 +65,9 @@ public final class ReduceProperties extends OperatorDescriptorSingle {
if (in.getShipStrategy() == ShipStrategyType.FORWARD ||
(node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty()))
{
+ if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
+ LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
+ }
return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
DriverStrategy.SORTED_REDUCE, this.keyList);
}