You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/04/21 01:37:54 UTC
[1/3] flink git commit: [FLINK-3665] [dataSet] Add support for sort
orders in range partitioning.
Repository: flink
Updated Branches:
refs/heads/master c33325e3f -> a43bade0d
[FLINK-3665] [dataSet] Add support for sort orders in range partitioning.
This closes #1848
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d8fb2308
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d8fb2308
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d8fb2308
Branch: refs/heads/master
Commit: d8fb23085839a514fbd0b56cc681dd52aac503ac
Parents: c33325e
Author: dawid <da...@gmail.com>
Authored: Sat Apr 2 13:10:59 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Apr 20 22:52:28 2016 +0200
----------------------------------------------------------------------
.../apache/flink/api/common/operators/Keys.java | 43 +-
.../operators/base/PartitionOperatorBase.java | 15 +-
.../common/operators/ExpressionKeysTest.java | 75 ++++
.../operators/SelectorFunctionKeysTest.java | 39 ++
.../api/java/operators/PartitionOperator.java | 55 ++-
.../java/operator/PartitionOperatorTest.java | 407 +++++++++++++++++++
.../flink/optimizer/dag/PartitionNode.java | 24 +-
.../test/javaApiOperators/PartitionITCase.java | 284 ++++++++++++-
8 files changed, 905 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
index 0b771e8..6085ebf 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
@@ -32,6 +32,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor;
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Pattern;
import static org.apache.flink.util.Preconditions.checkArgument;
import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -45,6 +46,8 @@ public abstract class Keys<T> {
public abstract TypeInformation<?>[] getKeyFieldTypes();
+ public abstract TypeInformation<?>[] getOriginalKeyFieldTypes();
+
public abstract <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo);
public boolean isEmpty() {
@@ -82,6 +85,7 @@ public abstract class Keys<T> {
private final TypeInformation<T> inputType;
private final TypeInformation<K> keyType;
private final List<FlatFieldDescriptor> keyFields;
+ private final TypeInformation<?>[] originalKeyTypes;
public SelectorFunctionKeys(KeySelector<T, K> keyExtractor, TypeInformation<T> inputType, TypeInformation<K> keyType) {
@@ -99,6 +103,7 @@ public abstract class Keys<T> {
this.inputType = inputType;
this.keyType = keyType;
+ this.originalKeyTypes = new TypeInformation[] {keyType};
if (keyType instanceof CompositeType) {
this.keyFields = ((CompositeType<T>)keyType).getFlatFields(ExpressionKeys.SELECT_ALL_CHAR);
}
@@ -142,7 +147,12 @@ public abstract class Keys<T> {
}
return fieldTypes;
}
-
+
+ @Override
+ public TypeInformation<?>[] getOriginalKeyFieldTypes() {
+ return originalKeyTypes;
+ }
+
@Override
public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) {
@@ -184,9 +194,13 @@ public abstract class Keys<T> {
public static final String SELECT_ALL_CHAR = "*";
public static final String SELECT_ALL_CHAR_SCALA = "_";
-
+ private static final Pattern WILD_CARD_REGEX = Pattern.compile("[\\.]?("
+ + "\\" + SELECT_ALL_CHAR + "|"
+ + "\\" + SELECT_ALL_CHAR_SCALA +")$");
+
// Flattened fields representing keys fields
private List<FlatFieldDescriptor> keyFields;
+ private TypeInformation<?>[] originalKeyTypes;
/**
* ExpressionKeys that is defined by the full data type.
@@ -233,7 +247,7 @@ public abstract class Keys<T> {
} else {
rangeCheckFields(keyPositions, type.getArity() - 1);
}
-
+
checkArgument(keyPositions.length > 0, "Grouping fields can not be empty at this point");
// extract key field types
@@ -242,10 +256,13 @@ public abstract class Keys<T> {
// for each key position, find all (nested) field types
String[] fieldNames = cType.getFieldNames();
+ this.originalKeyTypes = new TypeInformation<?>[keyPositions.length];
ArrayList<FlatFieldDescriptor> tmpList = new ArrayList<>();
- for (int keyPos : keyPositions) {
+ for (int i = 0; i < keyPositions.length; i++) {
+ int keyPos = keyPositions[i];
tmpList.clear();
// get all flat fields
+ this.originalKeyTypes[i] = cType.getTypeAt(keyPos);
cType.getFlatFields(fieldNames[keyPos], 0, tmpList);
// check if fields are of key type
for(FlatFieldDescriptor ffd : tmpList) {
@@ -274,9 +291,12 @@ public abstract class Keys<T> {
if (type instanceof CompositeType){
CompositeType<T> cType = (CompositeType<T>) type;
+ this.originalKeyTypes = new TypeInformation<?>[keyExpressions.length];
// extract the keys on their flat position
- for (String keyExpr : keyExpressions) {
+ for (int i = 0; i < keyExpressions.length; i++) {
+ String keyExpr = keyExpressions[i];
+
if (keyExpr == null) {
throw new InvalidProgramException("Expression key may not be null.");
}
@@ -296,6 +316,13 @@ public abstract class Keys<T> {
}
// add flat fields to key fields
keyFields.addAll(flatFields);
+
+ String strippedKeyExpr = WILD_CARD_REGEX.matcher(keyExpr).replaceAll("");
+ if (strippedKeyExpr.isEmpty()) {
+ this.originalKeyTypes[i] = type;
+ } else {
+ this.originalKeyTypes[i] = cType.getTypeAt(strippedKeyExpr);
+ }
}
}
else {
@@ -318,6 +345,7 @@ public abstract class Keys<T> {
// add full type as key
keyFields.add(new FlatFieldDescriptor(0, type));
}
+ this.originalKeyTypes = new TypeInformation[] {type};
}
}
@@ -348,6 +376,11 @@ public abstract class Keys<T> {
}
@Override
+ public TypeInformation<?>[] getOriginalKeyFieldTypes() {
+ return originalKeyTypes;
+ }
+
+ @Override
public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) {
if (keyFields.size() != 1) {
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
index 4b802aa..1afa45a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.Partitioner;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.functions.util.NoOpFunction;
+import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.SingleInputOperator;
import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
@@ -51,13 +52,15 @@ public class PartitionOperatorBase<IN> extends SingleInputOperator<IN, IN, NoOpF
private Partitioner<?> customPartitioner;
private DataDistribution distribution;
+
+ private Ordering ordering;
public PartitionOperatorBase(UnaryOperatorInformation<IN, IN> operatorInfo, PartitionMethod pMethod, int[] keys, String name) {
super(new UserCodeObjectWrapper<NoOpFunction>(new NoOpFunction()), operatorInfo, keys, name);
this.partitionMethod = pMethod;
}
-
+
public PartitionOperatorBase(UnaryOperatorInformation<IN, IN> operatorInfo, PartitionMethod pMethod, String name) {
super(new UserCodeObjectWrapper<NoOpFunction>(new NoOpFunction()), operatorInfo, name);
this.partitionMethod = pMethod;
@@ -76,7 +79,15 @@ public class PartitionOperatorBase<IN> extends SingleInputOperator<IN, IN, NoOpF
public DataDistribution getDistribution() {
return this.distribution;
}
-
+
+ public void setOrdering(Ordering ordering) {
+ this.ordering = ordering;
+ }
+
+ public Ordering getOrdering() {
+ return ordering;
+ }
+
public void setDistribution(DataDistribution distribution) {
this.distribution = distribution;
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
index 8e65a79..2a6ea48 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
@@ -281,6 +281,81 @@ public class ExpressionKeysTest {
ek = new ExpressionKeys<>("*", ti);
Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6,7}, ek.computeLogicalKeyPositions());
+ ek = new ExpressionKeys<>("f2.p1.*", ti);
+ Assert.assertArrayEquals(new int[] {4,5}, ek.computeLogicalKeyPositions());
+ }
+
+ @Test
+ public void testOriginalTypes() {
+
+ TypeInformation<Tuple3<Integer, Pojo1, PojoWithMultiplePojos>> ti =
+ new TupleTypeInfo<>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ TypeExtractor.getForClass(Pojo1.class),
+ TypeExtractor.getForClass(PojoWithMultiplePojos.class)
+ );
+
+ ExpressionKeys<Tuple3<Integer, Pojo1, PojoWithMultiplePojos>> ek;
+
+ ek = new ExpressionKeys<>(0, ti);
+ Assert.assertArrayEquals(new TypeInformation[] {BasicTypeInfo.INT_TYPE_INFO}, ek.getOriginalKeyFieldTypes());
+
+ ek = new ExpressionKeys<>(1, ti);
+ Assert.assertArrayEquals(new TypeInformation[] {TypeExtractor.getForClass(Pojo1.class)}, ek.getOriginalKeyFieldTypes());
+
+ ek = new ExpressionKeys<>(2, ti);
+ Assert.assertArrayEquals(new TypeInformation[] {TypeExtractor.getForClass(PojoWithMultiplePojos.class)}, ek.getOriginalKeyFieldTypes());
+
+ ek = new ExpressionKeys<>(new int[]{}, ti, true);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { BasicTypeInfo.INT_TYPE_INFO,
+ TypeExtractor.getForClass(Pojo1.class),
+ TypeExtractor.getForClass(PojoWithMultiplePojos.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("*", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { ti },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("f1", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo1.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("f1.*", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo1.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("f2.*", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { TypeExtractor.getForClass(PojoWithMultiplePojos.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("f2.p2", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo2.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("f2.p2.*", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo2.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
+ ek = new ExpressionKeys<>("f2.p2._", ti);
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo2.class) },
+ ek.getOriginalKeyFieldTypes()
+ );
+
}
@Test(expected = InvalidProgramException.class)
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
index ad6fefa..d63e24c 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
@@ -116,6 +116,45 @@ public class SelectorFunctionKeysTest {
Assert.assertTrue(sk2.areCompatible(ek1));
}
+ @Test
+ public void testOriginalTypes1() throws Keys.IncompatibleKeysException {
+ TypeInformation<Tuple2<Integer, String>> t2 = new TupleTypeInfo<>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
+ Keys<Tuple2<Integer, String>> k = new Keys.SelectorFunctionKeys<>(
+ new KeySelector2(),
+ t2,
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { BasicTypeInfo.STRING_TYPE_INFO },
+ k.getOriginalKeyFieldTypes()
+ );
+ }
+
+ @Test
+ public void testOriginalTypes2() throws Exception {
+ final TupleTypeInfo<Tuple2<Integer, String>> t1 = new TupleTypeInfo<>(
+ BasicTypeInfo.INT_TYPE_INFO,
+ BasicTypeInfo.STRING_TYPE_INFO
+ );
+ TypeInformation<PojoWithMultiplePojos> t2 = TypeExtractor.getForClass(PojoWithMultiplePojos.class);
+
+ Keys<PojoWithMultiplePojos> sk = new Keys.SelectorFunctionKeys<>(
+ new KeySelector3(),
+ t2,
+ t1
+ );
+
+ Assert.assertArrayEquals(
+ new TypeInformation<?>[] { t1 },
+ sk.getOriginalKeyFieldTypes()
+ );
+ }
+
@SuppressWarnings("serial")
public static class KeySelector1 implements KeySelector<Pojo2, String> {
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
index dc4a018..fb6d579 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
@@ -18,14 +18,16 @@
package org.apache.flink.api.java.operators;
-import com.google.common.base.Preconditions;
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.Partitioner;
import org.apache.flink.api.common.operators.Keys;
import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.UnaryOperatorInformation;
import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod;
@@ -33,6 +35,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys;
import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Preconditions;
import java.util.Arrays;
@@ -49,6 +52,7 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
private final String partitionLocationName;
private final Partitioner<?> customPartitioner;
private final DataDistribution distribution;
+ private Order[] orders;
public PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys, String partitionLocationName) {
@@ -98,6 +102,24 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
this.customPartitioner = customPartitioner;
this.distribution = distribution;
}
+
+ /**
+ * Sets the order of keys for range partitioning.
+ * NOTE: Only valid for {@link PartitionMethod.RANGE}.
+ *
+ * @param orders array of orders for each specified partition key
+ * @return The partitioneOperator with properly set orders for given keys
+ */
+ @PublicEvolving
+ public PartitionOperator<T> withOrders(Order... orders) {
+ Preconditions.checkState(pMethod == PartitionMethod.RANGE, "Orders cannot be applied for %s partition " +
+ "method", pMethod);
+ Preconditions.checkArgument(pKeys.getOriginalKeyFieldTypes().length == orders.length, "The number of key " +
+ "fields and orders should be the same.");
+ this.orders = orders;
+
+ return this;
+ }
// --------------------------------------------------------------------------------------------
// Properties
@@ -142,6 +164,7 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
partitionedInput.setParallelism(getParallelism());
partitionedInput.setDistribution(distribution);
partitionedInput.setCustomPartitioner(customPartitioner);
+ partitionedInput.setOrdering(computeOrdering(pKeys, orders));
return partitionedInput;
}
@@ -149,7 +172,8 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
@SuppressWarnings("unchecked")
Keys.SelectorFunctionKeys<T, ?> selectorKeys = (Keys.SelectorFunctionKeys<T, ?>) pKeys;
- return translateSelectorFunctionPartitioner(selectorKeys, pMethod, name, input, getParallelism(), customPartitioner);
+ return translateSelectorFunctionPartitioner(selectorKeys, pMethod, name, input, getParallelism(),
+ customPartitioner, orders);
}
else {
throw new UnsupportedOperationException("Unrecognized key type.");
@@ -161,6 +185,29 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
}
}
+ private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) {
+ Ordering ordering = new Ordering();
+ final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions();
+
+ if (orders == null) {
+ for (int key : logicalKeyPositions) {
+ ordering.appendOrdering(key, null, Order.ASCENDING);
+ }
+ } else {
+ final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes();
+ int index = 0;
+ for (int i = 0; i < originalKeyFieldTypes.length; i++) {
+ final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields();
+ for (int j = index; j < index + typeTotalFields; j++) {
+ ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]);
+ }
+ index += typeTotalFields;
+ }
+ }
+
+ return ordering;
+ }
+
@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
SelectorFunctionKeys<T, ?> rawKeys,
@@ -168,7 +215,8 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
String name,
Operator<T> input,
int partitionDop,
- Partitioner<?> customPartitioner)
+ Partitioner<?> customPartitioner,
+ Order[] orders)
{
final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
@@ -180,6 +228,7 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
keyedPartitionedInput.setInput(keyedInput);
keyedPartitionedInput.setCustomPartitioner(customPartitioner);
keyedPartitionedInput.setParallelism(partitionDop);
+ keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));
return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java
new file mode 100644
index 0000000..34f72a2
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.operator;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.DataSource;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class PartitionOperatorTest {
+
+ public static class CustomPojo implements Serializable, Comparable<CustomPojo> {
+ private Integer number;
+ private String name;
+
+ public CustomPojo() {
+ }
+
+ public CustomPojo(Integer number,
+ String name) {
+ this.number = number;
+ this.name = name;
+ }
+
+ public Integer getNumber() {
+ return number;
+ }
+
+ public void setNumber(Integer number) {
+ this.number = number;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public int compareTo(CustomPojo o) {
+ return Integer.compare(this.number, o.number);
+ }
+ }
+
+ public static class NestedPojo implements Serializable {
+ private CustomPojo nested;
+ private Long outer;
+
+ public NestedPojo() {
+ }
+
+ public NestedPojo(CustomPojo nested,
+ Long outer) {
+ this.nested = nested;
+ this.outer = outer;
+ }
+
+ public CustomPojo getNested() {
+ return nested;
+ }
+
+ public void setNested(CustomPojo nested) {
+ this.nested = nested;
+ }
+
+ public Long getOuter() {
+ return outer;
+ }
+
+ public void setOuter(Long outer) {
+ this.outer = outer;
+ }
+ }
+
+ private DataSet<Tuple2<Integer, String>> getTupleDataSet(ExecutionEnvironment env) {
+ return env.fromElements(
+ new Tuple2<>(1, "first"),
+ new Tuple2<>(2, "second"),
+ new Tuple2<>(3, "third"),
+ new Tuple2<>(4, "fourth"),
+ new Tuple2<>(5, "fifth"),
+ new Tuple2<>(6, "sixth")
+ );
+ }
+
+ private DataSet<CustomPojo> getPojoDataSet(ExecutionEnvironment env) {
+ return env.fromElements(
+ new CustomPojo(1, "first"),
+ new CustomPojo(2, "second"),
+ new CustomPojo(3, "third"),
+ new CustomPojo(4, "fourth"),
+ new CustomPojo(5, "fifth"),
+ new CustomPojo(6, "sixth")
+ );
+ }
+
+ private DataSet<NestedPojo> getNestedPojoDataSet(ExecutionEnvironment env) {
+ return env.fromElements(
+ new NestedPojo(new CustomPojo(1, "first"), 1L),
+ new NestedPojo(new CustomPojo(2, "second"), 2L),
+ new NestedPojo(new CustomPojo(3, "third"), 3L),
+ new NestedPojo(new CustomPojo(4, "fourth"), 4L),
+ new NestedPojo(new CustomPojo(5, "fifth"), 5L),
+ new NestedPojo(new CustomPojo(6, "sixth"), 6L)
+ );
+ }
+
+ @Test
+ public void testRebalance() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.rebalance();
+ }
+
+ @Test
+ public void testHashPartitionByField1() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByHash(0);
+ }
+
+ @Test
+ public void testHashPartitionByField2() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByHash(0, 1);
+ }
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testHashPartitionByFieldOutOfRange() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByHash(0, 1, 2);
+ }
+
+ @Test
+ public void testHashPartitionByFieldName1() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionByHash("number");
+ }
+
+ @Test
+ public void testHashPartitionByFieldName2() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionByHash("number", "name");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testHashPartitionByInvalidFieldName() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionByHash("number", "name", "invalidField");
+ }
+
+ @Test
+ public void testRangePartitionByFieldName1() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionByRange("number");
+ }
+
+ @Test
+ public void testRangePartitionByFieldName2() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionByRange("number", "name");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRangePartitionByInvalidFieldName() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionByRange("number", "name", "invalidField");
+ }
+
+ @Test
+ public void testRangePartitionByField1() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByRange(0);
+ }
+
+ @Test
+ public void testRangePartitionByField2() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByRange(0, 1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRangePartitionWithEmptyIndicesKey() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSource<Tuple2<Tuple2<Integer, Integer>, Integer>> ds = env.fromElements(
+ new Tuple2<>(new Tuple2<>(1, 1), 1),
+ new Tuple2<>(new Tuple2<>(2, 2), 2),
+ new Tuple2<>(new Tuple2<>(2, 2), 2)
+ );
+ ds.partitionByRange(new int[]{});
+ }
+
+ @Test(expected = IndexOutOfBoundsException.class)
+ public void testRangePartitionByFieldOutOfRange() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByRange(0, 1, 2);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testHashPartitionWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByHash(1).withOrders(Order.ASCENDING);
+ }
+
+ @Test(expected = IllegalStateException.class)
+ public void testRebalanceWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.rebalance().withOrders(Order.ASCENDING);
+ }
+
+ @Test
+ public void testRangePartitionWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByRange(0).withOrders(Order.ASCENDING);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRangePartitionWithTooManyOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionByRange(0).withOrders(Order.ASCENDING, Order.DESCENDING);
+ }
+
+ @Test
+ public void testRangePartitionByComplexKeyWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSource<Tuple2<Tuple2<Integer, Integer>, Integer>> ds = env.fromElements(
+ new Tuple2<>(new Tuple2<>(1, 1), 1),
+ new Tuple2<>(new Tuple2<>(2, 2), 2),
+ new Tuple2<>(new Tuple2<>(2, 2), 2)
+ );
+ ds.partitionByRange(0, 1).withOrders(Order.ASCENDING, Order.DESCENDING);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRangePartitionByComplexKeyWithTooManyOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSource<Tuple2<Tuple2<Integer, Integer>, Integer>> ds = env.fromElements(
+ new Tuple2<>(new Tuple2<>(1, 1), 1),
+ new Tuple2<>(new Tuple2<>(2, 2), 2),
+ new Tuple2<>(new Tuple2<>(2, 2), 2)
+ );
+ ds.partitionByRange(0).withOrders(Order.ASCENDING, Order.DESCENDING);
+ }
+
+ @Test
+ public void testRangePartitionBySelectorComplexKeyWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<NestedPojo> ds = getNestedPojoDataSet(env);
+ ds.partitionByRange(new KeySelector<NestedPojo, CustomPojo>() {
+ @Override
+ public CustomPojo getKey(NestedPojo value) throws Exception {
+ return value.getNested();
+ }
+ }).withOrders(Order.ASCENDING);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testRangePartitionBySelectorComplexKeyWithTooManyOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<NestedPojo> ds = getNestedPojoDataSet(env);
+ ds.partitionByRange(new KeySelector<NestedPojo, CustomPojo>() {
+ @Override
+ public CustomPojo getKey(NestedPojo value) throws Exception {
+ return value.getNested();
+ }
+ }).withOrders(Order.ASCENDING, Order.DESCENDING);
+ }
+
+ @Test
+ public void testRangePartitionCustomPartitionerByFieldId() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionCustom(new Partitioner<Integer>() {
+ @Override
+ public int partition(Integer key,
+ int numPartitions) {
+ return 1;
+ }
+ }, 0);
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testRangePartitionInvalidCustomPartitionerByFieldId() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+ ds.partitionCustom(new Partitioner<Integer>() {
+ @Override
+ public int partition(Integer key,
+ int numPartitions) {
+ return 1;
+ }
+ }, 1);
+ }
+
+ @Test
+ public void testRangePartitionCustomPartitionerByFieldName() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionCustom(new Partitioner<Integer>() {
+ @Override
+ public int partition(Integer key,
+ int numPartitions) {
+ return 1;
+ }
+ }, "number");
+ }
+
+ @Test(expected = InvalidProgramException.class)
+ public void testRangePartitionInvalidCustomPartitionerByFieldName() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionCustom(new Partitioner<Integer>() {
+ @Override
+ public int partition(Integer key,
+ int numPartitions) {
+ return 1;
+ }
+ }, "name");
+ }
+
+ @Test
+ public void testRangePartitionCustomPartitionerByKeySelector() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ final DataSet<CustomPojo> ds = getPojoDataSet(env);
+ ds.partitionCustom(new Partitioner<Integer>() {
+ @Override
+ public int partition(Integer key,
+ int numPartitions) {
+ return 1;
+ }
+ }, new KeySelector<CustomPojo, Integer>() {
+ @Override
+ public Integer getKey(CustomPojo value) throws Exception {
+ return value.getNumber();
+ }
+ });
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
index 9ecea6b..63ae1ad 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
@@ -24,7 +24,6 @@ import java.util.List;
import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.SemanticProperties;
import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
@@ -40,6 +39,7 @@ import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
import org.apache.flink.optimizer.plan.Channel;
import org.apache.flink.optimizer.plan.SingleInputPlanNode;
import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Preconditions;
/**
* The optimizer's internal representation of a <i>Partition</i> operator node.
@@ -52,7 +52,8 @@ public class PartitionNode extends SingleInputNode {
super(operator);
OperatorDescriptorSingle descr = new PartitionDescriptor(
- this.getOperator().getPartitionMethod(), this.keys, operator.getCustomPartitioner(), operator.getDistribution());
+ this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(),
+ operator.getDistribution());
this.possibleProperties = Collections.singletonList(descr);
}
@@ -90,13 +91,20 @@ public class PartitionNode extends SingleInputNode {
private final PartitionMethod pMethod;
private final Partitioner<?> customPartitioner;
private final DataDistribution distribution;
-
- public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Partitioner<?> customPartitioner, DataDistribution distribution) {
+ private final Ordering ordering;
+
+ public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Ordering ordering, Partitioner<?>
+ customPartitioner, DataDistribution distribution) {
super(pKeys);
-
+
+ Preconditions.checkArgument(pMethod != PartitionMethod.RANGE
+ || pKeys.toFieldList().isExactMatch(ordering.getInvolvedIndexes()),
+ "Partition keys must match the given ordering.");
+
this.pMethod = pMethod;
this.customPartitioner = customPartitioner;
this.distribution = distribution;
+ this.ordering = ordering;
}
@Override
@@ -124,12 +132,6 @@ public class PartitionNode extends SingleInputNode {
rgps.setCustomPartitioned(this.keys, this.customPartitioner);
break;
case RANGE:
- // Initiate Ordering as ascending here as no order parameter in API level,
- // we could revisit this while order is required in future optimization.
- Ordering ordering = new Ordering();
- for (int field : this.keys) {
- ordering.appendOrdering(field, null, Order.ASCENDING);
- }
rgps.setRangePartitioned(ordering, distribution);
break;
default:
http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
index 04953c7..85d70e3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
@@ -18,9 +18,11 @@
package org.apache.flink.test.javaApiOperators;
+import java.io.Serializable;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import org.apache.flink.api.common.InvalidProgramException;
@@ -29,6 +31,7 @@ import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.MapPartitionFunction;
import org.apache.flink.api.common.functions.ReduceFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.operators.Order;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.KeySelector;
@@ -498,9 +501,9 @@ public class PartitionITCase extends MultipleProgramsTestBase {
DataSource<Long> dataSource = env.generateSequence(0, 10000);
KeySelector<Long, Long> keyExtractor = new ObjectSelfKeySelector();
- MapPartitionFunction<Long, Tuple2<Long, Long>> MinMaxSelector = new MinMaxSelector();
+ MapPartitionFunction<Long, Tuple2<Long, Long>> MinMaxSelector = new MinMaxSelector<>(new LongComparator(true));
- Comparator<Tuple2<Long, Long>> tuple2Comparator = new Tuple2Comparator();
+ Comparator<Tuple2<Long, Long>> tuple2Comparator = new Tuple2Comparator(new LongComparator(true));
List<Tuple2<Long, Long>> collected = dataSource.partitionByRange(keyExtractor).mapPartition(MinMaxSelector).collect();
Collections.sort(collected, tuple2Comparator);
@@ -546,6 +549,212 @@ public class PartitionITCase extends MultipleProgramsTestBase {
result.collect(); // should fail
}
+
+
+ @Test
+ public void testRangePartitionerOnSequenceDataWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple2<Long, Long>> dataSet = env.generateSequence(0, 10000)
+ .map(new MapFunction<Long, Tuple2<Long, Long>>() {
+ @Override
+ public Tuple2<Long, Long> map(Long value) throws Exception {
+ return new Tuple2<>(value / 5000, value % 5000);
+ }
+ });
+
+ final Tuple2Comparator<Long> tuple2Comparator = new Tuple2Comparator<>(new LongComparator(true),
+ new LongComparator(false));
+
+ MinMaxSelector<Tuple2<Long, Long>> minMaxSelector = new MinMaxSelector<>(tuple2Comparator);
+
+ final List<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>> collected = dataSet.partitionByRange(0, 1)
+ .withOrders(Order.ASCENDING, Order.DESCENDING)
+ .mapPartition(minMaxSelector)
+ .collect();
+
+ Collections.sort(collected, new Tuple2Comparator<>(tuple2Comparator));
+
+ Tuple2<Long, Long> previousMax = null;
+ for (Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> tuple2 : collected) {
+ assertTrue("Min element in each partition should be smaller than max.",
+ tuple2Comparator.compare(tuple2.f0, tuple2.f1) <= 0);
+ if (previousMax == null) {
+ previousMax = tuple2.f1;
+ } else {
+ assertTrue("Partitions overlap. Previous max should be smaller than current min.",
+ tuple2Comparator.compare(previousMax, tuple2.f0) < 0);
+ if (previousMax.f0.equals(tuple2.f0.f0)) {
+ //check that ordering on the second key is correct
+ assertEquals("Ordering on the second field should be continous.",
+ previousMax.f1 - 1, tuple2.f0.f1.longValue());
+ }
+ previousMax = tuple2.f1;
+ }
+ }
+ }
+
+ @Test
+ public void testRangePartitionerOnSequenceNestedDataWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ final DataSet<Tuple2<Tuple2<Long, Long>, Long>> dataSet = env.generateSequence(0, 10000)
+ .map(new MapFunction<Long, Tuple2<Tuple2<Long, Long>, Long>>() {
+ @Override
+ public Tuple2<Tuple2<Long, Long>, Long> map(Long value) throws Exception {
+ return new Tuple2<>(new Tuple2<>(value / 5000, value % 5000), value);
+ }
+ });
+
+ final Tuple2Comparator<Long> tuple2Comparator = new Tuple2Comparator<>(new LongComparator(true),
+ new LongComparator(true));
+ MinMaxSelector<Tuple2<Long, Long>> minMaxSelector = new MinMaxSelector<>(tuple2Comparator);
+
+ final List<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>> collected = dataSet.partitionByRange(0)
+ .withOrders(Order.ASCENDING)
+ .mapPartition(new MapPartitionFunction<Tuple2<Tuple2<Long,Long>,Long>, Tuple2<Long, Long>>() {
+ @Override
+ public void mapPartition(Iterable<Tuple2<Tuple2<Long, Long>, Long>> values,
+ Collector<Tuple2<Long, Long>> out) throws Exception {
+ for (Tuple2<Tuple2<Long, Long>, Long> value : values) {
+ out.collect(value.f0);
+ }
+ }
+ })
+ .mapPartition(minMaxSelector)
+ .collect();
+
+ Collections.sort(collected, new Tuple2Comparator<>(tuple2Comparator));
+
+ Tuple2<Long, Long> previousMax = null;
+ for (Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> tuple2 : collected) {
+ assertTrue("Min element in each partition should be smaller than max.",
+ tuple2Comparator.compare(tuple2.f0, tuple2.f1) <= 0);
+ if (previousMax == null) {
+ previousMax = tuple2.f1;
+ } else {
+ assertTrue("Partitions overlap. Previous max should be smaller than current min.",
+ tuple2Comparator.compare(previousMax, tuple2.f0) < 0);
+ if (previousMax.f0.equals(tuple2.f0.f0)) {
+ assertEquals("Ordering on the second field should be continous.",
+ previousMax.f1 + 1, tuple2.f0.f1.longValue());
+ }
+ previousMax = tuple2.f1;
+ }
+ }
+ }
+
+ @Test
+ public void testRangePartitionerWithKeySelectorOnSequenceNestedDataWithOrders() throws Exception {
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ final DataSet<Tuple2<ComparablePojo, Long>> dataSet = env.generateSequence(0, 10000)
+ .map(new MapFunction<Long, Tuple2<ComparablePojo, Long>>() {
+ @Override
+ public Tuple2<ComparablePojo, Long> map(Long value) throws Exception {
+ return new Tuple2<>(new ComparablePojo(value / 5000, value % 5000), value);
+ }
+ });
+
+ final List<Tuple2<ComparablePojo, ComparablePojo>> collected = dataSet
+ .partitionByRange(new KeySelector<Tuple2<ComparablePojo, Long>, ComparablePojo>() {
+ @Override
+ public ComparablePojo getKey(Tuple2<ComparablePojo, Long> value) throws Exception {
+ return value.f0;
+ }
+ })
+ .withOrders(Order.ASCENDING)
+ .mapPartition(new MinMaxSelector<>(new ComparablePojoComparator()))
+ .mapPartition(new ExtractComparablePojo())
+ .collect();
+
+ final Comparator<Tuple2<ComparablePojo, ComparablePojo>> pojoComparator =
+ new Comparator<Tuple2<ComparablePojo, ComparablePojo>>() {
+ @Override
+ public int compare(Tuple2<ComparablePojo, ComparablePojo> o1,
+ Tuple2<ComparablePojo, ComparablePojo> o2) {
+ return o1.f0.compareTo(o2.f1);
+ }
+ };
+ Collections.sort(collected, pojoComparator);
+
+ ComparablePojo previousMax = null;
+ for (Tuple2<ComparablePojo, ComparablePojo> element : collected) {
+ assertTrue("Min element in each partition should be smaller than max.",
+ element.f0.compareTo(element.f1) <= 0);
+ if (previousMax == null) {
+ previousMax = element.f1;
+ } else {
+ assertTrue("Partitions overlap. Previous max should be smaller than current min.",
+ previousMax.compareTo(element.f0) < 0);
+ if (previousMax.first.equals(element.f0.first)) {
+ assertEquals("Ordering on the second field should be continous.",
+ previousMax.second - 1, element.f0.second.longValue());
+ }
+ previousMax = element.f1;
+ }
+ }
+ }
+
+ private static class ExtractComparablePojo implements MapPartitionFunction<
+ Tuple2<Tuple2<ComparablePojo, Long>, Tuple2<ComparablePojo, Long>>,
+ Tuple2<ComparablePojo, ComparablePojo>> {
+
+ @Override
+ public void mapPartition(Iterable<Tuple2<Tuple2<ComparablePojo, Long>, Tuple2<ComparablePojo, Long>>> values,
+ Collector<Tuple2<ComparablePojo, ComparablePojo>> out) throws Exception {
+ for (Tuple2<Tuple2<ComparablePojo, Long>, Tuple2<ComparablePojo, Long>> value : values) {
+ out.collect(new Tuple2<>(value.f0.f0, value.f1.f0));
+ }
+ }
+ }
+
+ private static class ComparablePojoComparator implements Comparator<Tuple2<ComparablePojo, Long>>, Serializable {
+
+ @Override
+ public int compare(Tuple2<ComparablePojo, Long> o1,
+ Tuple2<ComparablePojo, Long> o2) {
+ return o1.f0.compareTo(o2.f0);
+ }
+ }
+
+ private static class ComparablePojo implements Comparable<ComparablePojo> {
+ private Long first;
+ private Long second;
+
+ public Long getFirst() {
+ return first;
+ }
+
+ public void setFirst(Long first) {
+ this.first = first;
+ }
+
+ public Long getSecond() {
+ return second;
+ }
+
+ public void setSecond(Long second) {
+ this.second = second;
+ }
+
+ public ComparablePojo(Long first,
+ Long second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ public ComparablePojo() {
+ }
+
+ @Override
+ public int compareTo(ComparablePojo o) {
+ final int firstResult = Long.compare(this.first, o.first);
+ if (firstResult == 0) {
+ return (-1) * Long.compare(this.second, o.second);
+ }
+
+ return firstResult;
+ }
+ }
+
private static class ObjectSelfKeySelector implements KeySelector<Long, Long> {
@Override
public Long getKey(Long value) throws Exception {
@@ -553,36 +762,61 @@ public class PartitionITCase extends MultipleProgramsTestBase {
}
}
- private static class MinMaxSelector implements MapPartitionFunction<Long, Tuple2<Long, Long>> {
+ private static class MinMaxSelector<T> implements MapPartitionFunction<T, Tuple2<T, T>> {
+
+ private final Comparator<T> comparator;
+
+ public MinMaxSelector(Comparator<T> comparator) {
+ this.comparator = comparator;
+ }
+
@Override
- public void mapPartition(Iterable<Long> values, Collector<Tuple2<Long, Long>> out) throws Exception {
- long max = Long.MIN_VALUE;
- long min = Long.MAX_VALUE;
- for (long value : values) {
- if (value > max) {
+ public void mapPartition(Iterable<T> values, Collector<Tuple2<T, T>> out) throws Exception {
+ Iterator<T> itr = values.iterator();
+ T min = itr.next();
+ T max = min;
+ T value;
+ while (itr.hasNext()) {
+ value= itr.next();
+ if (comparator.compare(value, min) < 0) {
+ min = value;
+ }
+ if (comparator.compare(value, max) > 0) {
max = value;
}
- if (value < min) {
- min = value;
- }
}
- Tuple2<Long, Long> result = new Tuple2<>(min, max);
+
+ Tuple2<T, T> result = new Tuple2<>(min, max);
out.collect(result);
}
}
- private static class Tuple2Comparator implements Comparator<Tuple2<Long, Long>> {
+ private static class Tuple2Comparator<T> implements Comparator<Tuple2<T, T>>, Serializable {
+
+ private final Comparator<T> firstComparator;
+ private final Comparator<T> secondComparator;
+
+ public Tuple2Comparator(Comparator<T> comparator) {
+ this(comparator, comparator);
+ }
+
+ public Tuple2Comparator(Comparator<T> firstComparator,
+ Comparator<T> secondComparator) {
+ this.firstComparator = firstComparator;
+ this.secondComparator = secondComparator;
+ }
+
@Override
- public int compare(Tuple2<Long, Long> first, Tuple2<Long, Long> second) {
- long result = first.f0 - second.f0;
+ public int compare(Tuple2<T, T> first, Tuple2<T, T> second) {
+ long result = firstComparator.compare(first.f0, second.f0);
if (result > 0) {
return 1;
} else if (result < 0) {
return -1;
}
- result = first.f1 - second.f1;
+ result = secondComparator.compare(first.f1, second.f1);
if (result > 0) {
return 1;
} else if (result < 0) {
@@ -593,4 +827,22 @@ public class PartitionITCase extends MultipleProgramsTestBase {
}
}
+ private static class LongComparator implements Comparator<Long>, Serializable {
+
+ private final boolean ascending;
+
+ public LongComparator(boolean ascending) {
+ this.ascending = ascending;
+ }
+
+ @Override
+ public int compare(Long o1, Long o2) {
+ if (ascending) {
+ return Long.compare(o1, o2);
+ } else {
+ return (-1) * Long.compare(o1, o2);
+ }
+ }
+ }
+
}
[2/3] flink git commit: [FLINK-2998] [dataSet] Add support for
explicit range partitioning for joins and coGroup.
Posted by fh...@apache.org.
[FLINK-2998] [dataSet] Add support for explicit range partitioning for joins and coGroup.
This closes #1838
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/605b6d87
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/605b6d87
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/605b6d87
Branch: refs/heads/master
Commit: 605b6d870d0da38fb5446675709c7243127cdff1
Parents: d8fb230
Author: gallenvara <ga...@126.com>
Authored: Tue Mar 29 22:36:21 2016 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Apr 20 22:55:49 2016 +0200
----------------------------------------------------------------------
.../api/java/operators/PartitionOperator.java | 5 +-
.../dataproperties/GlobalProperties.java | 3 +
.../operators/AbstractJoinDescriptor.java | 9 +-
.../optimizer/operators/CoGroupDescriptor.java | 11 +-
.../operators/OperatorDescriptorDual.java | 23 ++++
...oGroupGlobalPropertiesCompatibilityTest.java | 124 +++++++++++++++++++
.../operators/CoGroupWithDistributionTest.java | 99 +++++++++++++++
.../JoinGlobalPropertiesCompatibilityTest.java | 123 ++++++++++++++++++
.../operators/JoinWithDistributionTest.java | 97 +++++++++++++++
.../optimizer/operators/TestDistribution.java | 71 +++++++++++
.../operators/shipping/OutputEmitter.java | 4 +-
.../test/javaApiOperators/CoGroupITCase.java | 81 +++++++++++-
.../CustomDistributionITCase.java | 77 +++++++++++-
.../flink/test/javaApiOperators/JoinITCase.java | 85 ++++++++++++-
14 files changed, 792 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
index fb6d579..2ed0300 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
@@ -88,8 +88,9 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
Preconditions.checkArgument(distribution == null || pMethod == PartitionMethod.RANGE, "Customized data distribution is only neccessary for range partition.");
if (distribution != null) {
- Preconditions.checkArgument(distribution.getNumberOfFields() == pKeys.getNumberOfKeyFields(), "The number of key fields in the distribution and range partitioner should be the same.");
- Preconditions.checkArgument(Arrays.equals(distribution.getKeyTypes(), pKeys.getKeyFieldTypes()), "The types of key from the distribution and range partitioner are not equal.");
+ Preconditions.checkArgument(pKeys.getNumberOfKeyFields() <= distribution.getNumberOfFields(), "The distribution must provide at least as many fields as flat key fields are specified.");
+ Preconditions.checkArgument(Arrays.equals(pKeys.getKeyFieldTypes(), Arrays.copyOfRange(distribution.getKeyTypes(), 0, pKeys.getNumberOfKeyFields())),
+ "The types of the flat key fields must be equal to the types of the fields of the distribution.");
}
if (customPartitioner != null) {
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
index ca17c2b..e64782f 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
@@ -311,6 +311,7 @@ public class GlobalProperties implements Cloneable {
gp.partitioning = PartitioningProperty.RANGE_PARTITIONED;
gp.ordering = newOrdering;
gp.partitioningFields = newOrdering.getInvolvedIndexes();
+ gp.distribution = this.distribution;
}
break;
case HASH_PARTITIONED:
@@ -436,6 +437,7 @@ public class GlobalProperties implements Cloneable {
throw new CompilerException("Unsupported partitioning strategy");
}
+ channel.setDataDistribution(this.distribution);
DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
}
@@ -490,6 +492,7 @@ public class GlobalProperties implements Cloneable {
newProps.partitioning = this.partitioning;
newProps.partitioningFields = this.partitioningFields;
newProps.ordering = this.ordering;
+ newProps.distribution = this.distribution;
newProps.customPartitioner = this.customPartitioner;
newProps.uniqueFieldCombinations = this.uniqueFieldCombinations == null ? null : new HashSet<FieldSet>(this.uniqueFieldCombinations);
return newProps;
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
index 6c2776e..1f2f42a 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
@@ -140,11 +140,12 @@ public abstract class AbstractJoinDescriptor extends OperatorDescriptorDual {
}
else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
- produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
+ produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+ produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {
- // Return false anyway now, we need both the partition key and data distribution
- // information to make sure whether the range partitions are equivalent.
- return false;
+ return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
+ checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
+ produced1.getDataDistribution().equals(produced2.getDataDistribution());
}
else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
index b99b1c1..0e3fb55 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
@@ -149,12 +149,13 @@ public class CoGroupDescriptor extends OperatorDescriptorDual {
}
else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
- produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
-
- // Return false anyway now, we need both the partition key and data distribution
- // information to make sure whether the range partitions are equivalent.
- return false;
+ produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+ produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {
+ return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
+ checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
+ produced1.getDataDistribution().equals(produced2.getDataDistribution());
+
}
else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
index 17ea8a5..e5e43ca 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
@@ -125,6 +125,29 @@ public abstract class OperatorDescriptorDual implements AbstractOperatorDescript
return true;
}
+ protected boolean checkSameOrdering(GlobalProperties produced1, GlobalProperties produced2, int numRelevantFields) {
+ Ordering prod1 = produced1.getPartitioningOrdering();
+ Ordering prod2 = produced2.getPartitioningOrdering();
+
+ if (prod1 == null || prod2 == null) {
+ throw new CompilerException("The given properties do not meet this operators requirements.");
+ }
+
+ // check that order of fields is equivalent
+ if (!checkEquivalentFieldPositionsInKeyFields(
+ prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) {
+ return false;
+ }
+
+ // check that both inputs have the same directions of order
+ for (int i = 0; i < numRelevantFields; i++) {
+ if (prod1.getOrder(i) != prod2.getOrder(i)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
protected boolean checkSameOrdering(LocalProperties produced1, LocalProperties produced2, int numRelevantFields) {
Ordering prod1 = produced1.getOrdering();
Ordering prod2 = produced2.getOrdering();
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
index 23f8897..76f48e9 100644
--- a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
@@ -21,6 +21,8 @@ package org.apache.flink.optimizer.operators;
import static org.junit.Assert.*;
import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
@@ -95,6 +97,52 @@ public class CoGroupGlobalPropertiesCompatibilityTest {
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
+
+ TestDistribution dist1 = new TestDistribution(1);
+ TestDistribution dist2 = new TestDistribution(1);
+
+ // test compatible range partitioning with one ordering
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering2.appendOrdering(field, null, Order.ASCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+ // test compatible range partitioning with two orderings
+ {
+ Ordering ordering1 = new Ordering();
+ ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
+ ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
+ Ordering ordering2 = new Ordering();
+ ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
+ ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
}
catch (Exception e) {
e.printStackTrace();
@@ -152,6 +200,82 @@ public class CoGroupGlobalPropertiesCompatibilityTest {
assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
+
+ TestDistribution dist1 = new TestDistribution(1);
+ TestDistribution dist2 = new TestDistribution(1);
+
+ // test incompatible range partitioning with different key size
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ ordering2.appendOrdering(field, null, Order.ASCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test incompatible range partitioning with different ordering
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering2.appendOrdering(field, null, Order.DESCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ TestDistribution dist3 = new TestDistribution(1);
+ TestDistribution dist4 = new TestDistribution(2);
+
+ // test incompatible range partitioning with different distribution
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering2.appendOrdering(field, null, Order.ASCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist3);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist4);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist3);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist4);
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java
new file mode 100644
index 0000000..2d37c7c
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.utils.DataSetUtils;
+import org.apache.flink.optimizer.plan.*;
+import org.apache.flink.optimizer.util.CompilerTestBase;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class CoGroupWithDistributionTest extends CompilerTestBase {
+
+ @Test
+ public void CoGroupWithSameDistributionTest() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ TestDistribution testDistribution1 = new TestDistribution(3);
+ TestDistribution testDistribution2 = new TestDistribution(3);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, testDistribution1, 0)
+ .coGroup(DataSetUtils.partitionByRange(set2, testDistribution2, 0))
+ .where(0).equalTo(0).with(new CoGroupFunc());
+
+ coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+ Channel input1 = coGroup.getInput1();
+ Channel input2 = coGroup.getInput2();
+ assertEquals(ShipStrategyType.FORWARD, input1.getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, input2.getShipStrategy());
+
+ }
+
+ @Test
+ public void CoGroupWithDifferentDistributionTest() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ TestDistribution testDistribution1 = new TestDistribution(3);
+ TestDistribution testDistribution2 = new TestDistribution(2);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, testDistribution1, 0)
+ .coGroup(DataSetUtils.partitionByRange(set2, testDistribution2, 0))
+ .where(0).equalTo(0).with(new CoGroupFunc());
+
+ coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+ Channel input1 = coGroup.getInput1();
+ Channel input2 = coGroup.getInput2();
+ assertEquals(ShipStrategyType.PARTITION_HASH, input1.getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, input2.getShipStrategy());
+
+ }
+
+ public static class CoGroupFunc implements CoGroupFunction<Tuple3<Integer, Integer, Integer>,
+ Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+ @Override
+ public void coGroup(Iterable<Tuple3<Integer, Integer, Integer>> first, Iterable<Tuple3<Integer, Integer, Integer>> second,
+ Collector<Tuple3<Integer, Integer, Integer>> out) throws Exception {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
index 1d559c2..610cbd8 100644
--- a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
@@ -21,6 +21,8 @@ package org.apache.flink.optimizer.operators;
import static org.junit.Assert.*;
import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
import org.apache.flink.api.common.operators.util.FieldList;
import org.apache.flink.optimizer.dataproperties.GlobalProperties;
import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
@@ -95,6 +97,51 @@ public class JoinGlobalPropertiesCompatibilityTest {
assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
+
+ TestDistribution dist1 = new TestDistribution(1);
+ TestDistribution dist2 = new TestDistribution(1);
+ // test compatible range partitioning with one ordering
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering2.appendOrdering(field, null, Order.ASCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+ // test compatible range partitioning with two orderings
+ {
+ Ordering ordering1 = new Ordering();
+ ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
+ ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
+ Ordering ordering2 = new Ordering();
+ ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
+ ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
}
catch (Exception e) {
e.printStackTrace();
@@ -152,6 +199,82 @@ public class JoinGlobalPropertiesCompatibilityTest {
assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
}
+
+ TestDistribution dist1 = new TestDistribution(1);
+ TestDistribution dist2 = new TestDistribution(1);
+
+ // test incompatible range partitioning with different key size
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ ordering2.appendOrdering(field, null, Order.ASCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ // test incompatible range partitioning with different ordering
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering2.appendOrdering(field, null, Order.DESCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist1);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist2);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist1);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist2);
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
+
+ TestDistribution dist3 = new TestDistribution(1);
+ TestDistribution dist4 = new TestDistribution(2);
+
+ // test incompatible range partitioning with different distribution
+ {
+ Ordering ordering1 = new Ordering();
+ for (int field : keysLeft) {
+ ordering1.appendOrdering(field, null, Order.ASCENDING);
+ }
+ Ordering ordering2 = new Ordering();
+ for (int field : keysRight) {
+ ordering2.appendOrdering(field, null, Order.ASCENDING);
+ }
+
+ RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+ reqLeft.setRangePartitioned(ordering1, dist3);
+ RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+ reqRight.setRangePartitioned(ordering2, dist4);
+
+ GlobalProperties propsLeft = new GlobalProperties();
+ propsLeft.setRangePartitioned(ordering1, dist3);
+ GlobalProperties propsRight = new GlobalProperties();
+ propsRight.setRangePartitioned(ordering2, dist4);
+ assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+ }
}
catch (Exception e) {
e.printStackTrace();
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java
new file mode 100644
index 0000000..0c579f6
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.utils.DataSetUtils;
+import org.apache.flink.optimizer.plan.*;
+import org.apache.flink.optimizer.util.CompilerTestBase;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class JoinWithDistributionTest extends CompilerTestBase {
+
+ @Test
+ public void JoinWithSameDistributionTest() throws Exception {
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ TestDistribution dist1 = new TestDistribution(3);
+ TestDistribution dist2 = new TestDistribution(3);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, dist1, 0)
+ .join(DataSetUtils.partitionByRange(set2, dist2, 0))
+ .where(0).equalTo(0).with(new JoinFunc());
+
+ coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+ Channel input1 = join.getInput1();
+ Channel input2 = join.getInput2();
+ assertEquals(ShipStrategyType.FORWARD, input1.getShipStrategy());
+ assertEquals(ShipStrategyType.FORWARD, input2.getShipStrategy());
+
+ }
+
+ @Test
+ public void JoinWithDifferentDistributionTest() throws Exception{
+ ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+ DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+ DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+ TestDistribution dist1 = new TestDistribution(3);
+ TestDistribution dist2 = new TestDistribution(4);
+
+ DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, dist1, 0)
+ .join(DataSetUtils.partitionByRange(set2, dist2, 0))
+ .where(0).equalTo(0).with(new JoinFunc());
+
+ coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+ Plan plan = env.createProgramPlan();
+ OptimizedPlan oPlan = compileWithStats(plan);
+
+ SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+ DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+ Channel input1 = join.getInput1();
+ Channel input2 = join.getInput2();
+ assertEquals(ShipStrategyType.PARTITION_HASH, input1.getShipStrategy());
+ assertEquals(ShipStrategyType.PARTITION_HASH, input2.getShipStrategy());
+
+ }
+
+ public static class JoinFunc implements JoinFunction<Tuple3<Integer, Integer, Integer>,
+ Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+ @Override
+ public Tuple3<Integer, Integer, Integer> join(Tuple3<Integer, Integer, Integer> first, Tuple3<Integer, Integer, Integer> second) throws Exception {
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java
new file mode 100644
index 0000000..d99930b
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+
+public class TestDistribution implements DataDistribution {
+
+ public int boundary;
+
+ public TestDistribution(int boundary) {
+ this.boundary = boundary;
+ }
+
+ @Override
+ public Object[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+ return new Object[0];
+ }
+
+ @Override
+ public int getNumberOfFields() {
+ return 1;
+ }
+
+ @Override
+ public TypeInformation[] getKeyTypes() {
+ return new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO};
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ boolean isEqual = true;
+ TestDistribution dist = (TestDistribution)obj;
+ if (this.boundary != dist.boundary) {
+ isEqual = false;
+ }
+ return isEqual;
+ }
+}
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
index 4e20842..e6f3d26 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
@@ -258,11 +258,11 @@ public class OutputEmitter<T> implements ChannelSelector<SerializationDelegate<T
private final int compareRecordAndBoundary(T record, Object[] boundary) {
this.comparator.extractKeys(record, keys, 0);
- if (flatComparators.length != keys.length || flatComparators.length != boundary.length) {
+ if (flatComparators.length != keys.length || flatComparators.length > boundary.length) {
throw new RuntimeException("Can not compare keys with boundary due to mismatched length.");
}
- for (int i=0; i<flatComparators.length; i++) {
+ for (int i = 0; i < flatComparators.length; i++) {
int result = flatComparators[i].compare(keys[i], boundary[i]);
if (result != 0) {
return result;
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
index 7bc8480..5b7caa7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
@@ -18,8 +18,11 @@
package org.apache.flink.test.javaApiOperators;
+import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.CoGroupFunction;
import org.apache.flink.api.common.functions.RichCoGroupFunction;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.DataSet;
import org.apache.flink.api.java.ExecutionEnvironment;
import org.apache.flink.api.java.functions.KeySelector;
@@ -28,7 +31,10 @@ import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.utils.DataSetUtils;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
@@ -40,6 +46,7 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -286,7 +293,7 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
DataSet<Tuple3<Integer, Long, String>> coGrouped = ds1.coGroup(ds2).
- where(0,4).equalTo(0,1).with(new Tuple5Tuple3CoGroup());
+ where(0, 4).equalTo(0, 1).with(new Tuple5Tuple3CoGroup());
List<Tuple3<Integer, Long, String>> result = coGrouped.collect();
@@ -516,6 +523,37 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
compareResultAsText(result, expected);
}
+ @Test
+ public void testCoGroupWithRangePartitioning() throws Exception {
+ /*
+ * Test coGroup on tuples with multiple key field positions and same customized distribution
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds1 = CollectionDataSets.get5TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
+
+ env.setParallelism(4);
+ TestDistribution testDis = new TestDistribution();
+ DataSet<Tuple3<Integer, Long, String>> coGrouped =
+ DataSetUtils.partitionByRange(ds1, testDis, 0, 4)
+ .coGroup(DataSetUtils.partitionByRange(ds2, testDis, 0, 1))
+ .where(0, 4)
+ .equalTo(0, 1)
+ .with(new Tuple5Tuple3CoGroup());
+
+ List<Tuple3<Integer, Long, String>> result = coGrouped.collect();
+
+ String expected = "1,1,Hallo\n" +
+ "2,2,Hallo Welt\n" +
+ "3,2,Hallo Welt wie gehts?\n" +
+ "3,2,ABC\n" +
+ "5,3,HIJ\n" +
+ "5,3,IJK\n";
+
+ compareResultAsTuples(result, expected);
+ }
// --------------------------------------------------------------------------------------------
@@ -797,4 +835,45 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
}
}
}
+
+ public static class TestDistribution implements DataDistribution {
+ public Object[][] boundaries = new Object[][]{
+ new Object[]{2, 2L},
+ new Object[]{5, 4L},
+ new Object[]{10, 12L},
+ new Object[]{21, 6L}
+ };
+
+ public TestDistribution() {}
+
+ @Override
+ public Object[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+ return boundaries[bucketNum];
+ }
+
+ @Override
+ public int getNumberOfFields() {
+ return 2;
+ }
+
+ @Override
+ public TypeInformation[] getKeyTypes() {
+ return new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO};
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof TestDistribution;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
index 427e83e..c6bc08e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
@@ -49,13 +49,13 @@ public class CustomDistributionITCase {
ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- DataSet<Tuple3<Integer, Long, String>> input1 = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
final TestDataDist1 dist = new TestDataDist1();
env.setParallelism(dist.getParallelism());
DataSet<Boolean> result = DataSetUtils
- .partitionByRange(input1, dist, 0)
+ .partitionByRange(input, dist, 0)
.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, Long, String>, Boolean>() {
@Override
@@ -104,7 +104,7 @@ public class CustomDistributionITCase {
ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
- DataSet<Tuple3<Integer, Integer, String>> input1 = env.fromElements(
+ DataSet<Tuple3<Integer, Integer, String>> input = env.fromElements(
new Tuple3<>(1, 5, "Hi"),
new Tuple3<>(1, 6, "Hi"),
new Tuple3<>(1, 7, "Hi"),
@@ -127,7 +127,7 @@ public class CustomDistributionITCase {
env.setParallelism(dist.getParallelism());
DataSet<Boolean> result = DataSetUtils
- .partitionByRange(input1, dist, 0, 1)
+ .partitionByRange(input, dist, 0, 1)
.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, Integer, String>, Boolean>() {
@Override
@@ -175,6 +175,75 @@ public class CustomDistributionITCase {
env.execute();
}
+ @Test
+ public void testPartitionKeyLessDistribution() throws Exception{
+ /*
+ * Test the number of partition keys less than the number of distribution fields
+ */
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
+ final TestDataDist2 dist = new TestDataDist2();
+
+ env.setParallelism(dist.getParallelism());
+
+ DataSet<Boolean> result = DataSetUtils
+ .partitionByRange(input, dist, 0)
+ .mapPartition(new RichMapPartitionFunction<Tuple3<Integer, Long, String>, Boolean>() {
+
+ @Override
+ public void mapPartition(Iterable<Tuple3<Integer, Long, String>> values, Collector<Boolean> out) throws Exception {
+ int pIdx = getRuntimeContext().getIndexOfThisSubtask();
+
+ for (Tuple3<Integer, Long, String> s : values) {
+ boolean correctlyPartitioned = true;
+ if (pIdx == 0) {
+ Integer[] upper = dist.boundaries[0];
+ if (s.f0.compareTo(upper[0]) > 0) {
+ correctlyPartitioned = false;
+ }
+ }
+ else if (pIdx > 0 && pIdx < dist.getParallelism() - 1) {
+ Integer[] lower = dist.boundaries[pIdx - 1];
+ Integer[] upper = dist.boundaries[pIdx];
+ if (s.f0.compareTo(upper[0]) > 0 || (s.f0.compareTo(lower[0]) <= 0)) {
+ correctlyPartitioned = false;
+ }
+ }
+ else {
+ Integer[] lower = dist.boundaries[pIdx - 1];
+ if ((s.f0.compareTo(lower[0]) <= 0)) {
+ correctlyPartitioned = false;
+ }
+ }
+
+ if (!correctlyPartitioned) {
+ fail("Record was not correctly partitioned: " + s.toString());
+ }
+ }
+ }
+ }
+ );
+
+ result.output(new DiscardingOutputFormat<Boolean>());
+ env.execute();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testPartitionMoreThanDistribution() throws Exception{
+ /*
+ * Test the number of partition keys larger than the number of distribution fields
+ */
+
+ ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
+ final TestDataDist2 dist = new TestDataDist2();
+
+ DataSet<Tuple3<Integer, Long, String>> result = DataSetUtils
+ .partitionByRange(input, dist, 0, 1, 2);
+ }
+
/**
* The class is used to do the tests of range partition with one key.
*/
http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
index 61e07fe..0d8c80b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
@@ -18,11 +18,15 @@
package org.apache.flink.test.javaApiOperators;
+import java.io.IOException;
import java.util.Collection;
import java.util.List;
+import org.apache.flink.api.common.distributions.DataDistribution;
import org.apache.flink.api.common.functions.FlatJoinFunction;
import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.common.functions.RichFlatJoinFunction;
import org.apache.flink.api.java.tuple.Tuple2;
@@ -30,7 +34,10 @@ import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.api.java.tuple.Tuple5;
import org.apache.flink.api.java.tuple.Tuple6;
import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.utils.DataSetUtils;
import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
@@ -87,8 +94,8 @@ public class JoinITCase extends MultipleProgramsTestBase {
DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
DataSet<Tuple2<String, String>> joinDs =
ds1.join(ds2)
- .where(0,1)
- .equalTo(0,4)
+ .where(0, 1)
+ .equalTo(0, 4)
.with(new T3T5FlatJoin());
List<Tuple2<String, String>> result = joinDs.collect();
@@ -680,6 +687,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
compareResultAsTuples(result, expected);
}
+ @Test
public void testJoinWithAtomicType2() throws Exception {
final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
@@ -696,6 +704,38 @@ public class JoinITCase extends MultipleProgramsTestBase {
compareResultAsTuples(result, expected);
}
+ @Test
+ public void testJoinWithRangePartitioning() throws Exception {
+ /*
+ * Test Join on tuples with multiple key field positions and same customized distribution
+ */
+
+ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+ DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
+ DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
+
+ env.setParallelism(4);
+ TestDistribution testDis = new TestDistribution();
+ DataSet<Tuple2<String, String>> joinDs =
+ DataSetUtils.partitionByRange(ds1, testDis, 0, 1)
+ .join(DataSetUtils.partitionByRange(ds2, testDis, 0, 4))
+ .where(0, 1)
+ .equalTo(0, 4)
+ .with(new T3T5FlatJoin());
+
+ List<Tuple2<String, String>> result = joinDs.collect();
+
+ String expected = "Hi,Hallo\n" +
+ "Hello,Hallo Welt\n" +
+ "Hello world,Hallo Welt wie gehts?\n" +
+ "Hello world,ABC\n" +
+ "I am fine.,HIJ\n" +
+ "I am fine.,IJK\n";
+
+ compareResultAsTuples(result, expected);
+ }
+
public static class T3T5FlatJoin implements FlatJoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<String, String>> {
@Override
@@ -778,4 +818,45 @@ public class JoinITCase extends MultipleProgramsTestBase {
return new Tuple2<String, String>(first.myString, second.f2);
}
}
+
+ public static class TestDistribution implements DataDistribution {
+ public Object boundaries[][] = new Object[][]{
+ new Object[]{2, 2L},
+ new Object[]{5, 4L},
+ new Object[]{10, 12L},
+ new Object[]{21, 6L}
+ };
+
+ public TestDistribution() {}
+
+ @Override
+ public Object[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+ return boundaries[bucketNum];
+ }
+
+ @Override
+ public int getNumberOfFields() {
+ return 2;
+ }
+
+ @Override
+ public TypeInformation[] getKeyTypes() {
+ return new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO};
+ }
+
+ @Override
+ public void write(DataOutputView out) throws IOException {
+
+ }
+
+ @Override
+ public void read(DataInputView in) throws IOException {
+
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj instanceof TestDistribution;
+ }
+ }
}
[3/3] flink git commit: [FLINK-3781] Ensure BlobClient is closed.
Posted by fh...@apache.org.
[FLINK-3781] Ensure BlobClient is closed.
This closes #1908
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a43bade0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a43bade0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a43bade0
Branch: refs/heads/master
Commit: a43bade0d87d373ff236152d13d8d56e17220605
Parents: 605b6d8
Author: Chenguang He <ga...@gmail.com>
Authored: Mon Apr 18 13:52:45 2016 -0500
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Apr 20 23:00:56 2016 +0200
----------------------------------------------------------------------
.../org/apache/flink/runtime/blob/BlobCache.java | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/a43bade0/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
index 4d33364..bb0aacb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
@@ -71,13 +71,13 @@ public final class BlobCache implements BlobService {
// configure the number of fetch retries
final int fetchRetries = configuration.getInteger(
- ConfigConstants.BLOB_FETCH_RETRIES_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_RETRIES);
+ ConfigConstants.BLOB_FETCH_RETRIES_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_RETRIES);
if (fetchRetries >= 0) {
this.numFetchRetries = fetchRetries;
}
else {
LOG.warn("Invalid value for {}. System will attempt no retires on failed fetches of BLOBs.",
- ConfigConstants.BLOB_FETCH_RETRIES_KEY);
+ ConfigConstants.BLOB_FETCH_RETRIES_KEY);
this.numFetchRetries = 0;
}
@@ -89,7 +89,7 @@ public final class BlobCache implements BlobService {
* Returns the URL for the BLOB with the given key. The method will first attempt to serve
* the BLOB from its local cache. If the BLOB is not in the cache, the method will try to download it
* from this cache's BLOB server.
- *
+ *
* @param requiredBlob The key of the desired BLOB.
* @return URL referring to the local storage location of the BLOB.
* @throws IOException Thrown if an I/O error occurs while downloading the BLOBs from the BLOB server.
@@ -163,7 +163,7 @@ public final class BlobCache implements BlobService {
}
catch (IOException e) {
String message = "Failed to fetch BLOB " + requiredBlob + " from " + serverAddress +
- " and store it under " + localJarFile.getAbsolutePath();
+ " and store it under " + localJarFile.getAbsolutePath();
if (attempt < numFetchRetries) {
attempt++;
if (LOG.isDebugEnabled()) {
@@ -200,10 +200,14 @@ public final class BlobCache implements BlobService {
* @param key referring to the file to be deleted
*/
public void deleteGlobal(BlobKey key) throws IOException {
- delete(key);
BlobClient bc = createClient();
- bc.delete(key);
- bc.close();
+ try {
+ delete(key);
+ bc.delete(key);
+ }
+ finally {
+ bc.close();
+ }
}
@Override