You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2016/04/21 01:37:54 UTC

[1/3] flink git commit: [FLINK-3665] [dataSet] Add support for sort orders in range partitioning.

Repository: flink
Updated Branches:
  refs/heads/master c33325e3f -> a43bade0d


[FLINK-3665] [dataSet] Add support for sort orders in range partitioning.

This closes #1848


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/d8fb2308
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/d8fb2308
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/d8fb2308

Branch: refs/heads/master
Commit: d8fb23085839a514fbd0b56cc681dd52aac503ac
Parents: c33325e
Author: dawid <da...@gmail.com>
Authored: Sat Apr 2 13:10:59 2016 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Apr 20 22:52:28 2016 +0200

----------------------------------------------------------------------
 .../apache/flink/api/common/operators/Keys.java |  43 +-
 .../operators/base/PartitionOperatorBase.java   |  15 +-
 .../common/operators/ExpressionKeysTest.java    |  75 ++++
 .../operators/SelectorFunctionKeysTest.java     |  39 ++
 .../api/java/operators/PartitionOperator.java   |  55 ++-
 .../java/operator/PartitionOperatorTest.java    | 407 +++++++++++++++++++
 .../flink/optimizer/dag/PartitionNode.java      |  24 +-
 .../test/javaApiOperators/PartitionITCase.java  | 284 ++++++++++++-
 8 files changed, 905 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
index 0b771e8..6085ebf 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/Keys.java
@@ -32,6 +32,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.regex.Pattern;
 
 import static org.apache.flink.util.Preconditions.checkArgument;
 import static org.apache.flink.util.Preconditions.checkNotNull;
@@ -45,6 +46,8 @@ public abstract class Keys<T> {
 
 	public abstract TypeInformation<?>[] getKeyFieldTypes();
 
+	public abstract TypeInformation<?>[] getOriginalKeyFieldTypes();
+
 	public abstract <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo);
 
 	public boolean isEmpty() {
@@ -82,6 +85,7 @@ public abstract class Keys<T> {
 		private final TypeInformation<T> inputType;
 		private final TypeInformation<K> keyType;
 		private final List<FlatFieldDescriptor> keyFields;
+		private final TypeInformation<?>[] originalKeyTypes;
 
 		public SelectorFunctionKeys(KeySelector<T, K> keyExtractor, TypeInformation<T> inputType, TypeInformation<K> keyType) {
 
@@ -99,6 +103,7 @@ public abstract class Keys<T> {
 			this.inputType = inputType;
 			this.keyType = keyType;
 
+			this.originalKeyTypes = new TypeInformation[] {keyType};
 			if (keyType instanceof CompositeType) {
 				this.keyFields = ((CompositeType<T>)keyType).getFlatFields(ExpressionKeys.SELECT_ALL_CHAR);
 			}
@@ -142,7 +147,12 @@ public abstract class Keys<T> {
 			}
 			return fieldTypes;
 		}
-		
+
+		@Override
+		public TypeInformation<?>[] getOriginalKeyFieldTypes() {
+			return originalKeyTypes;
+		}
+
 		@Override
 		public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) {
 
@@ -184,9 +194,13 @@ public abstract class Keys<T> {
 		
 		public static final String SELECT_ALL_CHAR = "*";
 		public static final String SELECT_ALL_CHAR_SCALA = "_";
-		
+		private static final Pattern WILD_CARD_REGEX = Pattern.compile("[\\.]?("
+				+ "\\" + SELECT_ALL_CHAR + "|"
+				+ "\\" + SELECT_ALL_CHAR_SCALA +")$");
+
 		// Flattened fields representing keys fields
 		private List<FlatFieldDescriptor> keyFields;
+		private TypeInformation<?>[] originalKeyTypes;
 
 		/**
 		 * ExpressionKeys that is defined by the full data type.
@@ -233,7 +247,7 @@ public abstract class Keys<T> {
 			} else {
 				rangeCheckFields(keyPositions, type.getArity() - 1);
 			}
-			
+
 			checkArgument(keyPositions.length > 0, "Grouping fields can not be empty at this point");
 
 			// extract key field types
@@ -242,10 +256,13 @@ public abstract class Keys<T> {
 
 			// for each key position, find all (nested) field types
 			String[] fieldNames = cType.getFieldNames();
+			this.originalKeyTypes = new TypeInformation<?>[keyPositions.length];
 			ArrayList<FlatFieldDescriptor> tmpList = new ArrayList<>();
-			for (int keyPos : keyPositions) {
+			for (int i = 0; i < keyPositions.length; i++) {
+				int keyPos = keyPositions[i];
 				tmpList.clear();
 				// get all flat fields
+				this.originalKeyTypes[i] = cType.getTypeAt(keyPos);
 				cType.getFlatFields(fieldNames[keyPos], 0, tmpList);
 				// check if fields are of key type
 				for(FlatFieldDescriptor ffd : tmpList) {
@@ -274,9 +291,12 @@ public abstract class Keys<T> {
 
 			if (type instanceof CompositeType){
 				CompositeType<T> cType = (CompositeType<T>) type;
+				this.originalKeyTypes = new TypeInformation<?>[keyExpressions.length];
 
 				// extract the keys on their flat position
-				for (String keyExpr : keyExpressions) {
+				for (int i = 0; i < keyExpressions.length; i++) {
+					String keyExpr = keyExpressions[i];
+
 					if (keyExpr == null) {
 						throw new InvalidProgramException("Expression key may not be null.");
 					}
@@ -296,6 +316,13 @@ public abstract class Keys<T> {
 					}
 					// add flat fields to key fields
 					keyFields.addAll(flatFields);
+
+					String strippedKeyExpr = WILD_CARD_REGEX.matcher(keyExpr).replaceAll("");
+					if (strippedKeyExpr.isEmpty()) {
+						this.originalKeyTypes[i] = type;
+					} else {
+						this.originalKeyTypes[i] = cType.getTypeAt(strippedKeyExpr);
+					}
 				}
 			}
 			else {
@@ -318,6 +345,7 @@ public abstract class Keys<T> {
 					// add full type as key
 					keyFields.add(new FlatFieldDescriptor(0, type));
 				}
+				this.originalKeyTypes = new TypeInformation[] {type};
 			}
 		}
 		
@@ -348,6 +376,11 @@ public abstract class Keys<T> {
 		}
 
 		@Override
+		public TypeInformation<?>[] getOriginalKeyFieldTypes() {
+			return originalKeyTypes;
+		}
+
+		@Override
 		public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) {
 
 			if (keyFields.size() != 1) {

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
----------------------------------------------------------------------
diff --git a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
index 4b802aa..1afa45a 100644
--- a/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
+++ b/flink-core/src/main/java/org/apache/flink/api/common/operators/base/PartitionOperatorBase.java
@@ -26,6 +26,7 @@ import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.Partitioner;
 import org.apache.flink.api.common.functions.RuntimeContext;
 import org.apache.flink.api.common.functions.util.NoOpFunction;
+import org.apache.flink.api.common.operators.Ordering;
 import org.apache.flink.api.common.operators.SingleInputOperator;
 import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
 import org.apache.flink.api.common.operators.UnaryOperatorInformation;
@@ -51,13 +52,15 @@ public class PartitionOperatorBase<IN> extends SingleInputOperator<IN, IN, NoOpF
 	private Partitioner<?> customPartitioner;
 	
 	private DataDistribution distribution;
+
+	private Ordering ordering;
 	
 	
 	public PartitionOperatorBase(UnaryOperatorInformation<IN, IN> operatorInfo, PartitionMethod pMethod, int[] keys, String name) {
 		super(new UserCodeObjectWrapper<NoOpFunction>(new NoOpFunction()), operatorInfo, keys, name);
 		this.partitionMethod = pMethod;
 	}
-	
+
 	public PartitionOperatorBase(UnaryOperatorInformation<IN, IN> operatorInfo, PartitionMethod pMethod, String name) {
 		super(new UserCodeObjectWrapper<NoOpFunction>(new NoOpFunction()), operatorInfo, name);
 		this.partitionMethod = pMethod;
@@ -76,7 +79,15 @@ public class PartitionOperatorBase<IN> extends SingleInputOperator<IN, IN, NoOpF
 	public DataDistribution getDistribution() {
 		return  this.distribution;
 	}
-	
+
+	public void setOrdering(Ordering ordering) {
+		this.ordering = ordering;
+	}
+
+	public Ordering getOrdering() {
+		return ordering;
+	}
+
 	public void setDistribution(DataDistribution distribution) {
 		this.distribution = distribution;
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
index 8e65a79..2a6ea48 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/ExpressionKeysTest.java
@@ -281,6 +281,81 @@ public class ExpressionKeysTest {
 		ek = new ExpressionKeys<>("*", ti);
 		Assert.assertArrayEquals(new int[] {0,1,2,3,4,5,6,7}, ek.computeLogicalKeyPositions());
 
+		ek = new ExpressionKeys<>("f2.p1.*", ti);
+		Assert.assertArrayEquals(new int[] {4,5}, ek.computeLogicalKeyPositions());
+	}
+
+	@Test
+	public void testOriginalTypes() {
+
+		TypeInformation<Tuple3<Integer, Pojo1, PojoWithMultiplePojos>> ti =
+				new TupleTypeInfo<>(
+						BasicTypeInfo.INT_TYPE_INFO,
+						TypeExtractor.getForClass(Pojo1.class),
+						TypeExtractor.getForClass(PojoWithMultiplePojos.class)
+				);
+
+		ExpressionKeys<Tuple3<Integer, Pojo1, PojoWithMultiplePojos>> ek;
+
+		ek = new ExpressionKeys<>(0, ti);
+		Assert.assertArrayEquals(new TypeInformation[] {BasicTypeInfo.INT_TYPE_INFO}, ek.getOriginalKeyFieldTypes());
+
+		ek = new ExpressionKeys<>(1, ti);
+		Assert.assertArrayEquals(new TypeInformation[] {TypeExtractor.getForClass(Pojo1.class)}, ek.getOriginalKeyFieldTypes());
+
+		ek = new ExpressionKeys<>(2, ti);
+		Assert.assertArrayEquals(new TypeInformation[] {TypeExtractor.getForClass(PojoWithMultiplePojos.class)}, ek.getOriginalKeyFieldTypes());
+
+		ek = new ExpressionKeys<>(new int[]{}, ti, true);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { BasicTypeInfo.INT_TYPE_INFO,
+						TypeExtractor.getForClass(Pojo1.class),
+						TypeExtractor.getForClass(PojoWithMultiplePojos.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("*", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { ti },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("f1", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo1.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("f1.*", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo1.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("f2.*", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { TypeExtractor.getForClass(PojoWithMultiplePojos.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("f2.p2", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo2.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("f2.p2.*", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo2.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
+		ek = new ExpressionKeys<>("f2.p2._", ti);
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { TypeExtractor.getForClass(Pojo2.class) },
+				ek.getOriginalKeyFieldTypes()
+		);
+
 	}
 
 	@Test(expected = InvalidProgramException.class)

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
----------------------------------------------------------------------
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java b/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
index ad6fefa..d63e24c 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/operators/SelectorFunctionKeysTest.java
@@ -116,6 +116,45 @@ public class SelectorFunctionKeysTest {
 		Assert.assertTrue(sk2.areCompatible(ek1));
 	}
 
+	@Test
+	public void testOriginalTypes1() throws Keys.IncompatibleKeysException {
+		TypeInformation<Tuple2<Integer, String>> t2 = new TupleTypeInfo<>(
+				BasicTypeInfo.INT_TYPE_INFO,
+				BasicTypeInfo.STRING_TYPE_INFO
+		);
+
+		Keys<Tuple2<Integer, String>> k = new Keys.SelectorFunctionKeys<>(
+				new KeySelector2(),
+				t2,
+				BasicTypeInfo.STRING_TYPE_INFO
+		);
+
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { BasicTypeInfo.STRING_TYPE_INFO },
+				k.getOriginalKeyFieldTypes()
+		);
+	}
+
+	@Test
+	public void testOriginalTypes2() throws Exception {
+		final TupleTypeInfo<Tuple2<Integer, String>> t1 = new TupleTypeInfo<>(
+				BasicTypeInfo.INT_TYPE_INFO,
+				BasicTypeInfo.STRING_TYPE_INFO
+		);
+		TypeInformation<PojoWithMultiplePojos> t2 = TypeExtractor.getForClass(PojoWithMultiplePojos.class);
+
+		Keys<PojoWithMultiplePojos> sk = new Keys.SelectorFunctionKeys<>(
+				new KeySelector3(),
+				t2,
+				t1
+		);
+
+		Assert.assertArrayEquals(
+				new TypeInformation<?>[] { t1 },
+				sk.getOriginalKeyFieldTypes()
+		);
+	}
+
 	@SuppressWarnings("serial")
 	public static class KeySelector1 implements KeySelector<Pojo2, String> {
 

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
index dc4a018..fb6d579 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
@@ -18,14 +18,16 @@
 
 package org.apache.flink.api.java.operators;
 
-import com.google.common.base.Preconditions;
 
 import org.apache.flink.annotation.Internal;
 import org.apache.flink.annotation.Public;
+import org.apache.flink.annotation.PublicEvolving;
 import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.Partitioner;
 import org.apache.flink.api.common.operators.Keys;
 import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
 import org.apache.flink.api.common.operators.UnaryOperatorInformation;
 import org.apache.flink.api.common.operators.base.PartitionOperatorBase;
 import org.apache.flink.api.common.operators.base.PartitionOperatorBase.PartitionMethod;
@@ -33,6 +35,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys;
 import org.apache.flink.api.java.tuple.Tuple2;
+import org.apache.flink.util.Preconditions;
 
 import java.util.Arrays;
 
@@ -49,6 +52,7 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 	private final String partitionLocationName;
 	private final Partitioner<?> customPartitioner;
 	private final DataDistribution distribution;
+	private Order[] orders;
 
 
 	public PartitionOperator(DataSet<T> input, PartitionMethod pMethod, Keys<T> pKeys, String partitionLocationName) {
@@ -98,6 +102,24 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 		this.customPartitioner = customPartitioner;
 		this.distribution = distribution;
 	}
+
+	/**
+	 * Sets the order of keys for range partitioning.
+	 * NOTE: Only valid for {@link PartitionMethod.RANGE}.
+	 *
+	 * @param orders array of orders for each specified partition key
+	 * @return The partitioneOperator with properly set orders for given keys
+     */
+	@PublicEvolving
+	public PartitionOperator<T> withOrders(Order... orders) {
+		Preconditions.checkState(pMethod == PartitionMethod.RANGE, "Orders cannot be applied for %s partition " +
+				"method", pMethod);
+		Preconditions.checkArgument(pKeys.getOriginalKeyFieldTypes().length == orders.length, "The number of key " +
+				"fields and orders should be the same.");
+		this.orders = orders;
+
+		return this;
+	}
 	
 	// --------------------------------------------------------------------------------------------
 	//  Properties
@@ -142,6 +164,7 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 				partitionedInput.setParallelism(getParallelism());
 				partitionedInput.setDistribution(distribution);
 				partitionedInput.setCustomPartitioner(customPartitioner);
+				partitionedInput.setOrdering(computeOrdering(pKeys, orders));
 				
 				return partitionedInput;
 			}
@@ -149,7 +172,8 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 				
 				@SuppressWarnings("unchecked")
 				Keys.SelectorFunctionKeys<T, ?> selectorKeys = (Keys.SelectorFunctionKeys<T, ?>) pKeys;
-				return translateSelectorFunctionPartitioner(selectorKeys, pMethod, name, input, getParallelism(), customPartitioner);
+				return translateSelectorFunctionPartitioner(selectorKeys, pMethod, name, input, getParallelism(),
+						customPartitioner, orders);
 			}
 			else {
 				throw new UnsupportedOperationException("Unrecognized key type.");
@@ -161,6 +185,29 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 		}
 	}
 
+	private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) {
+		Ordering ordering = new Ordering();
+		final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions();
+
+		if (orders == null) {
+			for (int key : logicalKeyPositions) {
+				ordering.appendOrdering(key, null, Order.ASCENDING);
+			}
+		} else {
+			final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes();
+			int index = 0;
+			for (int i = 0; i < originalKeyFieldTypes.length; i++) {
+				final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields();
+				for (int j = index; j < index + typeTotalFields; j++) {
+					ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]);
+				}
+				index += typeTotalFields;
+			}
+		}
+
+		return ordering;
+	}
+
 	@SuppressWarnings("unchecked")
 	private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
 		SelectorFunctionKeys<T, ?> rawKeys,
@@ -168,7 +215,8 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 		String name,
 		Operator<T> input,
 		int partitionDop,
-		Partitioner<?> customPartitioner)
+		Partitioner<?> customPartitioner,
+		Order[] orders)
 	{
 		final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
 		TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
@@ -180,6 +228,7 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 		keyedPartitionedInput.setInput(keyedInput);
 		keyedPartitionedInput.setCustomPartitioner(customPartitioner);
 		keyedPartitionedInput.setParallelism(partitionDop);
+		keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));
 
 		return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
 	}

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java
----------------------------------------------------------------------
diff --git a/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java b/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java
new file mode 100644
index 0000000..34f72a2
--- /dev/null
+++ b/flink-java/src/test/java/org/apache/flink/api/java/operator/PartitionOperatorTest.java
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.java.operator;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.functions.KeySelector;
+import org.apache.flink.api.java.operators.DataSource;
+import org.apache.flink.api.java.tuple.Tuple2;
+import org.junit.Test;
+
+import java.io.Serializable;
+
+public class PartitionOperatorTest {
+
+    public static class CustomPojo implements Serializable, Comparable<CustomPojo> {
+        private Integer number;
+        private String name;
+
+        public CustomPojo() {
+        }
+
+        public CustomPojo(Integer number,
+                   String name) {
+            this.number = number;
+            this.name = name;
+        }
+
+        public Integer getNumber() {
+            return number;
+        }
+
+        public void setNumber(Integer number) {
+            this.number = number;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public void setName(String name) {
+            this.name = name;
+        }
+
+        @Override
+        public int compareTo(CustomPojo o) {
+            return Integer.compare(this.number, o.number);
+        }
+    }
+
+    public static class NestedPojo implements Serializable {
+        private CustomPojo nested;
+        private Long outer;
+
+        public NestedPojo() {
+        }
+
+        public NestedPojo(CustomPojo nested,
+                          Long outer) {
+            this.nested = nested;
+            this.outer = outer;
+        }
+
+        public CustomPojo getNested() {
+            return nested;
+        }
+
+        public void setNested(CustomPojo nested) {
+            this.nested = nested;
+        }
+
+        public Long getOuter() {
+            return outer;
+        }
+
+        public void setOuter(Long outer) {
+            this.outer = outer;
+        }
+    }
+
+    private DataSet<Tuple2<Integer, String>> getTupleDataSet(ExecutionEnvironment env) {
+        return env.fromElements(
+                new Tuple2<>(1, "first"),
+                new Tuple2<>(2, "second"),
+                new Tuple2<>(3, "third"),
+                new Tuple2<>(4, "fourth"),
+                new Tuple2<>(5, "fifth"),
+                new Tuple2<>(6, "sixth")
+        );
+    }
+
+    private DataSet<CustomPojo> getPojoDataSet(ExecutionEnvironment env) {
+        return env.fromElements(
+                new CustomPojo(1, "first"),
+                new CustomPojo(2, "second"),
+                new CustomPojo(3, "third"),
+                new CustomPojo(4, "fourth"),
+                new CustomPojo(5, "fifth"),
+                new CustomPojo(6, "sixth")
+        );
+    }
+
+    private DataSet<NestedPojo> getNestedPojoDataSet(ExecutionEnvironment env) {
+        return env.fromElements(
+                new NestedPojo(new CustomPojo(1, "first"), 1L),
+                new NestedPojo(new CustomPojo(2, "second"), 2L),
+                new NestedPojo(new CustomPojo(3, "third"), 3L),
+                new NestedPojo(new CustomPojo(4, "fourth"), 4L),
+                new NestedPojo(new CustomPojo(5, "fifth"), 5L),
+                new NestedPojo(new CustomPojo(6, "sixth"), 6L)
+        );
+    }
+
+    @Test
+    public void testRebalance() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.rebalance();
+    }
+
+    @Test
+    public void testHashPartitionByField1() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByHash(0);
+    }
+
+    @Test
+    public void testHashPartitionByField2() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByHash(0, 1);
+    }
+
+    @Test(expected = IndexOutOfBoundsException.class)
+    public void testHashPartitionByFieldOutOfRange() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByHash(0, 1, 2);
+    }
+
+    @Test
+    public void testHashPartitionByFieldName1() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionByHash("number");
+    }
+
+    @Test
+    public void testHashPartitionByFieldName2() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionByHash("number", "name");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testHashPartitionByInvalidFieldName() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionByHash("number", "name", "invalidField");
+    }
+
+    @Test
+    public void testRangePartitionByFieldName1() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionByRange("number");
+    }
+
+    @Test
+    public void testRangePartitionByFieldName2() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionByRange("number", "name");
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testRangePartitionByInvalidFieldName() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionByRange("number", "name", "invalidField");
+    }
+
+    @Test
+    public void testRangePartitionByField1() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByRange(0);
+    }
+
+    @Test
+    public void testRangePartitionByField2() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByRange(0, 1);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testRangePartitionWithEmptyIndicesKey() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSource<Tuple2<Tuple2<Integer, Integer>, Integer>> ds = env.fromElements(
+                new Tuple2<>(new Tuple2<>(1, 1), 1),
+                new Tuple2<>(new Tuple2<>(2, 2), 2),
+                new Tuple2<>(new Tuple2<>(2, 2), 2)
+        );
+        ds.partitionByRange(new int[]{});
+    }
+
+    @Test(expected = IndexOutOfBoundsException.class)
+    public void testRangePartitionByFieldOutOfRange() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByRange(0, 1, 2);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testHashPartitionWithOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByHash(1).withOrders(Order.ASCENDING);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void testRebalanceWithOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.rebalance().withOrders(Order.ASCENDING);
+    }
+
+    @Test
+    public void testRangePartitionWithOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByRange(0).withOrders(Order.ASCENDING);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testRangePartitionWithTooManyOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionByRange(0).withOrders(Order.ASCENDING, Order.DESCENDING);
+    }
+
+    @Test
+    public void testRangePartitionByComplexKeyWithOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSource<Tuple2<Tuple2<Integer, Integer>, Integer>> ds = env.fromElements(
+                new Tuple2<>(new Tuple2<>(1, 1), 1),
+                new Tuple2<>(new Tuple2<>(2, 2), 2),
+                new Tuple2<>(new Tuple2<>(2, 2), 2)
+        );
+        ds.partitionByRange(0, 1).withOrders(Order.ASCENDING, Order.DESCENDING);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testRangePartitionByComplexKeyWithTooManyOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSource<Tuple2<Tuple2<Integer, Integer>, Integer>> ds = env.fromElements(
+                new Tuple2<>(new Tuple2<>(1, 1), 1),
+                new Tuple2<>(new Tuple2<>(2, 2), 2),
+                new Tuple2<>(new Tuple2<>(2, 2), 2)
+        );
+        ds.partitionByRange(0).withOrders(Order.ASCENDING, Order.DESCENDING);
+    }
+
+    @Test
+    public void testRangePartitionBySelectorComplexKeyWithOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<NestedPojo> ds = getNestedPojoDataSet(env);
+        ds.partitionByRange(new KeySelector<NestedPojo, CustomPojo>() {
+            @Override
+            public CustomPojo getKey(NestedPojo value) throws Exception {
+                return value.getNested();
+            }
+        }).withOrders(Order.ASCENDING);
+    }
+
+    @Test(expected = IllegalArgumentException.class)
+    public void testRangePartitionBySelectorComplexKeyWithTooManyOrders() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<NestedPojo> ds = getNestedPojoDataSet(env);
+        ds.partitionByRange(new KeySelector<NestedPojo, CustomPojo>() {
+            @Override
+            public CustomPojo getKey(NestedPojo value) throws Exception {
+                return value.getNested();
+            }
+        }).withOrders(Order.ASCENDING, Order.DESCENDING);
+    }
+
+    @Test
+    public void testRangePartitionCustomPartitionerByFieldId() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionCustom(new Partitioner<Integer>() {
+            @Override
+            public int partition(Integer key,
+                                 int numPartitions) {
+                return 1;
+            }
+        }, 0);
+    }
+
+    @Test(expected = InvalidProgramException.class)
+    public void testRangePartitionInvalidCustomPartitionerByFieldId() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<Tuple2<Integer, String>> ds = getTupleDataSet(env);
+        ds.partitionCustom(new Partitioner<Integer>() {
+            @Override
+            public int partition(Integer key,
+                                 int numPartitions) {
+                return 1;
+            }
+        }, 1);
+    }
+
+    @Test
+    public void testRangePartitionCustomPartitionerByFieldName() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionCustom(new Partitioner<Integer>() {
+            @Override
+            public int partition(Integer key,
+                                 int numPartitions) {
+                return 1;
+            }
+        }, "number");
+    }
+
+    @Test(expected = InvalidProgramException.class)
+    public void testRangePartitionInvalidCustomPartitionerByFieldName() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionCustom(new Partitioner<Integer>() {
+            @Override
+            public int partition(Integer key,
+                                 int numPartitions) {
+                return 1;
+            }
+        }, "name");
+    }
+
+    @Test
+    public void testRangePartitionCustomPartitionerByKeySelector() throws Exception {
+        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+        final DataSet<CustomPojo> ds = getPojoDataSet(env);
+        ds.partitionCustom(new Partitioner<Integer>() {
+            @Override
+            public int partition(Integer key,
+                                 int numPartitions) {
+                return 1;
+            }
+        }, new KeySelector<CustomPojo, Integer>() {
+            @Override
+            public Integer getKey(CustomPojo value) throws Exception {
+                return value.getNumber();
+            }
+        });
+    }
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
index 9ecea6b..63ae1ad 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dag/PartitionNode.java
@@ -24,7 +24,6 @@ import java.util.List;
 
 import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.Partitioner;
-import org.apache.flink.api.common.operators.Order;
 import org.apache.flink.api.common.operators.Ordering;
 import org.apache.flink.api.common.operators.SemanticProperties;
 import org.apache.flink.api.common.operators.SingleInputSemanticProperties;
@@ -40,6 +39,7 @@ import org.apache.flink.optimizer.operators.OperatorDescriptorSingle;
 import org.apache.flink.optimizer.plan.Channel;
 import org.apache.flink.optimizer.plan.SingleInputPlanNode;
 import org.apache.flink.runtime.operators.DriverStrategy;
+import org.apache.flink.util.Preconditions;
 
 /**
  * The optimizer's internal representation of a <i>Partition</i> operator node.
@@ -52,7 +52,8 @@ public class PartitionNode extends SingleInputNode {
 		super(operator);
 		
 		OperatorDescriptorSingle descr = new PartitionDescriptor(
-					this.getOperator().getPartitionMethod(), this.keys, operator.getCustomPartitioner(), operator.getDistribution());
+					this.getOperator().getPartitionMethod(), this.keys, operator.getOrdering(), operator.getCustomPartitioner(),
+					operator.getDistribution());
 		this.possibleProperties = Collections.singletonList(descr);
 	}
 
@@ -90,13 +91,20 @@ public class PartitionNode extends SingleInputNode {
 		private final PartitionMethod pMethod;
 		private final Partitioner<?> customPartitioner;
 		private final DataDistribution distribution;
-		
-		public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Partitioner<?> customPartitioner, DataDistribution distribution) {
+		private final Ordering ordering;
+
+		public PartitionDescriptor(PartitionMethod pMethod, FieldSet pKeys, Ordering ordering, Partitioner<?>
+				customPartitioner, DataDistribution distribution) {
 			super(pKeys);
-			
+
+			Preconditions.checkArgument(pMethod != PartitionMethod.RANGE
+					|| pKeys.toFieldList().isExactMatch(ordering.getInvolvedIndexes()),
+					"Partition keys must match the given ordering.");
+
 			this.pMethod = pMethod;
 			this.customPartitioner = customPartitioner;
 			this.distribution = distribution;
+			this.ordering = ordering;
 		}
 		
 		@Override
@@ -124,12 +132,6 @@ public class PartitionNode extends SingleInputNode {
 				rgps.setCustomPartitioned(this.keys, this.customPartitioner);
 				break;
 			case RANGE:
-				// Initiate Ordering as ascending here as no order parameter in API level,
-				// we could revisit this while order is required in future optimization.
-				Ordering ordering = new Ordering();
-				for (int field : this.keys) {
-					ordering.appendOrdering(field, null, Order.ASCENDING);
-				}
 				rgps.setRangePartitioned(ordering, distribution);
 				break;
 			default:

http://git-wip-us.apache.org/repos/asf/flink/blob/d8fb2308/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
index 04953c7..85d70e3 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/PartitionITCase.java
@@ -18,9 +18,11 @@
 
 package org.apache.flink.test.javaApiOperators;
 
+import java.io.Serializable;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 
 import org.apache.flink.api.common.InvalidProgramException;
@@ -29,6 +31,7 @@ import org.apache.flink.api.common.functions.MapFunction;
 import org.apache.flink.api.common.functions.MapPartitionFunction;
 import org.apache.flink.api.common.functions.ReduceFunction;
 import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.api.common.operators.Order;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -498,9 +501,9 @@ public class PartitionITCase extends MultipleProgramsTestBase {
 		DataSource<Long> dataSource = env.generateSequence(0, 10000);
 		KeySelector<Long, Long> keyExtractor = new ObjectSelfKeySelector();
 
-		MapPartitionFunction<Long, Tuple2<Long, Long>> MinMaxSelector = new MinMaxSelector();
+		MapPartitionFunction<Long, Tuple2<Long, Long>> MinMaxSelector = new MinMaxSelector<>(new LongComparator(true));
 
-		Comparator<Tuple2<Long, Long>> tuple2Comparator = new Tuple2Comparator();
+		Comparator<Tuple2<Long, Long>> tuple2Comparator = new Tuple2Comparator(new LongComparator(true));
 
 		List<Tuple2<Long, Long>> collected = dataSource.partitionByRange(keyExtractor).mapPartition(MinMaxSelector).collect();
 		Collections.sort(collected, tuple2Comparator);
@@ -546,6 +549,212 @@ public class PartitionITCase extends MultipleProgramsTestBase {
 		result.collect(); // should fail
 	}
 
+
+
+	@Test
+	public void testRangePartitionerOnSequenceDataWithOrders() throws Exception {
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		DataSet<Tuple2<Long, Long>> dataSet = env.generateSequence(0, 10000)
+				.map(new MapFunction<Long, Tuple2<Long, Long>>() {
+			@Override
+			public Tuple2<Long, Long> map(Long value) throws Exception {
+				return new Tuple2<>(value / 5000, value % 5000);
+			}
+		});
+
+		final Tuple2Comparator<Long> tuple2Comparator = new Tuple2Comparator<>(new LongComparator(true),
+																			   new LongComparator(false));
+
+		MinMaxSelector<Tuple2<Long, Long>> minMaxSelector = new MinMaxSelector<>(tuple2Comparator);
+
+		final List<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>> collected = dataSet.partitionByRange(0, 1)
+				.withOrders(Order.ASCENDING, Order.DESCENDING)
+				.mapPartition(minMaxSelector)
+				.collect();
+
+		Collections.sort(collected, new Tuple2Comparator<>(tuple2Comparator));
+
+		Tuple2<Long, Long> previousMax = null;
+		for (Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> tuple2 : collected) {
+            assertTrue("Min element in each partition should be smaller than max.",
+                    tuple2Comparator.compare(tuple2.f0, tuple2.f1) <= 0);
+			if (previousMax == null) {
+				previousMax = tuple2.f1;
+			} else {
+                assertTrue("Partitions overlap. Previous max should be smaller than current min.",
+                        tuple2Comparator.compare(previousMax, tuple2.f0) < 0);
+				if (previousMax.f0.equals(tuple2.f0.f0)) {
+                    //check that ordering on the second key is correct
+					assertEquals("Ordering on the second field should be continous.",
+                            previousMax.f1 - 1, tuple2.f0.f1.longValue());
+				}
+				previousMax = tuple2.f1;
+			}
+		}
+	}
+
+	@Test
+	public void testRangePartitionerOnSequenceNestedDataWithOrders() throws Exception {
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		final DataSet<Tuple2<Tuple2<Long, Long>, Long>> dataSet = env.generateSequence(0, 10000)
+				.map(new MapFunction<Long, Tuple2<Tuple2<Long, Long>, Long>>() {
+					@Override
+					public Tuple2<Tuple2<Long, Long>, Long> map(Long value) throws Exception {
+						return new Tuple2<>(new Tuple2<>(value / 5000, value % 5000), value);
+					}
+				});
+
+		final Tuple2Comparator<Long> tuple2Comparator = new Tuple2Comparator<>(new LongComparator(true),
+				new LongComparator(true));
+		MinMaxSelector<Tuple2<Long, Long>> minMaxSelector = new MinMaxSelector<>(tuple2Comparator);
+
+		final List<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>> collected = dataSet.partitionByRange(0)
+				.withOrders(Order.ASCENDING)
+				.mapPartition(new MapPartitionFunction<Tuple2<Tuple2<Long,Long>,Long>, Tuple2<Long, Long>>() {
+					@Override
+					public void mapPartition(Iterable<Tuple2<Tuple2<Long, Long>, Long>> values,
+											 Collector<Tuple2<Long, Long>> out) throws Exception {
+						for (Tuple2<Tuple2<Long, Long>, Long> value : values) {
+							out.collect(value.f0);
+						}
+					}
+				})
+				.mapPartition(minMaxSelector)
+				.collect();
+
+		Collections.sort(collected, new Tuple2Comparator<>(tuple2Comparator));
+
+		Tuple2<Long, Long> previousMax = null;
+		for (Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>> tuple2 : collected) {
+            assertTrue("Min element in each partition should be smaller than max.",
+                    tuple2Comparator.compare(tuple2.f0, tuple2.f1) <= 0);
+			if (previousMax == null) {
+				previousMax = tuple2.f1;
+			} else {
+                assertTrue("Partitions overlap. Previous max should be smaller than current min.",
+                        tuple2Comparator.compare(previousMax, tuple2.f0) < 0);
+				if (previousMax.f0.equals(tuple2.f0.f0)) {
+					assertEquals("Ordering on the second field should be continous.",
+                            previousMax.f1 + 1, tuple2.f0.f1.longValue());
+				}
+				previousMax = tuple2.f1;
+			}
+		}
+	}
+
+	@Test
+	public void testRangePartitionerWithKeySelectorOnSequenceNestedDataWithOrders() throws Exception {
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		final DataSet<Tuple2<ComparablePojo, Long>> dataSet = env.generateSequence(0, 10000)
+				.map(new MapFunction<Long, Tuple2<ComparablePojo, Long>>() {
+					@Override
+					public Tuple2<ComparablePojo, Long> map(Long value) throws Exception {
+						return new Tuple2<>(new ComparablePojo(value / 5000, value % 5000), value);
+					}
+				});
+
+		final List<Tuple2<ComparablePojo, ComparablePojo>> collected = dataSet
+				.partitionByRange(new KeySelector<Tuple2<ComparablePojo, Long>, ComparablePojo>() {
+					@Override
+					public ComparablePojo getKey(Tuple2<ComparablePojo, Long> value) throws Exception {
+						return value.f0;
+					}
+				})
+				.withOrders(Order.ASCENDING)
+				.mapPartition(new MinMaxSelector<>(new ComparablePojoComparator()))
+				.mapPartition(new ExtractComparablePojo())
+				.collect();
+
+		final Comparator<Tuple2<ComparablePojo, ComparablePojo>> pojoComparator =
+				new Comparator<Tuple2<ComparablePojo, ComparablePojo>>() {
+			@Override
+			public int compare(Tuple2<ComparablePojo, ComparablePojo> o1,
+							   Tuple2<ComparablePojo, ComparablePojo> o2) {
+				return o1.f0.compareTo(o2.f1);
+			}
+		};
+		Collections.sort(collected, pojoComparator);
+
+		ComparablePojo previousMax = null;
+		for (Tuple2<ComparablePojo, ComparablePojo> element : collected) {
+			assertTrue("Min element in each partition should be smaller than max.",
+					element.f0.compareTo(element.f1) <= 0);
+			if (previousMax == null) {
+				previousMax = element.f1;
+			} else {
+				assertTrue("Partitions overlap. Previous max should be smaller than current min.",
+						previousMax.compareTo(element.f0) < 0);
+				if (previousMax.first.equals(element.f0.first)) {
+					assertEquals("Ordering on the second field should be continous.",
+							previousMax.second - 1, element.f0.second.longValue());
+				}
+				previousMax = element.f1;
+			}
+		}
+	}
+
+	private static class ExtractComparablePojo implements MapPartitionFunction<
+			Tuple2<Tuple2<ComparablePojo, Long>, Tuple2<ComparablePojo, Long>>,
+			Tuple2<ComparablePojo, ComparablePojo>> {
+
+		@Override
+		public void mapPartition(Iterable<Tuple2<Tuple2<ComparablePojo, Long>, Tuple2<ComparablePojo, Long>>> values,
+								 Collector<Tuple2<ComparablePojo, ComparablePojo>> out) throws Exception {
+			for (Tuple2<Tuple2<ComparablePojo, Long>, Tuple2<ComparablePojo, Long>> value : values) {
+				out.collect(new Tuple2<>(value.f0.f0, value.f1.f0));
+			}
+		}
+	}
+
+    private static class ComparablePojoComparator implements Comparator<Tuple2<ComparablePojo, Long>>, Serializable {
+
+		@Override
+		public int compare(Tuple2<ComparablePojo, Long> o1,
+						   Tuple2<ComparablePojo, Long> o2) {
+			return o1.f0.compareTo(o2.f0);
+		}
+	}
+
+	private static class ComparablePojo implements Comparable<ComparablePojo> {
+		private Long first;
+		private Long second;
+
+		public Long getFirst() {
+			return first;
+		}
+
+		public void setFirst(Long first) {
+			this.first = first;
+		}
+
+		public Long getSecond() {
+			return second;
+		}
+
+		public void setSecond(Long second) {
+			this.second = second;
+		}
+
+		public ComparablePojo(Long first,
+							  Long second) {
+			this.first = first;
+			this.second = second;
+		}
+
+		public ComparablePojo() {
+		}
+
+		@Override
+		public int compareTo(ComparablePojo o) {
+			final int firstResult = Long.compare(this.first, o.first);
+			if (firstResult == 0) {
+				return (-1) * Long.compare(this.second, o.second);
+			}
+
+			return firstResult;
+		}
+	}
+
 	private static class ObjectSelfKeySelector implements KeySelector<Long, Long> {
 		@Override
 		public Long getKey(Long value) throws Exception {
@@ -553,36 +762,61 @@ public class PartitionITCase extends MultipleProgramsTestBase {
 		}
 	}
 
-	private static class MinMaxSelector implements MapPartitionFunction<Long, Tuple2<Long, Long>> {
+	private static class MinMaxSelector<T> implements MapPartitionFunction<T, Tuple2<T, T>> {
+
+		private final Comparator<T> comparator;
+
+		public MinMaxSelector(Comparator<T> comparator) {
+			this.comparator = comparator;
+		}
+
 		@Override
-		public void mapPartition(Iterable<Long> values, Collector<Tuple2<Long, Long>> out) throws Exception {
-			long max = Long.MIN_VALUE;
-			long min = Long.MAX_VALUE;
-			for (long value : values) {
-				if (value > max) {
+		public void mapPartition(Iterable<T> values, Collector<Tuple2<T, T>> out) throws Exception {
+			Iterator<T> itr = values.iterator();
+			T min = itr.next();
+			T max = min;
+			T value;
+			while (itr.hasNext()) {
+				value= itr.next();
+				if (comparator.compare(value, min) < 0) {
+					min = value;
+				}
+				if (comparator.compare(value, max) > 0) {
 					max = value;
 				}
 
-				if (value < min) {
-					min = value;
-				}
 			}
-			Tuple2<Long, Long> result = new Tuple2<>(min, max);
+
+			Tuple2<T, T> result = new Tuple2<>(min, max);
 			out.collect(result);
 		}
 	}
 
-	private static class Tuple2Comparator implements Comparator<Tuple2<Long, Long>> {
+	private static class Tuple2Comparator<T> implements Comparator<Tuple2<T, T>>, Serializable {
+
+		private final Comparator<T> firstComparator;
+		private final Comparator<T> secondComparator;
+
+		public Tuple2Comparator(Comparator<T> comparator) {
+			this(comparator, comparator);
+		}
+
+		public Tuple2Comparator(Comparator<T> firstComparator,
+								Comparator<T> secondComparator) {
+			this.firstComparator = firstComparator;
+			this.secondComparator = secondComparator;
+		}
+
 		@Override
-		public int compare(Tuple2<Long, Long> first, Tuple2<Long, Long> second) {
-			long result = first.f0 - second.f0;
+		public int compare(Tuple2<T, T> first, Tuple2<T, T> second) {
+			long result = firstComparator.compare(first.f0, second.f0);
 			if (result > 0) {
 				return 1;
 			} else if (result < 0) {
 				return -1;
 			}
 
-			result = first.f1 - second.f1;
+			result = secondComparator.compare(first.f1, second.f1);
 			if (result > 0) {
 				return 1;
 			} else if (result < 0) {
@@ -593,4 +827,22 @@ public class PartitionITCase extends MultipleProgramsTestBase {
 		}
 	}
 
+	private static class LongComparator implements Comparator<Long>, Serializable {
+
+		private final boolean ascending;
+
+		public LongComparator(boolean ascending) {
+			this.ascending = ascending;
+		}
+
+		@Override
+		public int compare(Long o1, Long o2) {
+			if (ascending) {
+				return Long.compare(o1, o2);
+			} else {
+				return (-1) * Long.compare(o1, o2);
+			}
+		}
+	}
+
 }


[2/3] flink git commit: [FLINK-2998] [dataSet] Add support for explicit range partitioning for joins and coGroup.

Posted by fh...@apache.org.
[FLINK-2998] [dataSet] Add support for explicit range partitioning for joins and coGroup.

This closes #1838


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/605b6d87
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/605b6d87
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/605b6d87

Branch: refs/heads/master
Commit: 605b6d870d0da38fb5446675709c7243127cdff1
Parents: d8fb230
Author: gallenvara <ga...@126.com>
Authored: Tue Mar 29 22:36:21 2016 +0800
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Apr 20 22:55:49 2016 +0200

----------------------------------------------------------------------
 .../api/java/operators/PartitionOperator.java   |   5 +-
 .../dataproperties/GlobalProperties.java        |   3 +
 .../operators/AbstractJoinDescriptor.java       |   9 +-
 .../optimizer/operators/CoGroupDescriptor.java  |  11 +-
 .../operators/OperatorDescriptorDual.java       |  23 ++++
 ...oGroupGlobalPropertiesCompatibilityTest.java | 124 +++++++++++++++++++
 .../operators/CoGroupWithDistributionTest.java  |  99 +++++++++++++++
 .../JoinGlobalPropertiesCompatibilityTest.java  | 123 ++++++++++++++++++
 .../operators/JoinWithDistributionTest.java     |  97 +++++++++++++++
 .../optimizer/operators/TestDistribution.java   |  71 +++++++++++
 .../operators/shipping/OutputEmitter.java       |   4 +-
 .../test/javaApiOperators/CoGroupITCase.java    |  81 +++++++++++-
 .../CustomDistributionITCase.java               |  77 +++++++++++-
 .../flink/test/javaApiOperators/JoinITCase.java |  85 ++++++++++++-
 14 files changed, 792 insertions(+), 20 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
index fb6d579..2ed0300 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/PartitionOperator.java
@@ -88,8 +88,9 @@ public class PartitionOperator<T> extends SingleInputOperator<T, T, PartitionOpe
 		Preconditions.checkArgument(distribution == null || pMethod == PartitionMethod.RANGE, "Customized data distribution is only neccessary for range partition.");
 		
 		if (distribution != null) {
-			Preconditions.checkArgument(distribution.getNumberOfFields() == pKeys.getNumberOfKeyFields(), "The number of key fields in the distribution and range partitioner should be the same.");
-			Preconditions.checkArgument(Arrays.equals(distribution.getKeyTypes(), pKeys.getKeyFieldTypes()), "The types of key from the distribution and range partitioner are not equal.");
+			Preconditions.checkArgument(pKeys.getNumberOfKeyFields() <= distribution.getNumberOfFields(), "The distribution must provide at least as many fields as flat key fields are specified.");
+			Preconditions.checkArgument(Arrays.equals(pKeys.getKeyFieldTypes(), Arrays.copyOfRange(distribution.getKeyTypes(), 0, pKeys.getNumberOfKeyFields())),
+					"The types of the flat key fields must be equal to the types of the fields of the distribution.");
 		}
 		
 		if (customPartitioner != null) {

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
index ca17c2b..e64782f 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/dataproperties/GlobalProperties.java
@@ -311,6 +311,7 @@ public class GlobalProperties implements Cloneable {
 					gp.partitioning = PartitioningProperty.RANGE_PARTITIONED;
 					gp.ordering = newOrdering;
 					gp.partitioningFields = newOrdering.getInvolvedIndexes();
+					gp.distribution = this.distribution;
 				}
 				break;
 			case HASH_PARTITIONED:
@@ -436,6 +437,7 @@ public class GlobalProperties implements Cloneable {
 				throw new CompilerException("Unsupported partitioning strategy");
 		}
 
+		channel.setDataDistribution(this.distribution);
 		DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
 		channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
 	}
@@ -490,6 +492,7 @@ public class GlobalProperties implements Cloneable {
 		newProps.partitioning = this.partitioning;
 		newProps.partitioningFields = this.partitioningFields;
 		newProps.ordering = this.ordering;
+		newProps.distribution = this.distribution;
 		newProps.customPartitioner = this.customPartitioner;
 		newProps.uniqueFieldCombinations = this.uniqueFieldCombinations == null ? null : new HashSet<FieldSet>(this.uniqueFieldCombinations);
 		return newProps;

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
index 6c2776e..1f2f42a 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/AbstractJoinDescriptor.java
@@ -140,11 +140,12 @@ public abstract class AbstractJoinDescriptor extends OperatorDescriptorDual {
 
 			}
 			else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
-					produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
+					produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+					produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {
 
-				// Return false anyway now, we need both the partition key and data distribution
-				// information to make sure whether the range partitions are equivalent.
-				return false;
+				return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
+						checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
+						produced1.getDataDistribution().equals(produced2.getDataDistribution());
 
 			}
 			else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
index b99b1c1..0e3fb55 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/CoGroupDescriptor.java
@@ -149,12 +149,13 @@ public class CoGroupDescriptor extends OperatorDescriptorDual {
 
 		}
 		else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
-				produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED) {
-
-			// Return false anyway now, we need both the partition key and data distribution
-			// information to make sure whether the range partitions are equivalent.
-			return false;
+				produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
+				produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {
 
+			return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
+					checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
+					produced1.getDataDistribution().equals(produced2.getDataDistribution());
+			
 		}
 		else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
 				produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
index 17ea8a5..e5e43ca 100644
--- a/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
+++ b/flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/OperatorDescriptorDual.java
@@ -125,6 +125,29 @@ public abstract class OperatorDescriptorDual implements AbstractOperatorDescript
 		return true;
 	}
 
+	protected boolean checkSameOrdering(GlobalProperties produced1, GlobalProperties produced2, int numRelevantFields) {
+		Ordering prod1 = produced1.getPartitioningOrdering();
+		Ordering prod2 = produced2.getPartitioningOrdering();
+
+		if (prod1 == null || prod2 == null) {
+			throw new CompilerException("The given properties do not meet this operators requirements.");
+		}
+
+		// check that order of fields is equivalent
+		if (!checkEquivalentFieldPositionsInKeyFields(
+				prod1.getInvolvedIndexes(), prod2.getInvolvedIndexes(), numRelevantFields)) {
+			return false;
+		}
+
+		// check that both inputs have the same directions of order
+		for (int i = 0; i < numRelevantFields; i++) {
+			if (prod1.getOrder(i) != prod2.getOrder(i)) {
+				return false;
+			}
+		}
+		return true;
+	}
+
 	protected boolean checkSameOrdering(LocalProperties produced1, LocalProperties produced2, int numRelevantFields) {
 		Ordering prod1 = produced1.getOrdering();
 		Ordering prod2 = produced2.getOrdering();

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
index 23f8897..76f48e9 100644
--- a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupGlobalPropertiesCompatibilityTest.java
@@ -21,6 +21,8 @@ package org.apache.flink.optimizer.operators;
 import static org.junit.Assert.*;
 
 import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
 import org.apache.flink.api.common.operators.util.FieldList;
 import org.apache.flink.optimizer.dataproperties.GlobalProperties;
 import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
@@ -95,6 +97,52 @@ public class CoGroupGlobalPropertiesCompatibilityTest {
 				
 				assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
 			}
+
+			TestDistribution dist1 = new TestDistribution(1);
+			TestDistribution dist2 = new TestDistribution(1);
+			
+			// test compatible range partitioning with one ordering
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering2.appendOrdering(field, null, Order.ASCENDING);
+				}
+				
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
+			// test compatible range partitioning with two orderings
+			{
+				Ordering ordering1 = new Ordering();
+				ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
+				ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
+				Ordering ordering2 = new Ordering();
+				ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
+				ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
 		}
 		catch (Exception e) {
 			e.printStackTrace();
@@ -152,6 +200,82 @@ public class CoGroupGlobalPropertiesCompatibilityTest {
 				
 				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
 			}
+
+			TestDistribution dist1 = new TestDistribution(1);
+			TestDistribution dist2 = new TestDistribution(1);
+
+			// test incompatible range partitioning with different key size
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+					ordering2.appendOrdering(field, null, Order.ASCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
+
+			// test incompatible range partitioning with different ordering
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering2.appendOrdering(field, null, Order.DESCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
+
+			TestDistribution dist3 = new TestDistribution(1);
+			TestDistribution dist4 = new TestDistribution(2);
+
+			// test incompatible range partitioning with different distribution
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering2.appendOrdering(field, null, Order.ASCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist3);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist4);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist3);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist4);
+				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
 		}
 		catch (Exception e) {
 			e.printStackTrace();

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java
new file mode 100644
index 0000000..2d37c7c
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/CoGroupWithDistributionTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.CoGroupFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.utils.DataSetUtils;
+import org.apache.flink.optimizer.plan.*;
+import org.apache.flink.optimizer.util.CompilerTestBase;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.apache.flink.util.Collector;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class CoGroupWithDistributionTest extends CompilerTestBase {
+
+	@Test
+	 public void CoGroupWithSameDistributionTest() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+		DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+		TestDistribution testDistribution1 = new TestDistribution(3);
+		TestDistribution testDistribution2 = new TestDistribution(3);
+
+		DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, testDistribution1, 0)
+				.coGroup(DataSetUtils.partitionByRange(set2, testDistribution2, 0))
+				.where(0).equalTo(0).with(new CoGroupFunc());
+
+		coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+		Plan plan = env.createProgramPlan();
+		OptimizedPlan oPlan = compileWithStats(plan);
+
+		SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+		DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+		Channel input1 = coGroup.getInput1();
+		Channel input2 = coGroup.getInput2();
+		assertEquals(ShipStrategyType.FORWARD, input1.getShipStrategy());
+		assertEquals(ShipStrategyType.FORWARD, input2.getShipStrategy());
+
+	}
+
+	@Test
+	public void CoGroupWithDifferentDistributionTest() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+		DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+		TestDistribution testDistribution1 = new TestDistribution(3);
+		TestDistribution testDistribution2 = new TestDistribution(2);
+
+		DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, testDistribution1, 0)
+				.coGroup(DataSetUtils.partitionByRange(set2, testDistribution2, 0))
+				.where(0).equalTo(0).with(new CoGroupFunc());
+
+		coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+		Plan plan = env.createProgramPlan();
+		OptimizedPlan oPlan = compileWithStats(plan);
+
+		SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+		DualInputPlanNode coGroup= (DualInputPlanNode)sink.getInput().getSource();
+		Channel input1 = coGroup.getInput1();
+		Channel input2 = coGroup.getInput2();
+		assertEquals(ShipStrategyType.PARTITION_HASH, input1.getShipStrategy());
+		assertEquals(ShipStrategyType.PARTITION_HASH, input2.getShipStrategy());
+
+	}
+
+	public static class CoGroupFunc implements CoGroupFunction<Tuple3<Integer, Integer, Integer>,
+			Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+		@Override
+		public void coGroup(Iterable<Tuple3<Integer, Integer, Integer>> first, Iterable<Tuple3<Integer, Integer, Integer>> second,
+							Collector<Tuple3<Integer, Integer, Integer>> out) throws Exception {
+
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
index 1d559c2..610cbd8 100644
--- a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinGlobalPropertiesCompatibilityTest.java
@@ -21,6 +21,8 @@ package org.apache.flink.optimizer.operators;
 import static org.junit.Assert.*;
 
 import org.apache.flink.api.common.functions.Partitioner;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
 import org.apache.flink.api.common.operators.util.FieldList;
 import org.apache.flink.optimizer.dataproperties.GlobalProperties;
 import org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties;
@@ -95,6 +97,51 @@ public class JoinGlobalPropertiesCompatibilityTest {
 				
 				assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
 			}
+
+			TestDistribution dist1 = new TestDistribution(1);
+			TestDistribution dist2 = new TestDistribution(1);
+			// test compatible range partitioning with one ordering
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering2.appendOrdering(field, null, Order.ASCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
+			// test compatible range partitioning with two orderings
+			{
+				Ordering ordering1 = new Ordering();
+				ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
+				ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
+				Ordering ordering2 = new Ordering();
+				ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
+				ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
 		}
 		catch (Exception e) {
 			e.printStackTrace();
@@ -152,6 +199,82 @@ public class JoinGlobalPropertiesCompatibilityTest {
 				
 				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
 			}
+
+			TestDistribution dist1 = new TestDistribution(1);
+			TestDistribution dist2 = new TestDistribution(1);
+
+			// test incompatible range partitioning with different key size
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+					ordering2.appendOrdering(field, null, Order.ASCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
+
+			// test incompatible range partitioning with different ordering
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering2.appendOrdering(field, null, Order.DESCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist1);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist2);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist1);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist2);
+				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
+
+			TestDistribution dist3 = new TestDistribution(1);
+			TestDistribution dist4 = new TestDistribution(2);
+
+			// test incompatible range partitioning with different distribution
+			{
+				Ordering ordering1 = new Ordering();
+				for (int field : keysLeft) {
+					ordering1.appendOrdering(field, null, Order.ASCENDING);
+				}
+				Ordering ordering2 = new Ordering();
+				for (int field : keysRight) {
+					ordering2.appendOrdering(field, null, Order.ASCENDING);
+				}
+
+				RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
+				reqLeft.setRangePartitioned(ordering1, dist3);
+				RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
+				reqRight.setRangePartitioned(ordering2, dist4);
+
+				GlobalProperties propsLeft = new GlobalProperties();
+				propsLeft.setRangePartitioned(ordering1, dist3);
+				GlobalProperties propsRight = new GlobalProperties();
+				propsRight.setRangePartitioned(ordering2, dist4);
+				assertFalse(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
+			}
 		}
 		catch (Exception e) {
 			e.printStackTrace();

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java
new file mode 100644
index 0000000..0c579f6
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/JoinWithDistributionTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.Plan;
+import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.ExecutionEnvironment;
+import org.apache.flink.api.java.io.DiscardingOutputFormat;
+import org.apache.flink.api.java.tuple.Tuple3;
+import org.apache.flink.api.java.utils.DataSetUtils;
+import org.apache.flink.optimizer.plan.*;
+import org.apache.flink.optimizer.util.CompilerTestBase;
+import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class JoinWithDistributionTest extends CompilerTestBase {
+
+	@Test
+	public void JoinWithSameDistributionTest() throws Exception {
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+		DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+		TestDistribution dist1 = new TestDistribution(3);
+		TestDistribution dist2 = new TestDistribution(3);
+
+		DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, dist1, 0)
+				.join(DataSetUtils.partitionByRange(set2, dist2, 0))
+				.where(0).equalTo(0).with(new JoinFunc());
+
+		coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+		Plan plan = env.createProgramPlan();
+		OptimizedPlan oPlan = compileWithStats(plan);
+
+		SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+		DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+		Channel input1 = join.getInput1();
+		Channel input2 = join.getInput2();
+		assertEquals(ShipStrategyType.FORWARD, input1.getShipStrategy());
+		assertEquals(ShipStrategyType.FORWARD, input2.getShipStrategy());
+
+	}
+
+	@Test
+	public void JoinWithDifferentDistributionTest() throws Exception{
+		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+		DataSet<Tuple3<Integer, Integer, Integer>> set1 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+		DataSet<Tuple3<Integer, Integer, Integer>> set2 = env.readCsvFile(IN_FILE).types(Integer.class, Integer.class, Integer.class);
+
+		TestDistribution dist1 = new TestDistribution(3);
+		TestDistribution dist2 = new TestDistribution(4);
+
+		DataSet<Tuple3<Integer, Integer, Integer>> coGrouped = DataSetUtils.partitionByRange(set1, dist1, 0)
+				.join(DataSetUtils.partitionByRange(set2, dist2, 0))
+				.where(0).equalTo(0).with(new JoinFunc());
+
+		coGrouped.output(new DiscardingOutputFormat<Tuple3<Integer, Integer, Integer>>());
+		Plan plan = env.createProgramPlan();
+		OptimizedPlan oPlan = compileWithStats(plan);
+
+		SinkPlanNode sink = oPlan.getDataSinks().iterator().next();
+		DualInputPlanNode join = (DualInputPlanNode)sink.getInput().getSource();
+		Channel input1 = join.getInput1();
+		Channel input2 = join.getInput2();
+		assertEquals(ShipStrategyType.PARTITION_HASH, input1.getShipStrategy());
+		assertEquals(ShipStrategyType.PARTITION_HASH, input2.getShipStrategy());
+
+	}
+
+	public static class JoinFunc implements JoinFunction<Tuple3<Integer, Integer, Integer>,
+			Tuple3<Integer, Integer, Integer>, Tuple3<Integer, Integer, Integer>> {
+
+		@Override
+		public Tuple3<Integer, Integer, Integer> join(Tuple3<Integer, Integer, Integer> first, Tuple3<Integer, Integer, Integer> second) throws Exception {
+			return null;
+		}
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java
----------------------------------------------------------------------
diff --git a/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java
new file mode 100644
index 0000000..d99930b
--- /dev/null
+++ b/flink-optimizer/src/test/java/org/apache/flink/optimizer/operators/TestDistribution.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.optimizer.operators;
+
+import org.apache.flink.api.common.distributions.DataDistribution;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+
+import java.io.IOException;
+
+public class TestDistribution implements DataDistribution {
+
+	public int boundary;
+
+	public TestDistribution(int boundary) {
+		this.boundary = boundary;
+	}
+
+	@Override
+	public Object[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+		return new Object[0];
+	}
+
+	@Override
+	public int getNumberOfFields() {
+		return 1;
+	}
+
+	@Override
+	public TypeInformation[] getKeyTypes() {
+		return new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO};
+	}
+
+	@Override
+	public void write(DataOutputView out) throws IOException {
+
+	}
+
+	@Override
+	public void read(DataInputView in) throws IOException {
+
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		boolean isEqual = true;
+		TestDistribution dist = (TestDistribution)obj;
+		if (this.boundary != dist.boundary) {
+			isEqual = false;
+		}
+		return isEqual;
+	}
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
index 4e20842..e6f3d26 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/operators/shipping/OutputEmitter.java
@@ -258,11 +258,11 @@ public class OutputEmitter<T> implements ChannelSelector<SerializationDelegate<T
 	private final int compareRecordAndBoundary(T record, Object[] boundary) {
 		this.comparator.extractKeys(record, keys, 0);
 
-		if (flatComparators.length != keys.length || flatComparators.length != boundary.length) {
+		if (flatComparators.length != keys.length || flatComparators.length > boundary.length) {
 			throw new RuntimeException("Can not compare keys with boundary due to mismatched length.");
 		}
 
-		for (int i=0; i<flatComparators.length; i++) {
+		for (int i = 0; i < flatComparators.length; i++) {
 			int result = flatComparators[i].compare(keys[i], boundary[i]);
 			if (result != 0) {
 				return result;

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
index 7bc8480..5b7caa7 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CoGroupITCase.java
@@ -18,8 +18,11 @@
 
 package org.apache.flink.test.javaApiOperators;
 
+import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.CoGroupFunction;
 import org.apache.flink.api.common.functions.RichCoGroupFunction;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.DataSet;
 import org.apache.flink.api.java.ExecutionEnvironment;
 import org.apache.flink.api.java.functions.KeySelector;
@@ -28,7 +31,10 @@ import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.utils.DataSetUtils;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
 import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
 import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
@@ -40,6 +46,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -286,7 +293,7 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
 		DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
 
 		DataSet<Tuple3<Integer, Long, String>> coGrouped = ds1.coGroup(ds2).
-				where(0,4).equalTo(0,1).with(new Tuple5Tuple3CoGroup());
+				where(0, 4).equalTo(0, 1).with(new Tuple5Tuple3CoGroup());
 
 		List<Tuple3<Integer, Long, String>> result = coGrouped.collect();
 		
@@ -516,6 +523,37 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
 		compareResultAsText(result, expected);
 	}
 
+	@Test
+	public void testCoGroupWithRangePartitioning() throws Exception {
+		/*
+		 * Test coGroup on tuples with multiple key field positions and same customized distribution
+		 */
+
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+		DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds1 = CollectionDataSets.get5TupleDataSet(env);
+		DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.get3TupleDataSet(env);
+
+		env.setParallelism(4);
+		TestDistribution testDis = new TestDistribution();
+		DataSet<Tuple3<Integer, Long, String>> coGrouped =
+				DataSetUtils.partitionByRange(ds1, testDis, 0, 4)
+						.coGroup(DataSetUtils.partitionByRange(ds2, testDis, 0, 1))
+						.where(0, 4)
+						.equalTo(0, 1)
+						.with(new Tuple5Tuple3CoGroup());
+
+		List<Tuple3<Integer, Long, String>> result = coGrouped.collect();
+
+		String expected = "1,1,Hallo\n" +
+				"2,2,Hallo Welt\n" +
+				"3,2,Hallo Welt wie gehts?\n" +
+				"3,2,ABC\n" +
+				"5,3,HIJ\n" +
+				"5,3,IJK\n";
+
+		compareResultAsTuples(result, expected);
+	}
 
 
 	// --------------------------------------------------------------------------------------------
@@ -797,4 +835,45 @@ public class CoGroupITCase extends MultipleProgramsTestBase {
 			}
 		}
 	}
+
+	public static class TestDistribution implements DataDistribution {
+		public Object[][] boundaries = new Object[][]{
+				new Object[]{2, 2L},
+				new Object[]{5, 4L},
+				new Object[]{10, 12L},
+				new Object[]{21, 6L}
+		};
+
+		public TestDistribution() {}
+
+		@Override
+		public Object[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+			return boundaries[bucketNum];
+		}
+
+		@Override
+		public int getNumberOfFields() {
+			return 2;
+		}
+
+		@Override
+		public TypeInformation[] getKeyTypes() {
+			return new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO};
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+
+		}
+
+		@Override
+		public void read(DataInputView in) throws IOException {
+
+		}
+
+		@Override
+		public boolean equals(Object obj) {
+			return obj instanceof TestDistribution; 
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
index 427e83e..c6bc08e 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/CustomDistributionITCase.java
@@ -49,13 +49,13 @@ public class CustomDistributionITCase {
 
 		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
 
-		DataSet<Tuple3<Integer, Long, String>> input1 = CollectionDataSets.get3TupleDataSet(env);
+		DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
 		final TestDataDist1 dist = new TestDataDist1();
 
 		env.setParallelism(dist.getParallelism());
 
 		DataSet<Boolean> result = DataSetUtils
-			.partitionByRange(input1, dist, 0)
+			.partitionByRange(input, dist, 0)
 			.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, Long, String>, Boolean>() {
 
 				@Override
@@ -104,7 +104,7 @@ public class CustomDistributionITCase {
 
 		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
 
-		DataSet<Tuple3<Integer, Integer, String>> input1 = env.fromElements(
+		DataSet<Tuple3<Integer, Integer, String>> input = env.fromElements(
 						new Tuple3<>(1, 5, "Hi"),
 						new Tuple3<>(1, 6, "Hi"),
 						new Tuple3<>(1, 7, "Hi"),
@@ -127,7 +127,7 @@ public class CustomDistributionITCase {
 		env.setParallelism(dist.getParallelism());
 
 		DataSet<Boolean> result = DataSetUtils
-			.partitionByRange(input1, dist, 0, 1)
+			.partitionByRange(input, dist, 0, 1)
 			.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, Integer, String>, Boolean>() {
 
 				@Override
@@ -175,6 +175,75 @@ public class CustomDistributionITCase {
 		env.execute();
 	}
 
+	@Test
+	public void testPartitionKeyLessDistribution() throws Exception{
+		/*
+		 * Test the number of partition keys less than the number of distribution fields
+		 */
+		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+
+		DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
+		final TestDataDist2 dist = new TestDataDist2();
+
+		env.setParallelism(dist.getParallelism());
+
+		DataSet<Boolean> result = DataSetUtils
+			.partitionByRange(input, dist, 0)
+			.mapPartition(new RichMapPartitionFunction<Tuple3<Integer, Long, String>, Boolean>() {
+
+				@Override
+				public void mapPartition(Iterable<Tuple3<Integer, Long, String>> values, Collector<Boolean> out) throws Exception {
+					int pIdx = getRuntimeContext().getIndexOfThisSubtask();
+
+					for (Tuple3<Integer, Long, String> s : values) {
+						boolean correctlyPartitioned = true;
+						if (pIdx == 0) {
+							Integer[] upper = dist.boundaries[0];
+							if (s.f0.compareTo(upper[0]) > 0) {
+								correctlyPartitioned = false;
+							}
+						}
+						else if (pIdx > 0 && pIdx < dist.getParallelism() - 1) {
+							Integer[] lower = dist.boundaries[pIdx - 1];
+							Integer[] upper = dist.boundaries[pIdx];
+							if (s.f0.compareTo(upper[0]) > 0 || (s.f0.compareTo(lower[0]) <= 0)) {
+								correctlyPartitioned = false;
+							}
+						}
+						else {
+							Integer[] lower = dist.boundaries[pIdx - 1];
+							if ((s.f0.compareTo(lower[0]) <= 0)) {
+								correctlyPartitioned = false;
+							}
+						}
+
+						if (!correctlyPartitioned) {
+							fail("Record was not correctly partitioned: " + s.toString());
+						}
+					}
+				}
+			}
+			);
+
+		result.output(new DiscardingOutputFormat<Boolean>());
+		env.execute();
+	}
+
+	@Test(expected = IllegalArgumentException.class)
+	public void testPartitionMoreThanDistribution() throws Exception{
+		/*
+		 * Test the number of partition keys larger than the number of distribution fields
+		 */
+
+		ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
+
+		DataSet<Tuple3<Integer, Long, String>> input = CollectionDataSets.get3TupleDataSet(env);
+		final TestDataDist2 dist = new TestDataDist2();
+
+		DataSet<Tuple3<Integer, Long, String>> result = DataSetUtils
+				.partitionByRange(input, dist, 0, 1, 2);
+	}
+	
 	/**
 	 * The class is used to do the tests of range partition with one key.
 	 */

http://git-wip-us.apache.org/repos/asf/flink/blob/605b6d87/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
----------------------------------------------------------------------
diff --git a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
index 61e07fe..0d8c80b 100644
--- a/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
+++ b/flink-tests/src/test/java/org/apache/flink/test/javaApiOperators/JoinITCase.java
@@ -18,11 +18,15 @@
 
 package org.apache.flink.test.javaApiOperators;
 
+import java.io.IOException;
 import java.util.Collection;
 import java.util.List;
 
+import org.apache.flink.api.common.distributions.DataDistribution;
 import org.apache.flink.api.common.functions.FlatJoinFunction;
 import org.apache.flink.api.common.functions.JoinFunction;
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.api.java.functions.KeySelector;
 import org.apache.flink.api.common.functions.RichFlatJoinFunction;
 import org.apache.flink.api.java.tuple.Tuple2;
@@ -30,7 +34,10 @@ import org.apache.flink.api.java.tuple.Tuple3;
 import org.apache.flink.api.java.tuple.Tuple5;
 import org.apache.flink.api.java.tuple.Tuple6;
 import org.apache.flink.api.java.tuple.Tuple7;
+import org.apache.flink.api.java.utils.DataSetUtils;
 import org.apache.flink.configuration.Configuration;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
 import org.apache.flink.test.javaApiOperators.util.CollectionDataSets;
 import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.CustomType;
 import org.apache.flink.test.javaApiOperators.util.CollectionDataSets.POJO;
@@ -87,8 +94,8 @@ public class JoinITCase extends MultipleProgramsTestBase {
 		DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
 		DataSet<Tuple2<String, String>> joinDs =
 				ds1.join(ds2)
-				.where(0,1)
-				.equalTo(0,4)
+				.where(0, 1)
+				.equalTo(0, 4)
 				.with(new T3T5FlatJoin());
 
 		List<Tuple2<String, String>> result = joinDs.collect();
@@ -680,6 +687,7 @@ public class JoinITCase extends MultipleProgramsTestBase {
 		compareResultAsTuples(result, expected);
 	}
 
+	@Test
 	public void testJoinWithAtomicType2() throws Exception {
 		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
 
@@ -696,6 +704,38 @@ public class JoinITCase extends MultipleProgramsTestBase {
 		compareResultAsTuples(result, expected);
 	}
 
+	@Test
+	public void testJoinWithRangePartitioning() throws Exception {
+		/*
+		 * Test Join on tuples with multiple key field positions and same customized distribution
+		 */
+
+		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
+
+		DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.get3TupleDataSet(env);
+		DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env);
+
+		env.setParallelism(4);
+		TestDistribution testDis = new TestDistribution();
+		DataSet<Tuple2<String, String>> joinDs =
+				DataSetUtils.partitionByRange(ds1, testDis, 0, 1)
+						.join(DataSetUtils.partitionByRange(ds2, testDis, 0, 4))
+						.where(0, 1)
+						.equalTo(0, 4)
+						.with(new T3T5FlatJoin());
+
+		List<Tuple2<String, String>> result = joinDs.collect();
+
+		String expected = "Hi,Hallo\n" +
+				"Hello,Hallo Welt\n" +
+				"Hello world,Hallo Welt wie gehts?\n" +
+				"Hello world,ABC\n" +
+				"I am fine.,HIJ\n" +
+				"I am fine.,IJK\n";
+
+		compareResultAsTuples(result, expected);
+	}
+
 	public static class T3T5FlatJoin implements FlatJoinFunction<Tuple3<Integer, Long, String>, Tuple5<Integer, Long, Integer, String, Long>, Tuple2<String, String>> {
 
 		@Override
@@ -778,4 +818,45 @@ public class JoinITCase extends MultipleProgramsTestBase {
 			return new Tuple2<String, String>(first.myString, second.f2);
 		}
 	}
+	
+	public static class TestDistribution implements DataDistribution {
+		public Object boundaries[][] = new Object[][]{
+				new Object[]{2, 2L},
+				new Object[]{5, 4L},
+				new Object[]{10, 12L},
+				new Object[]{21, 6L}
+		};
+
+		public TestDistribution() {}
+
+		@Override
+		public Object[] getBucketBoundary(int bucketNum, int totalNumBuckets) {
+			return boundaries[bucketNum];
+		}
+
+		@Override
+		public int getNumberOfFields() {
+			return 2;
+		}
+
+		@Override
+		public TypeInformation[] getKeyTypes() {
+			return new TypeInformation[]{BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO};
+		}
+
+		@Override
+		public void write(DataOutputView out) throws IOException {
+
+		}
+
+		@Override
+		public void read(DataInputView in) throws IOException {
+
+		}
+		
+		@Override
+		public boolean equals(Object obj) {
+			return obj instanceof TestDistribution;
+		}
+	}
 }


[3/3] flink git commit: [FLINK-3781] Ensure BlobClient is closed.

Posted by fh...@apache.org.
[FLINK-3781] Ensure BlobClient is closed.

This closes #1908


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a43bade0
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a43bade0
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a43bade0

Branch: refs/heads/master
Commit: a43bade0d87d373ff236152d13d8d56e17220605
Parents: 605b6d8
Author: Chenguang He <ga...@gmail.com>
Authored: Mon Apr 18 13:52:45 2016 -0500
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Apr 20 23:00:56 2016 +0200

----------------------------------------------------------------------
 .../org/apache/flink/runtime/blob/BlobCache.java  | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/a43bade0/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
----------------------------------------------------------------------
diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
index 4d33364..bb0aacb 100644
--- a/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
+++ b/flink-runtime/src/main/java/org/apache/flink/runtime/blob/BlobCache.java
@@ -71,13 +71,13 @@ public final class BlobCache implements BlobService {
 
 		// configure the number of fetch retries
 		final int fetchRetries = configuration.getInteger(
-				ConfigConstants.BLOB_FETCH_RETRIES_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_RETRIES);
+			ConfigConstants.BLOB_FETCH_RETRIES_KEY, ConfigConstants.DEFAULT_BLOB_FETCH_RETRIES);
 		if (fetchRetries >= 0) {
 			this.numFetchRetries = fetchRetries;
 		}
 		else {
 			LOG.warn("Invalid value for {}. System will attempt no retires on failed fetches of BLOBs.",
-					ConfigConstants.BLOB_FETCH_RETRIES_KEY);
+				ConfigConstants.BLOB_FETCH_RETRIES_KEY);
 			this.numFetchRetries = 0;
 		}
 
@@ -89,7 +89,7 @@ public final class BlobCache implements BlobService {
 	 * Returns the URL for the BLOB with the given key. The method will first attempt to serve
 	 * the BLOB from its local cache. If the BLOB is not in the cache, the method will try to download it
 	 * from this cache's BLOB server.
-	 * 
+	 *
 	 * @param requiredBlob The key of the desired BLOB.
 	 * @return URL referring to the local storage location of the BLOB.
 	 * @throws IOException Thrown if an I/O error occurs while downloading the BLOBs from the BLOB server.
@@ -163,7 +163,7 @@ public final class BlobCache implements BlobService {
 				}
 				catch (IOException e) {
 					String message = "Failed to fetch BLOB " + requiredBlob + " from " + serverAddress +
-							" and store it under " + localJarFile.getAbsolutePath();
+						" and store it under " + localJarFile.getAbsolutePath();
 					if (attempt < numFetchRetries) {
 						attempt++;
 						if (LOG.isDebugEnabled()) {
@@ -200,10 +200,14 @@ public final class BlobCache implements BlobService {
 	 * @param key referring to the file to be deleted
 	 */
 	public void deleteGlobal(BlobKey key) throws IOException {
-		delete(key);
 		BlobClient bc = createClient();
-		bc.delete(key);
-		bc.close();
+		try {
+			delete(key);
+			bc.delete(key);
+		}
+		finally {
+			bc.close();
+		}
 	}
 
 	@Override