You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by fh...@apache.org on 2015/06/10 14:50:10 UTC

flink git commit: [FLINK-2196] [javaAPI] Moved misplaced SortPartitionOperator class

Repository: flink
Updated Branches:
  refs/heads/master b5b71e1c4 -> a8e085078


[FLINK-2196] [javaAPI] Moved misplaced SortPartitionOperator class

This closes #817


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/a8e08507
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/a8e08507
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/a8e08507

Branch: refs/heads/master
Commit: a8e085078499319113a213575c9f057144395f98
Parents: b5b71e1
Author: Fabian Hueske <fh...@apache.org>
Authored: Wed Jun 10 11:58:08 2015 +0200
Committer: Fabian Hueske <fh...@apache.org>
Committed: Wed Jun 10 14:49:15 2015 +0200

----------------------------------------------------------------------
 .../java/org/apache/flink/api/java/DataSet.java |   1 +
 .../flink/api/java/SortPartitionOperator.java   | 208 -------------------
 .../java/operators/SortPartitionOperator.java   | 207 ++++++++++++++++++
 .../org/apache/flink/api/scala/DataSet.scala    |   2 +-
 4 files changed, 209 insertions(+), 209 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/a8e08507/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
index f1eed3e..e217e53 100644
--- a/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
+++ b/flink-java/src/main/java/org/apache/flink/api/java/DataSet.java
@@ -71,6 +71,7 @@ import org.apache.flink.api.java.operators.PartitionOperator;
 import org.apache.flink.api.java.operators.ProjectOperator;
 import org.apache.flink.api.java.operators.ProjectOperator.Projection;
 import org.apache.flink.api.java.operators.ReduceOperator;
+import org.apache.flink.api.java.operators.SortPartitionOperator;
 import org.apache.flink.api.java.operators.SortedGrouping;
 import org.apache.flink.api.java.operators.UnionOperator;
 import org.apache.flink.api.java.operators.UnsortedGrouping;

http://git-wip-us.apache.org/repos/asf/flink/blob/a8e08507/flink-java/src/main/java/org/apache/flink/api/java/SortPartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/SortPartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/SortPartitionOperator.java
deleted file mode 100644
index 988144b..0000000
--- a/flink-java/src/main/java/org/apache/flink/api/java/SortPartitionOperator.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.api.java;
-
-import org.apache.flink.api.common.InvalidProgramException;
-import org.apache.flink.api.common.operators.Operator;
-import org.apache.flink.api.common.operators.Order;
-import org.apache.flink.api.common.operators.Ordering;
-import org.apache.flink.api.common.operators.UnaryOperatorInformation;
-import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeutils.CompositeType;
-import org.apache.flink.api.java.operators.Keys;
-import org.apache.flink.api.java.operators.SingleInputOperator;
-import org.apache.flink.api.java.typeutils.TupleTypeInfoBase;
-
-import java.util.Arrays;
-
-/**
- * This operator represents a DataSet with locally sorted partitions.
- *
- * @param <T> The type of the DataSet with locally sorted partitions.
- */
-public class SortPartitionOperator<T> extends SingleInputOperator<T, T, SortPartitionOperator<T>> {
-
-	private int[] sortKeyPositions;
-
-	private Order[] sortOrders;
-
-	private final String sortLocationName;
-
-
-	public SortPartitionOperator(DataSet<T> dataSet, int sortField, Order sortOrder, String sortLocationName) {
-		super(dataSet, dataSet.getType());
-		this.sortLocationName = sortLocationName;
-
-		int[] flatOrderKeys = getFlatFields(sortField);
-		this.appendSorting(flatOrderKeys, sortOrder);
-	}
-
-	public SortPartitionOperator(DataSet<T> dataSet, String sortField, Order sortOrder, String sortLocationName) {
-		super(dataSet, dataSet.getType());
-		this.sortLocationName = sortLocationName;
-
-		int[] flatOrderKeys = getFlatFields(sortField);
-		this.appendSorting(flatOrderKeys, sortOrder);
-	}
-
-	/**
-	 * Appends an additional sort order with the specified field in the specified order to the
-	 * local partition sorting of the DataSet.
-	 *
-	 * @param field The field index of the additional sort order of the local partition sorting.
-	 * @param order The order of the additional sort order of the local partition sorting.
-	 * @return The DataSet with sorted local partitions.
-	 */
-	public SortPartitionOperator<T> sortPartition(int field, Order order) {
-
-		int[] flatOrderKeys = getFlatFields(field);
-		this.appendSorting(flatOrderKeys, order);
-		return this;
-	}
-
-	/**
-	 * Appends an additional sort order with the specified field in the specified order to the
-	 * local partition sorting of the DataSet.
-	 *
-	 * @param field The field expression referring to the field of the additional sort order of
-	 *                 the local partition sorting.
-	 * @param order The order  of the additional sort order of the local partition sorting.
-	 * @return The DataSet with sorted local partitions.
-	 */
-	public SortPartitionOperator<T> sortPartition(String field, Order order) {
-		int[] flatOrderKeys = getFlatFields(field);
-		this.appendSorting(flatOrderKeys, order);
-		return this;
-	}
-
-	// --------------------------------------------------------------------------------------------
-	//  Key Extraction
-	// --------------------------------------------------------------------------------------------
-
-	private int[] getFlatFields(int field) {
-
-		if(!(super.getType() instanceof TupleTypeInfoBase<?>)) {
-			throw new InvalidProgramException("Field positions can only be specified on Tuple or " +
-					"Case Class types.");
-		}
-		else {
-			// check selected field is sortable type
-			TypeInformation<?> sortKeyType = ((TupleTypeInfoBase<?>) super.getType()).getTypeAt(field);
-			if (!sortKeyType.isSortKeyType()) {
-				throw new InvalidProgramException("Selected sort key is not a sortable type " + sortKeyType);
-			}
-		}
-
-		Keys.ExpressionKeys<T> ek;
-		try {
-			ek = new Keys.ExpressionKeys<T>(new int[]{field}, super.getType());
-		} catch(IllegalArgumentException iae) {
-			throw new InvalidProgramException("Invalid specification of field position.", iae);
-		}
-		return ek.computeLogicalKeyPositions();
-	}
-
-	private int[] getFlatFields(String fields) {
-
-		if(super.getType() instanceof CompositeType) {
-
-			// check selected field is sortable type
-			TypeInformation<?> sortKeyType = ((CompositeType<?>) super.getType()).getTypeAt(fields);
-			if (!sortKeyType.isSortKeyType()) {
-				throw new InvalidProgramException("Selected sort key is not a sortable type " + sortKeyType);
-			}
-
-			// compute flat field positions for (nested) sorting fields
-			Keys.ExpressionKeys<T> ek;
-			try {
-				ek = new Keys.ExpressionKeys<T>(new String[]{fields}, super.getType());
-			} catch(IllegalArgumentException iae) {
-				throw new InvalidProgramException("Invalid specification of field expression.", iae);
-			}
-			return ek.computeLogicalKeyPositions();
-		} else {
-
-			fields = fields.trim();
-			if (!(fields.equals("*") || fields.equals("_"))) {
-				throw new InvalidProgramException("Output sorting of non-composite types can only be defined on the full type. " +
-						"Use a field wildcard for that (\"*\" or \"_\")");
-			} else {
-
-				// check if selected field is sortable type
-				if (!super.getType().isSortKeyType()) {
-					throw new InvalidProgramException("Selected sort key cannot be sorted: " + super.getType());
-				}
-
-				return new int[]{0};
-			}
-		}
-	}
-
-	private void appendSorting(int[] flatOrderFields, Order order) {
-
-		if(this.sortKeyPositions == null) {
-			// set sorting info
-			this.sortKeyPositions = flatOrderFields;
-			this.sortOrders = new Order[flatOrderFields.length];
-			Arrays.fill(this.sortOrders, order);
-		} else {
-			// append sorting info to exising info
-			int oldLength = this.sortKeyPositions.length;
-			int newLength = oldLength + flatOrderFields.length;
-			this.sortKeyPositions = Arrays.copyOf(this.sortKeyPositions, newLength);
-			this.sortOrders = Arrays.copyOf(this.sortOrders, newLength);
-
-			for(int i=0; i<flatOrderFields.length; i++) {
-				this.sortKeyPositions[oldLength+i] = flatOrderFields[i];
-				this.sortOrders[oldLength+i] = order;
-			}
-		}
-	}
-
-	// --------------------------------------------------------------------------------------------
-	//  Translation
-	// --------------------------------------------------------------------------------------------
-
-	protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {
-
-		String name = "Sort at " + sortLocationName;
-
-		Ordering partitionOrdering = new Ordering();
-		for (int i = 0; i < this.sortKeyPositions.length; i++) {
-			partitionOrdering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]);
-		}
-
-		// distinguish between partition types
-		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<T, T>(getType(), getType());
-		SortPartitionOperatorBase<T> noop = new  SortPartitionOperatorBase<T>(operatorInfo, partitionOrdering, name);
-		noop.setInput(input);
-		if(this.getParallelism() < 0) {
-			// use parallelism of input if not explicitly specified
-			noop.setParallelism(input.getParallelism());
-		} else {
-			// use explicitly specified parallelism
-			noop.setParallelism(this.getParallelism());
-		}
-
-		return noop;
-
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/a8e08507/flink-java/src/main/java/org/apache/flink/api/java/operators/SortPartitionOperator.java
----------------------------------------------------------------------
diff --git a/flink-java/src/main/java/org/apache/flink/api/java/operators/SortPartitionOperator.java b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortPartitionOperator.java
new file mode 100644
index 0000000..3fce01b
--- /dev/null
+++ b/flink-java/src/main/java/org/apache/flink/api/java/operators/SortPartitionOperator.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.api.java.operators;
+
+import org.apache.flink.api.common.InvalidProgramException;
+import org.apache.flink.api.common.operators.Operator;
+import org.apache.flink.api.common.operators.Order;
+import org.apache.flink.api.common.operators.Ordering;
+import org.apache.flink.api.common.operators.UnaryOperatorInformation;
+import org.apache.flink.api.common.operators.base.SortPartitionOperatorBase;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.api.java.DataSet;
+import org.apache.flink.api.java.typeutils.TupleTypeInfoBase;
+
+import java.util.Arrays;
+
+/**
+ * This operator represents a DataSet with locally sorted partitions.
+ *
+ * @param <T> The type of the DataSet with locally sorted partitions.
+ */
+public class SortPartitionOperator<T> extends SingleInputOperator<T, T, SortPartitionOperator<T>> {
+
+	private int[] sortKeyPositions;
+
+	private Order[] sortOrders;
+
+	private final String sortLocationName;
+
+
+	public SortPartitionOperator(DataSet<T> dataSet, int sortField, Order sortOrder, String sortLocationName) {
+		super(dataSet, dataSet.getType());
+		this.sortLocationName = sortLocationName;
+
+		int[] flatOrderKeys = getFlatFields(sortField);
+		this.appendSorting(flatOrderKeys, sortOrder);
+	}
+
+	public SortPartitionOperator(DataSet<T> dataSet, String sortField, Order sortOrder, String sortLocationName) {
+		super(dataSet, dataSet.getType());
+		this.sortLocationName = sortLocationName;
+
+		int[] flatOrderKeys = getFlatFields(sortField);
+		this.appendSorting(flatOrderKeys, sortOrder);
+	}
+
+	/**
+	 * Appends an additional sort order with the specified field in the specified order to the
+	 * local partition sorting of the DataSet.
+	 *
+	 * @param field The field index of the additional sort order of the local partition sorting.
+	 * @param order The order of the additional sort order of the local partition sorting.
+	 * @return The DataSet with sorted local partitions.
+	 */
+	public SortPartitionOperator<T> sortPartition(int field, Order order) {
+
+		int[] flatOrderKeys = getFlatFields(field);
+		this.appendSorting(flatOrderKeys, order);
+		return this;
+	}
+
+	/**
+	 * Appends an additional sort order with the specified field in the specified order to the
+	 * local partition sorting of the DataSet.
+	 *
+	 * @param field The field expression referring to the field of the additional sort order of
+	 *                 the local partition sorting.
+	 * @param order The order  of the additional sort order of the local partition sorting.
+	 * @return The DataSet with sorted local partitions.
+	 */
+	public SortPartitionOperator<T> sortPartition(String field, Order order) {
+		int[] flatOrderKeys = getFlatFields(field);
+		this.appendSorting(flatOrderKeys, order);
+		return this;
+	}
+
+	// --------------------------------------------------------------------------------------------
+	//  Key Extraction
+	// --------------------------------------------------------------------------------------------
+
+	private int[] getFlatFields(int field) {
+
+		if(!(super.getType() instanceof TupleTypeInfoBase<?>)) {
+			throw new InvalidProgramException("Field positions can only be specified on Tuple or " +
+					"Case Class types.");
+		}
+		else {
+			// check selected field is sortable type
+			TypeInformation<?> sortKeyType = ((TupleTypeInfoBase<?>) super.getType()).getTypeAt(field);
+			if (!sortKeyType.isSortKeyType()) {
+				throw new InvalidProgramException("Selected sort key is not a sortable type " + sortKeyType);
+			}
+		}
+
+		Keys.ExpressionKeys<T> ek;
+		try {
+			ek = new Keys.ExpressionKeys<T>(new int[]{field}, super.getType());
+		} catch(IllegalArgumentException iae) {
+			throw new InvalidProgramException("Invalid specification of field position.", iae);
+		}
+		return ek.computeLogicalKeyPositions();
+	}
+
+	private int[] getFlatFields(String fields) {
+
+		if(super.getType() instanceof CompositeType) {
+
+			// check selected field is sortable type
+			TypeInformation<?> sortKeyType = ((CompositeType<?>) super.getType()).getTypeAt(fields);
+			if (!sortKeyType.isSortKeyType()) {
+				throw new InvalidProgramException("Selected sort key is not a sortable type " + sortKeyType);
+			}
+
+			// compute flat field positions for (nested) sorting fields
+			Keys.ExpressionKeys<T> ek;
+			try {
+				ek = new Keys.ExpressionKeys<T>(new String[]{fields}, super.getType());
+			} catch(IllegalArgumentException iae) {
+				throw new InvalidProgramException("Invalid specification of field expression.", iae);
+			}
+			return ek.computeLogicalKeyPositions();
+		} else {
+
+			fields = fields.trim();
+			if (!(fields.equals("*") || fields.equals("_"))) {
+				throw new InvalidProgramException("Output sorting of non-composite types can only be defined on the full type. " +
+						"Use a field wildcard for that (\"*\" or \"_\")");
+			} else {
+
+				// check if selected field is sortable type
+				if (!super.getType().isSortKeyType()) {
+					throw new InvalidProgramException("Selected sort key cannot be sorted: " + super.getType());
+				}
+
+				return new int[]{0};
+			}
+		}
+	}
+
+	private void appendSorting(int[] flatOrderFields, Order order) {
+
+		if(this.sortKeyPositions == null) {
+			// set sorting info
+			this.sortKeyPositions = flatOrderFields;
+			this.sortOrders = new Order[flatOrderFields.length];
+			Arrays.fill(this.sortOrders, order);
+		} else {
+			// append sorting info to exising info
+			int oldLength = this.sortKeyPositions.length;
+			int newLength = oldLength + flatOrderFields.length;
+			this.sortKeyPositions = Arrays.copyOf(this.sortKeyPositions, newLength);
+			this.sortOrders = Arrays.copyOf(this.sortOrders, newLength);
+
+			for(int i=0; i<flatOrderFields.length; i++) {
+				this.sortKeyPositions[oldLength+i] = flatOrderFields[i];
+				this.sortOrders[oldLength+i] = order;
+			}
+		}
+	}
+
+	// --------------------------------------------------------------------------------------------
+	//  Translation
+	// --------------------------------------------------------------------------------------------
+
+	protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {
+
+		String name = "Sort at " + sortLocationName;
+
+		Ordering partitionOrdering = new Ordering();
+		for (int i = 0; i < this.sortKeyPositions.length; i++) {
+			partitionOrdering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]);
+		}
+
+		// distinguish between partition types
+		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<T, T>(getType(), getType());
+		SortPartitionOperatorBase<T> noop = new  SortPartitionOperatorBase<T>(operatorInfo, partitionOrdering, name);
+		noop.setInput(input);
+		if(this.getParallelism() < 0) {
+			// use parallelism of input if not explicitly specified
+			noop.setParallelism(input.getParallelism());
+		} else {
+			// use explicitly specified parallelism
+			noop.setParallelism(this.getParallelism());
+		}
+
+		return noop;
+
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/flink/blob/a8e08507/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
index 2ade2bc..b14c9c2 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
@@ -33,7 +33,7 @@ import org.apache.flink.api.java.functions.{FirstReducer, KeySelector}
 import org.apache.flink.api.java.io.{DiscardingOutputFormat, PrintingOutputFormat, TextOutputFormat}
 import org.apache.flink.api.java.operators.Keys.ExpressionKeys
 import org.apache.flink.api.java.operators._
-import org.apache.flink.api.java.{DataSet => JavaDataSet, SortPartitionOperator, Utils}
+import org.apache.flink.api.java.{DataSet => JavaDataSet, Utils}
 import org.apache.flink.api.scala.operators.{ScalaAggregateOperator, ScalaCsvOutputFormat}
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.core.fs.{FileSystem, Path}