You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by al...@apache.org on 2014/09/22 14:29:41 UTC

[59/60] git commit: [scala] Add and* methods to AggregateDateSet

[scala] Add and* methods to AggregateDateSet

And create AggregateDataSet in the first place, to add the methods to.


Project: http://git-wip-us.apache.org/repos/asf/incubator-flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-flink/commit/c778d289
Tree: http://git-wip-us.apache.org/repos/asf/incubator-flink/tree/c778d289
Diff: http://git-wip-us.apache.org/repos/asf/incubator-flink/diff/c778d289

Branch: refs/heads/master
Commit: c778d289b3894832085af5821f046adc3396215d
Parents: b8a8780
Author: Aljoscha Krettek <al...@gmail.com>
Authored: Sat Sep 20 08:08:03 2014 +0200
Committer: Aljoscha Krettek <al...@gmail.com>
Committed: Mon Sep 22 13:43:03 2014 +0200

----------------------------------------------------------------------
 .../flink/api/scala/AggregateDataSet.scala      | 90 ++++++++++++++++++++
 .../org/apache/flink/api/scala/DataSet.scala    | 19 +----
 .../apache/flink/api/scala/GroupedDataSet.scala | 47 ++++------
 .../api/scala/ScalaAPICompletenessTest.scala    |  3 +
 .../scala/operators/AggregateOperatorTest.scala |  4 +-
 .../translation/AggregateTranslationTest.scala  |  2 +-
 6 files changed, 118 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c778d289/flink-scala/src/main/scala/org/apache/flink/api/scala/AggregateDataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/AggregateDataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/AggregateDataSet.scala
new file mode 100644
index 0000000..aac7fbb
--- /dev/null
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/AggregateDataSet.scala
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.flink.api.scala
+
+import org.apache.flink.api.java.aggregation.Aggregations
+import org.apache.flink.api.scala.operators.ScalaAggregateOperator
+
+import scala.reflect.ClassTag
+
+class AggregateDataSet[T: ClassTag](set: ScalaAggregateOperator[T])
+  extends DataSet[T](set) {
+
+  /**
+   * Adds the given aggregation on the given field to the previous aggregation operation.
+   *
+   * This only works on Tuple DataSets.
+   */
+  def and(agg: Aggregations, field: Int): AggregateDataSet[T] = {
+    set.and(agg, field)
+    this
+  }
+
+  /**
+   * Adds the given aggregation on the given field to the previous aggregation operation.
+   *
+   * This only works on CaseClass DataSets.
+   */
+  def and(agg: Aggregations, field: String): AggregateDataSet[T] = {
+    val fieldIndex = fieldNames2Indices(set.getType, Array(field))(0)
+    set.and(agg, fieldIndex)
+    this
+  }
+
+  /**
+   * Syntactic sugar for [[and]] with `SUM`
+   */
+  def andSum(field: Int) = {
+    and(Aggregations.SUM, field)
+  }
+
+  /**
+   * Syntactic sugar for [[and]] with `MAX`
+   */
+  def andMax(field: Int) = {
+    and(Aggregations.MAX, field)
+  }
+
+  /**
+   * Syntactic sugar for [[and]] with `MIN`
+   */
+  def andMin(field: Int) = {
+    and(Aggregations.MIN, field)
+  }
+
+  /**
+   * Syntactic sugar for [[and]] with `SUM`
+   */
+  def andSum(field: String) = {
+    and(Aggregations.SUM, field)
+  }
+
+  /**
+   * Syntactic sugar for [[and]] with `MAX`
+   */
+  def andMax(field: String) = {
+    and(Aggregations.MAX, field)
+  }
+
+  /**
+   * Syntactic sugar for [[and]] with `MIN`
+   */
+  def andMin(field: String) = {
+    and(Aggregations.MIN, field)
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c778d289/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
index 8aabce4..4aa432f 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/DataSet.scala
@@ -29,7 +29,6 @@ import org.apache.flink.api.java.operators.Keys.FieldPositionKeys
 import org.apache.flink.api.java.operators._
 import org.apache.flink.api.java.{DataSet => JavaDataSet}
 import org.apache.flink.api.scala.operators.{ScalaCsvOutputFormat, ScalaAggregateOperator}
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 import org.apache.flink.core.fs.FileSystem.WriteMode
 import org.apache.flink.core.fs.{FileSystem, Path}
 import org.apache.flink.types.TypeInformation
@@ -367,12 +366,8 @@ class DataSet[T: ClassTag](private[flink] val set: JavaDataSet[T]) {
    *
    * This only works on Tuple DataSets.
    */
-  def aggregate(agg: Aggregations, field: Int): DataSet[T] = set match {
-    case aggregation: ScalaAggregateOperator[T] =>
-      aggregation.and(agg, field)
-      wrap(aggregation)
-
-    case _ => wrap(new ScalaAggregateOperator[T](set, agg, field))
+  def aggregate(agg: Aggregations, field: Int): AggregateDataSet[T] = {
+    new AggregateDataSet(new ScalaAggregateOperator[T](set, agg, field))
   }
 
   /**
@@ -382,16 +377,10 @@ class DataSet[T: ClassTag](private[flink] val set: JavaDataSet[T]) {
    *
    * This only works on CaseClass DataSets.
    */
-  def aggregate(agg: Aggregations, field: String): DataSet[T] = {
+  def aggregate(agg: Aggregations, field: String): AggregateDataSet[T] = {
     val fieldIndex = fieldNames2Indices(set.getType, Array(field))(0)
 
-    set match {
-      case aggregation: ScalaAggregateOperator[T] =>
-        aggregation.and(agg, fieldIndex)
-        wrap(aggregation)
-
-      case _ => wrap(new ScalaAggregateOperator[T](set, agg, fieldIndex))
-    }
+    new AggregateDataSet(new ScalaAggregateOperator[T](set, agg, fieldIndex))
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c778d289/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala b/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
index 88a8c7c..0f2370d 100644
--- a/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
+++ b/flink-scala/src/main/scala/org/apache/flink/api/scala/GroupedDataSet.scala
@@ -19,7 +19,6 @@ package org.apache.flink.api.scala
 
 import org.apache.flink.api.common.InvalidProgramException
 import org.apache.flink.api.scala.operators.ScalaAggregateOperator
-import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 
 import scala.collection.JavaConverters._
 
@@ -67,7 +66,7 @@ trait GroupedDataSet[T] {
    *
    * This only works on Tuple DataSets.
    */
-  def aggregate(agg: Aggregations, field: Int): DataSet[T]
+  def aggregate(agg: Aggregations, field: Int): AggregateDataSet[T]
 
   /**
    * Creates a new [[DataSet]] by aggregating the specified field using the given aggregation
@@ -76,37 +75,37 @@ trait GroupedDataSet[T] {
    *
    * This only works on CaseClass DataSets.
    */
-  def aggregate(agg: Aggregations, field: String): DataSet[T]
+  def aggregate(agg: Aggregations, field: String): AggregateDataSet[T]
 
   /**
    * Syntactic sugar for [[aggregate]] with `SUM`
    */
-  def sum(field: Int): DataSet[T]
+  def sum(field: Int): AggregateDataSet[T]
 
   /**
    * Syntactic sugar for [[aggregate]] with `MAX`
    */
-  def max(field: Int): DataSet[T]
+  def max(field: Int): AggregateDataSet[T]
 
   /**
    * Syntactic sugar for [[aggregate]] with `MIN`
    */
-  def min(field: Int): DataSet[T]
+  def min(field: Int): AggregateDataSet[T]
 
   /**
    * Syntactic sugar for [[aggregate]] with `SUM`
    */
-  def sum(field: String): DataSet[T]
+  def sum(field: String): AggregateDataSet[T]
 
   /**
    * Syntactic sugar for [[aggregate]] with `MAX`
    */
-  def max(field: String): DataSet[T]
+  def max(field: String): AggregateDataSet[T]
 
   /**
    * Syntactic sugar for [[aggregate]] with `MIN`
    */
-  def min(field: String): DataSet[T]
+  def min(field: String): AggregateDataSet[T]
 
   /**
    * Creates a new [[DataSet]] by merging the elements of each group (elements with the same key)
@@ -194,47 +193,37 @@ private[flink] class GroupedDataSetImpl[T: ClassTag](
   /** Convenience methods for creating the [[UnsortedGrouping]] */
   private def createUnsortedGrouping(): Grouping[T] = new UnsortedGrouping[T](set, keys)
 
-  def aggregate(agg: Aggregations, field: String): DataSet[T] = {
+  def aggregate(agg: Aggregations, field: String): AggregateDataSet[T] = {
     val fieldIndex = fieldNames2Indices(set.getType, Array(field))(0)
 
-    set match {
-      case aggregation: ScalaAggregateOperator[T] =>
-        aggregation.and(agg, fieldIndex)
-        wrap(aggregation)
-
-      case _ => wrap(new ScalaAggregateOperator[T](createUnsortedGrouping(), agg, fieldIndex))
-    }
+    new AggregateDataSet(new ScalaAggregateOperator[T](createUnsortedGrouping(), agg, fieldIndex))
   }
 
-  def aggregate(agg: Aggregations, field: Int): DataSet[T] = set match {
-    case aggregation: ScalaAggregateOperator[T] =>
-      aggregation.and(agg, field)
-      wrap(aggregation)
-
-    case _ => wrap(new ScalaAggregateOperator[T](createUnsortedGrouping(), agg, field))
+  def aggregate(agg: Aggregations, field: Int): AggregateDataSet[T] = {
+    new AggregateDataSet(new ScalaAggregateOperator[T](createUnsortedGrouping(), agg, field))
   }
 
-  def sum(field: Int): DataSet[T] = {
+  def sum(field: Int) = {
     aggregate(Aggregations.SUM, field)
   }
 
-  def max(field: Int): DataSet[T] = {
+  def max(field: Int) = {
     aggregate(Aggregations.MAX, field)
   }
 
-  def min(field: Int): DataSet[T] = {
+  def min(field: Int) = {
     aggregate(Aggregations.MIN, field)
   }
 
-  def sum(field: String): DataSet[T] = {
+  def sum(field: String) = {
     aggregate(Aggregations.SUM, field)
   }
 
-  def max(field: String): DataSet[T] = {
+  def max(field: String) = {
     aggregate(Aggregations.MAX, field)
   }
 
-  def min(field: String): DataSet[T] = {
+  def min(field: String) = {
     aggregate(Aggregations.MIN, field)
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c778d289/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
index 97939f0..6e84271 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/ScalaAPICompletenessTest.scala
@@ -152,6 +152,9 @@ class ScalaAPICompletenessTest {
     checkMethods("SortedGrouping", "GroupedDataSet",
       classOf[SortedGrouping[_]], classOf[GroupedDataSet[_]])
 
+    checkMethods("AggregateOperator", "AggregateDataSet",
+      classOf[AggregateOperator[_]], classOf[AggregateDataSet[_]])
+
     checkMethods("SingleInputOperator", "DataSet",
       classOf[SingleInputOperator[_, _, _]], classOf[DataSet[_]])
 

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c778d289/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
index ea35138..4dbc5fc 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/AggregateOperatorTest.scala
@@ -108,10 +108,10 @@ class AggregateOperatorTest {
       val tupleDs = env.fromCollection(emptyTupleData)
 
       // should work: multiple aggregates
-      tupleDs.aggregate(Aggregations.SUM, 0).aggregate(Aggregations.MIN, 4)
+      tupleDs.aggregate(Aggregations.SUM, 0).and(Aggregations.MIN, 4)
 
       // should work: nested aggregates
-      tupleDs.aggregate(Aggregations.MIN, 2).aggregate(Aggregations.SUM, 1)
+      tupleDs.aggregate(Aggregations.MIN, 2).and(Aggregations.SUM, 1)
 
       // should not work: average on string
       try {

http://git-wip-us.apache.org/repos/asf/incubator-flink/blob/c778d289/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
----------------------------------------------------------------------
diff --git a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
index 7501f82..2ca0c01 100644
--- a/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
+++ b/flink-scala/src/test/scala/org/apache/flink/api/scala/operators/translation/AggregateTranslationTest.scala
@@ -35,7 +35,7 @@ class AggregateTranslationTest {
 
       val initialData = env.fromElements((3.141592, "foobar", 77L))
 
-      initialData.groupBy(0).aggregate(Aggregations.MIN, 1).aggregate(Aggregations.SUM, 2).print()
+      initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).print()
 
       val p: Plan = env.createProgramPlan()
       val sink = p.getDataSinks.iterator.next