You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by tw...@apache.org on 2018/11/20 10:33:53 UTC

[flink] branch release-1.6 updated: [FLINK-10763] [streaming] Fix interval join return type in Scala

This is an automated email from the ASF dual-hosted git repository.

twalthr pushed a commit to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.6 by this push:
     new febd1c6  [FLINK-10763] [streaming] Fix interval join return type in Scala
febd1c6 is described below

commit febd1c6f534a4108a0eaf6fa6847e2d01e0b0712
Author: Timo Walther <tw...@apache.org>
AuthorDate: Mon Nov 19 15:56:34 2018 +0100

    [FLINK-10763] [streaming] Fix interval join return type in Scala
    
    This commit fixes the wrong result type of interval joins in
    the DataStream Scala API. The API did not call the Scala type
    extraction stack but was using the default one offered by the
    DataStream Java API.
    
    This closes #7141.
---
 .../flink/streaming/api/scala/KeyedStream.scala    |  9 +++-
 .../streaming/api/scala/IntervalJoinITCase.scala   | 60 ++++++++++++----------
 2 files changed, 41 insertions(+), 28 deletions(-)

diff --git a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala
index 580dd46..f85e1b5 100644
--- a/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala
+++ b/flink-streaming-scala/src/main/scala/org/apache/flink/streaming/api/scala/KeyedStream.scala
@@ -201,7 +201,12 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T]
       * @return Returns a DataStream
       */
     @PublicEvolving
-    def process[OUT](processJoinFunction: ProcessJoinFunction[IN1, IN2, OUT]): DataStream[OUT] = {
+    def process[OUT: TypeInformation](
+        processJoinFunction: ProcessJoinFunction[IN1, IN2, OUT])
+      : DataStream[OUT] = {
+
+      val outType: TypeInformation[OUT] = implicitly[TypeInformation[OUT]]
+
       val javaJoined = new KeyedJavaStream.IntervalJoined[IN1, IN2, KEY](
         firstStream.javaStream.asInstanceOf[KeyedJavaStream[IN1, KEY]],
         secondStream.javaStream.asInstanceOf[KeyedJavaStream[IN2, KEY]],
@@ -209,7 +214,7 @@ class KeyedStream[T, K](javaStream: KeyedJavaStream[T, K]) extends DataStream[T]
         upperBound,
         lowerBoundInclusive,
         upperBoundInclusive)
-      asScalaStream(javaJoined.process(processJoinFunction))
+      asScalaStream(javaJoined.process(processJoinFunction, outType))
     }
   }
 
diff --git a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala
index 80701c6..a9ace36 100644
--- a/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala
+++ b/flink-streaming-scala/src/test/scala/org/apache/flink/streaming/api/scala/IntervalJoinITCase.scala
@@ -18,6 +18,7 @@
 
 package org.apache.flink.streaming.api.scala
 
+import org.apache.flink.api.scala.typeutils.CaseClassTypeInfo
 import org.apache.flink.streaming.api.TimeCharacteristic
 import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction
 import org.apache.flink.streaming.api.functions.sink.SinkFunction
@@ -25,7 +26,8 @@ import org.apache.flink.streaming.api.functions.timestamps.AscendingTimestampExt
 import org.apache.flink.streaming.api.windowing.time.Time
 import org.apache.flink.test.util.AbstractTestBase
 import org.apache.flink.util.Collector
-import org.junit.{Assert, Test}
+import org.junit.Assert.assertTrue
+import org.junit.Test
 
 import scala.collection.mutable.ListBuffer
 
@@ -47,22 +49,25 @@ class IntervalJoinITCase extends AbstractTestBase {
 
     val sink = new ResultSink()
 
-    dataStream1.intervalJoin(dataStream2)
+    val join = dataStream1.intervalJoin(dataStream2)
       .between(Time.milliseconds(0), Time.milliseconds(2))
-      .process(new CombineToStringJoinFunction())
-      .addSink(sink)
+      .process(new CombineJoinFunction())
+
+    assertTrue(join.dataType.isInstanceOf[CaseClassTypeInfo[_]])
+
+    join.addSink(sink)
 
     env.execute()
 
     sink.expectInAnyOrder(
-      "(key,0):(key,0)",
-      "(key,0):(key,1)",
-      "(key,0):(key,2)",
+      "(key:key,0)",
+      "(key:key,1)",
+      "(key:key,2)",
 
-      "(key,1):(key,1)",
-      "(key,1):(key,2)",
+      "(key:key,2)",
+      "(key:key,3)",
 
-      "(key,2):(key,2)"
+      "(key:key,4)"
     )
   }
 
@@ -82,18 +87,21 @@ class IntervalJoinITCase extends AbstractTestBase {
 
     val sink = new ResultSink()
 
-    dataStream1.intervalJoin(dataStream2)
+    val join = dataStream1.intervalJoin(dataStream2)
       .between(Time.milliseconds(0), Time.milliseconds(2))
       .lowerBoundExclusive()
       .upperBoundExclusive()
-      .process(new CombineToStringJoinFunction())
-      .addSink(sink)
+      .process(new CombineJoinFunction())
+
+    assertTrue(join.dataType.isInstanceOf[CaseClassTypeInfo[_]])
+
+    join.addSink(sink)
 
     env.execute()
 
     sink.expectInAnyOrder(
-      "(key,0):(key,1)",
-      "(key,1):(key,2)"
+      "(key:key,1)",
+      "(key:key,3)"
     )
   }
 }
@@ -102,14 +110,14 @@ object Companion {
   val results: ListBuffer[String] = new ListBuffer()
 }
 
-class ResultSink extends SinkFunction[String] {
+class ResultSink extends SinkFunction[(String, Long)] {
 
-  override def invoke(value: String, context: SinkFunction.Context[_]): Unit = {
-    Companion.results.append(value)
+  override def invoke(value: (String, Long), context: SinkFunction.Context[_]): Unit = {
+    Companion.results.append(value.toString())
   }
 
   def expectInAnyOrder(expected: String*): Unit = {
-    Assert.assertTrue(expected.toSet.equals(Companion.results.toSet))
+    assertTrue(expected.toSet.equals(Companion.results.toSet))
   }
 }
 
@@ -117,14 +125,14 @@ class TimestampExtractor extends AscendingTimestampExtractor[(String, Long)] {
   override def extractAscendingTimestamp(element: (String, Long)): Long = element._2
 }
 
-class CombineToStringJoinFunction
-  extends ProcessJoinFunction[(String, Long), (String, Long), String] {
+class CombineJoinFunction
+  extends ProcessJoinFunction[(String, Long), (String, Long), (String, Long)] {
 
   override def processElement(
-                        left: (String, Long),
-                        right: (String, Long),
-                        ctx: ProcessJoinFunction[(String, Long), (String, Long), String]#Context,
-                        out: Collector[String]): Unit = {
-    out.collect(left + ":" + right)
+      left: (String, Long),
+      right: (String, Long),
+      ctx: ProcessJoinFunction[(String, Long), (String, Long), (String, Long)]#Context,
+      out: Collector[(String, Long)]): Unit = {
+    out.collect((left._1 + ":" + right._1, left._2 + right._2))
   }
 }