You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ji...@apache.org on 2018/12/11 07:07:08 UTC
[flink] branch release-1.7 updated: In order to maintain state
compatibility revert FLINK-10543.
This is an automated email from the ASF dual-hosted git repository.
jincheng pushed a commit to branch release-1.7
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.7 by this push:
new d396633 In order to maintain state compatibility revert FLINK-10543.
d396633 is described below
commit d3966332d4fe37fd22f92c758a6a34097ed16ac6
Author: sunjincheng121 <su...@gmail.com>
AuthorDate: Tue Dec 11 14:43:40 2018 +0800
In order to maintain state compatibility revert FLINK-10543.
This reverts commit 0fa9ec030d6b102f4d24f0c7f8b58c0fab97fff6.
---
.../table/runtime/aggregate/CleanupState.scala | 57 ----
.../CoProcessFunctionWithCleanupState.scala | 69 -----
.../aggregate/GroupAggProcessFunction.scala | 4 +-
.../KeyedProcessFunctionWithCleanupState.scala | 38 ++-
.../aggregate/ProcTimeBoundedRangeOver.scala | 29 +-
.../aggregate/ProcTimeBoundedRowsOver.scala | 4 +-
.../runtime/aggregate/ProcTimeUnboundedOver.scala | 4 +-
.../ProcessFunctionWithCleanupState.scala | 41 ++-
.../aggregate/RowTimeBoundedRangeOver.scala | 20 +-
.../runtime/aggregate/RowTimeBoundedRowsOver.scala | 8 +-
.../runtime/aggregate/RowTimeUnboundedOver.scala | 8 +-
.../table/runtime/join/NonWindowFullJoin.scala | 3 +-
.../NonWindowFullJoinWithNonEquiPredicates.scala | 23 +-
.../table/runtime/join/NonWindowInnerJoin.scala | 3 +-
.../flink/table/runtime/join/NonWindowJoin.scala | 104 +++++--
.../runtime/join/NonWindowLeftRightJoin.scala | 3 +-
...nWindowLeftRightJoinWithNonEquiPredicates.scala | 25 +-
.../table/runtime/join/NonWindowOuterJoin.scala | 9 +-
.../NonWindowOuterJoinWithNonEquiPredicates.scala | 47 ++-
.../triggers/StateCleaningCountTrigger.scala | 13 +-
.../table/runtime/harness/JoinHarnessTest.scala | 320 ++++++++++-----------
.../StateCleaningCountTriggerHarnessTest.scala | 7 +-
.../KeyedProcessFunctionWithCleanupStateTest.scala | 10 +-
.../ProcessFunctionWithCleanupStateTest.scala | 4 +-
24 files changed, 430 insertions(+), 423 deletions(-)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala
deleted file mode 100644
index d9c8e2c..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.runtime.aggregate
-
-import org.apache.flink.api.common.state.ValueState
-import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.streaming.api.functions.co.CoProcessFunction
-import java.lang.{Long => JLong}
-
-import org.apache.flink.streaming.api.TimerService
-
-/**
- * Base class for clean up state, both for [[ProcessFunction]] and [[CoProcessFunction]].
- */
-trait CleanupState {
-
- def registerProcessingCleanupTimer(
- cleanupTimeState: ValueState[JLong],
- currentTime: Long,
- minRetentionTime: Long,
- maxRetentionTime: Long,
- timerService: TimerService): Unit = {
-
- // last registered timer
- val curCleanupTime = cleanupTimeState.value()
-
- // check if a cleanup timer is registered and
- // that the current cleanup timer won't delete state we need to keep
- if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
- // we need to register a new (later) timer
- val cleanupTime = currentTime + maxRetentionTime
- // register timer and remember clean-up time
- timerService.registerProcessingTimeTimer(cleanupTime)
- // delete expired timer
- if (curCleanupTime != null) {
- timerService.deleteProcessingTimeTimer(curCleanupTime)
- }
- cleanupTimeState.update(cleanupTime)
- }
- }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala
deleted file mode 100644
index 0c76636..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.runtime.aggregate
-
-import java.lang.{Long => JLong}
-
-import org.apache.flink.api.common.state.{State, ValueState, ValueStateDescriptor}
-import org.apache.flink.streaming.api.TimeDomain
-import org.apache.flink.streaming.api.functions.co.CoProcessFunction
-import org.apache.flink.table.api.{StreamQueryConfig, Types}
-
-abstract class CoProcessFunctionWithCleanupState[IN1, IN2, OUT](queryConfig: StreamQueryConfig)
- extends CoProcessFunction[IN1, IN2, OUT]
- with CleanupState {
-
- protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
- protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
- protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
-
- // holds the latest registered cleanup timer
- private var cleanupTimeState: ValueState[JLong] = _
-
- protected def initCleanupTimeState(stateName: String) {
- if (stateCleaningEnabled) {
- val cleanupTimeDescriptor: ValueStateDescriptor[JLong] =
- new ValueStateDescriptor[JLong](stateName, Types.LONG)
- cleanupTimeState = getRuntimeContext.getState(cleanupTimeDescriptor)
- }
- }
-
- protected def processCleanupTimer(
- ctx: CoProcessFunction[IN1, IN2, OUT]#Context,
- currentTime: Long): Unit = {
- if (stateCleaningEnabled) {
- registerProcessingCleanupTimer(
- cleanupTimeState,
- currentTime,
- minRetentionTime,
- maxRetentionTime,
- ctx.timerService()
- )
- }
- }
-
- protected def isProcessingTimeTimer(ctx: OnTimerContext): Boolean = {
- ctx.timeDomain() == TimeDomain.PROCESSING_TIME
- }
-
- protected def cleanupState(states: State*): Unit = {
- // clear all state
- states.foreach(_.clear())
- this.cleanupTimeState.clear()
- }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 2d72e6d..c59efe2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -86,7 +86,7 @@ class GroupAggProcessFunction(
val currentTime = ctx.timerService().currentProcessingTime()
// register state-cleanup timer
- processCleanupTimer(ctx, currentTime)
+ registerProcessingCleanupTimer(ctx, currentTime)
val input = inputC.row
@@ -172,7 +172,7 @@ class GroupAggProcessFunction(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
cleanupState(state, cntState)
function.cleanup()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
index edf5c2c..4d6840a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
@@ -25,15 +25,13 @@ import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFu
import org.apache.flink.table.api.{StreamQueryConfig, Types}
abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: StreamQueryConfig)
- extends KeyedProcessFunction[K, I, O]
- with CleanupState {
-
+ extends KeyedProcessFunction[K, I, O] {
protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
// holds the latest registered cleanup timer
- protected var cleanupTimeState: ValueState[JLong] = _
+ private var cleanupTimeState: ValueState[JLong] = _
protected def initCleanupTimeState(stateName: String) {
if (stateCleaningEnabled) {
@@ -43,17 +41,23 @@ abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: Stream
}
}
- protected def processCleanupTimer(
+ protected def registerProcessingCleanupTimer(
ctx: KeyedProcessFunction[K, I, O]#Context,
currentTime: Long): Unit = {
if (stateCleaningEnabled) {
- registerProcessingCleanupTimer(
- cleanupTimeState,
- currentTime,
- minRetentionTime,
- maxRetentionTime,
- ctx.timerService()
- )
+
+ // last registered timer
+ val curCleanupTime = cleanupTimeState.value()
+
+ // check if a cleanup timer is registered and
+ // that the current cleanup timer won't delete state we need to keep
+ if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
+ // we need to register a new (later) timer
+ val cleanupTime = currentTime + maxRetentionTime
+ // register timer and remember clean-up time
+ ctx.timerService().registerProcessingTimeTimer(cleanupTime)
+ cleanupTimeState.update(cleanupTime)
+ }
}
}
@@ -61,6 +65,16 @@ abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: Stream
ctx.timeDomain() == TimeDomain.PROCESSING_TIME
}
+ protected def needToCleanupState(timestamp: Long): Boolean = {
+ if (stateCleaningEnabled) {
+ val cleanupTime = cleanupTimeState.value()
+ // check that the triggered timer is the last registered processing time timer.
+ null != cleanupTime && timestamp == cleanupTime
+ } else {
+ false
+ }
+ }
+
protected def cleanupState(states: State*): Unit = {
// clear all state
states.foreach(_.clear())
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index 6126dc7..591b942 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -95,7 +95,7 @@ class ProcTimeBoundedRangeOver(
val currentTime = ctx.timerService.currentProcessingTime
// register state-cleanup timer
- processCleanupTimer(ctx, currentTime)
+ registerProcessingCleanupTimer(ctx, currentTime)
// buffer the event incoming event
@@ -117,14 +117,11 @@ class ProcTimeBoundedRangeOver(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (stateCleaningEnabled) {
- val cleanupTime = cleanupTimeState.value()
- if (null != cleanupTime && timestamp == cleanupTime) {
- // clean up and return
- cleanupState(rowMapState, accumulatorState)
- function.cleanup()
- return
- }
+ if (needToCleanupState(timestamp)) {
+ // clean up and return
+ cleanupState(rowMapState, accumulatorState)
+ function.cleanup()
+ return
}
// remove timestamp set outside of ProcessFunction.
@@ -134,10 +131,11 @@ class ProcTimeBoundedRangeOver(
// that have registered this time trigger 1 ms ago
val currentTime = timestamp - 1
+ var i = 0
// get the list of elements of current proctime
val currentElements = rowMapState.get(currentTime)
- // Expired clean-up timers pass the needToCleanupState check.
+ // Expired clean-up timers pass the needToCleanupState() check.
// Perform a null check to verify that we have data to process.
if (null == currentElements) {
return
@@ -158,6 +156,7 @@ class ProcTimeBoundedRangeOver(
// and eliminate them. Multiple elements could have been received at the same timestamp
// the removal of old elements happens only once per proctime as onTimer is called only once
val iter = rowMapState.iterator
+ val markToRemove = new ArrayList[Long]()
while (iter.hasNext) {
val entry = iter.next()
val elementKey = entry.getKey
@@ -170,9 +169,17 @@ class ProcTimeBoundedRangeOver(
function.retract(accumulators, retractRow)
iRemove += 1
}
- iter.remove()
+ // mark element for later removal not to modify the iterator over MapState
+ markToRemove.add(elementKey)
}
}
+ // need to remove in 2 steps not to have concurrent access errors via iterator to the MapState
+ i = 0
+ while (i < markToRemove.size()) {
+ rowMapState.remove(markToRemove.get(i))
+ i += 1
+ }
+
// add current elements to aggregator. Multiple elements might
// have arrived in the same proctime
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index fa58ac5..ccddaa5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -110,7 +110,7 @@ class ProcTimeBoundedRowsOver(
val currentTime = ctx.timerService.currentProcessingTime
// register state-cleanup timer
- processCleanupTimer(ctx, currentTime)
+ registerProcessingCleanupTimer(ctx, currentTime)
// initialize state for the processed element
var accumulators = accumulatorState.value
@@ -187,7 +187,7 @@ class ProcTimeBoundedRowsOver(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
cleanupState(rowMapState, accumulatorState, counterState, smallestTsState)
function.cleanup()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
index ce1a959..6e4c510 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
@@ -71,7 +71,7 @@ class ProcTimeUnboundedOver(
out: Collector[CRow]): Unit = {
// register state-cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
val input = inputC.row
@@ -95,7 +95,7 @@ class ProcTimeUnboundedOver(
ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
cleanupState(state)
function.cleanup()
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
index 7263de7..292fd3b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
@@ -26,35 +26,40 @@ import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.table.api.{StreamQueryConfig, Types}
abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryConfig)
- extends ProcessFunction[IN, OUT]
- with CleanupState {
+ extends ProcessFunction[IN, OUT]{
protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
// holds the latest registered cleanup timer
- protected var cleanupTimeState: ValueState[JLong] = _
+ private var cleanupTimeState: ValueState[JLong] = _
protected def initCleanupTimeState(stateName: String) {
if (stateCleaningEnabled) {
- val cleanupTimeDescriptor: ValueStateDescriptor[JLong] =
+ val inputCntDescriptor: ValueStateDescriptor[JLong] =
new ValueStateDescriptor[JLong](stateName, Types.LONG)
- cleanupTimeState = getRuntimeContext.getState(cleanupTimeDescriptor)
+ cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor)
}
}
- protected def processCleanupTimer(
+ protected def registerProcessingCleanupTimer(
ctx: ProcessFunction[IN, OUT]#Context,
currentTime: Long): Unit = {
if (stateCleaningEnabled) {
- registerProcessingCleanupTimer(
- cleanupTimeState,
- currentTime,
- minRetentionTime,
- maxRetentionTime,
- ctx.timerService()
- )
+
+ // last registered timer
+ val curCleanupTime = cleanupTimeState.value()
+
+ // check if a cleanup timer is registered and
+ // that the current cleanup timer won't delete state we need to keep
+ if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
+ // we need to register a new (later) timer
+ val cleanupTime = currentTime + maxRetentionTime
+ // register timer and remember clean-up time
+ ctx.timerService().registerProcessingTimeTimer(cleanupTime)
+ cleanupTimeState.update(cleanupTime)
+ }
}
}
@@ -62,6 +67,16 @@ abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryC
ctx.timeDomain() == TimeDomain.PROCESSING_TIME
}
+ protected def needToCleanupState(timestamp: Long): Boolean = {
+ if (stateCleaningEnabled) {
+ val cleanupTime = cleanupTimeState.value()
+ // check that the triggered timer is the last registered processing time timer.
+ null != cleanupTime && timestamp == cleanupTime
+ } else {
+ false
+ }
+ }
+
protected def cleanupState(states: State*): Unit = {
// clear all state
states.foreach(_.clear())
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index 7c509d6..b13acdf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -114,7 +114,7 @@ class RowTimeBoundedRangeOver(
val input = inputC.row
// register state-cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
// triggering timestamp for trigger calculation
val triggeringTs = input.getField(rowTimeIdx).asInstanceOf[Long]
@@ -143,7 +143,7 @@ class RowTimeBoundedRangeOver(
out: Collector[CRow]): Unit = {
if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
val keysIt = dataState.keys.iterator()
val lastProcessedTime = lastTriggeringTsState.value
@@ -164,7 +164,7 @@ class RowTimeBoundedRangeOver(
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
// We leave the state as it is and schedule a new cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
}
return
@@ -188,6 +188,9 @@ class RowTimeBoundedRangeOver(
aggregatesIndex = 0
}
+ // keep up timestamps of retract data
+ val retractTsList: JList[Long] = new JArrayList[Long]
+
// do retraction
val iter = dataState.iterator()
while (iter.hasNext) {
@@ -202,7 +205,7 @@ class RowTimeBoundedRangeOver(
function.retract(accumulators, retractRow)
dataListIndex += 1
}
- iter.remove()
+ retractTsList.add(dataTs)
}
}
@@ -227,13 +230,20 @@ class RowTimeBoundedRangeOver(
dataListIndex += 1
}
+ // remove the data that has been retracted
+ dataListIndex = 0
+ while (dataListIndex < retractTsList.size) {
+ dataState.remove(retractTsList.get(dataListIndex))
+ dataListIndex += 1
+ }
+
// update state
accumulatorState.update(accumulators)
}
lastTriggeringTsState.update(timestamp)
// update cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
override def close(): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index d01a499..e120d6b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -123,7 +123,7 @@ class RowTimeBoundedRowsOver(
val input = inputC.row
// register state-cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
// triggering timestamp for trigger calculation
val triggeringTs = input.getField(rowTimeIdx).asInstanceOf[Long]
@@ -152,7 +152,7 @@ class RowTimeBoundedRowsOver(
out: Collector[CRow]): Unit = {
if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
val keysIt = dataState.keys.iterator()
val lastProcessedTime = lastTriggeringTsState.value
@@ -173,7 +173,7 @@ class RowTimeBoundedRowsOver(
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
// We leave the state as it is and schedule a new cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
}
return
@@ -263,7 +263,7 @@ class RowTimeBoundedRowsOver(
lastTriggeringTsState.update(timestamp)
// update cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
override def close(): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index 690d0d0..181c768 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -108,7 +108,7 @@ abstract class RowTimeUnboundedOver(
val input = inputC.row
// register state-cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
val timestamp = input.getField(rowTimeIdx).asInstanceOf[Long]
val curWatermark = ctx.timerService().currentWatermark()
@@ -143,7 +143,7 @@ abstract class RowTimeUnboundedOver(
out: Collector[CRow]): Unit = {
if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
// we check whether there are still records which have not been processed yet
val noRecordsToProcess = !rowMapState.keys.iterator().hasNext
@@ -155,7 +155,7 @@ abstract class RowTimeUnboundedOver(
// There are records left to process because a watermark has not been received yet.
// This would only happen if the input stream has stopped. So we don't need to clean up.
// We leave the state as it is and schedule a new cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
}
return
@@ -207,7 +207,7 @@ abstract class RowTimeUnboundedOver(
}
// update cleanup timer
- processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
}
/**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
index 5b1069f..57c60f1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
@@ -66,12 +66,13 @@ class NonWindowFullJoin(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val inputRow = value.row
- updateCurrentSide(value, ctx, currentSideState)
+ updateCurrentSide(value, ctx, timerState, currentSideState)
cRowWrapper.reset()
cRowWrapper.setCollector(out)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
index 0166eef..9c27eb4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
@@ -68,13 +68,14 @@ class NonWindowFullJoinWithNonEquiPredicates(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val currentJoinCntState = getJoinCntState(joinCntState, recordFromLeft)
val inputRow = value.row
- val cntAndExpiredTime = updateCurrentSide(value, ctx, currentSideState)
+ val cntAndExpiredTime = updateCurrentSide(value, ctx, timerState, currentSideState)
if (!value.change && cntAndExpiredTime.f0 <= 0) {
currentJoinCntState.remove(inputRow)
}
@@ -98,18 +99,18 @@ class NonWindowFullJoinWithNonEquiPredicates(
}
/**
- * Called when a processing timer trigger.
- * Expire left/right expired records and expired joinCnt state.
+ * Removes records which are expired from left state. Register a new timer if the state still
+ * holds records after the clean-up. Also, clear leftJoinCnt map state when clear left
+ * rowMapState.
*/
- override def onTimer(
- timestamp: Long,
- ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
- out: Collector[CRow]): Unit = {
+ override def expireOutTimeRow(
+ curTime: Long,
+ rowMapState: MapState[Row, JTuple2[Long, Long]],
+ timerState: ValueState[Long],
+ isLeft: Boolean,
+ ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
- // expired timer has already been removed, delete state directly.
- if (stateCleaningEnabled) {
- cleanupState(leftState, rightState, joinCntState(0), joinCntState(1))
- }
+ expireOutTimeRow(curTime, rowMapState, timerState, isLeft, joinCntState, ctx)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
index 91a7507..2e5832c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
@@ -63,12 +63,13 @@ class NonWindowInnerJoin(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
isLeft: Boolean): Unit = {
val inputRow = value.row
- updateCurrentSide(value, ctx, currentSideState)
+ updateCurrentSide(value, ctx, timerState, currentSideState)
cRowWrapper.setCollector(out)
cRowWrapper.setChange(value.change)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
index e15cbfa..c59f4c2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.runtime.join
import org.apache.flink.api.common.functions.FlatJoinFunction
import org.apache.flink.api.common.functions.util.FunctionUtils
-import org.apache.flink.api.common.state.{MapState, MapStateDescriptor}
+import org.apache.flink.api.common.state.{MapState, MapStateDescriptor, ValueState, ValueStateDescriptor}
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
import org.apache.flink.api.java.typeutils.TupleTypeInfo
@@ -27,7 +27,6 @@ import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.codegen.Compiler
-import org.apache.flink.table.runtime.aggregate.CoProcessFunctionWithCleanupState
import org.apache.flink.table.runtime.types.CRow
import org.apache.flink.table.typeutils.TypeCheckUtils._
import org.apache.flink.table.util.Logging
@@ -49,7 +48,7 @@ abstract class NonWindowJoin(
genJoinFuncName: String,
genJoinFuncCode: String,
queryConfig: StreamQueryConfig)
- extends CoProcessFunctionWithCleanupState[CRow, CRow, CRow](queryConfig)
+ extends CoProcessFunction[CRow, CRow, CRow]
with Compiler[FlatJoinFunction[Row, Row, Row]]
with Logging {
@@ -63,6 +62,15 @@ abstract class NonWindowJoin(
protected var rightState: MapState[Row, JTuple2[Long, Long]] = _
protected var cRowWrapper: CRowWrappingMultiOutputCollector = _
+ protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
+ protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
+ protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
+
+ // state to record last timer of left stream, 0 means no timer
+ protected var leftTimer: ValueState[Long] = _
+ // state to record last timer of right stream, 0 means no timer
+ protected var rightTimer: ValueState[Long] = _
+
// other condition function
protected var joinFunction: FlatJoinFunction[Row, Row, Row] = _
@@ -70,8 +78,7 @@ abstract class NonWindowJoin(
protected var curProcessTime: Long = _
override def open(parameters: Configuration): Unit = {
- LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n " +
- s"Code:\n$genJoinFuncCode")
+ LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n Code:\n$genJoinFuncCode")
val clazz = compile(
getRuntimeContext.getUserCodeClassLoader,
genJoinFuncName,
@@ -93,7 +100,10 @@ abstract class NonWindowJoin(
rightState = getRuntimeContext.getMapState(rightStateDescriptor)
// initialize timer state
- initCleanupTimeState("NonWindowJoinCleanupTime")
+ val valueStateDescriptor1 = new ValueStateDescriptor[Long]("timervaluestate1", classOf[Long])
+ leftTimer = getRuntimeContext.getState(valueStateDescriptor1)
+ val valueStateDescriptor2 = new ValueStateDescriptor[Long]("timervaluestate2", classOf[Long])
+ rightTimer = getRuntimeContext.getState(valueStateDescriptor2)
cRowWrapper = new CRowWrappingMultiOutputCollector()
LOG.debug("Instantiating NonWindowJoin.")
@@ -112,7 +122,7 @@ abstract class NonWindowJoin(
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow]): Unit = {
- processElement(valueC, ctx, out, leftState, rightState, isLeft = true)
+ processElement(valueC, ctx, out, leftTimer, leftState, rightState, isLeft = true)
}
/**
@@ -128,7 +138,7 @@ abstract class NonWindowJoin(
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow]): Unit = {
- processElement(valueC, ctx, out, rightState, leftState, isLeft = false)
+ processElement(valueC, ctx, out, rightTimer, rightState, leftState, isLeft = false)
}
/**
@@ -144,13 +154,28 @@ abstract class NonWindowJoin(
ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
out: Collector[CRow]): Unit = {
- // expired timer has already been removed, delete state directly.
- if (stateCleaningEnabled) {
- cleanupState(leftState, rightState)
+ if (stateCleaningEnabled && leftTimer.value == timestamp) {
+ expireOutTimeRow(
+ timestamp,
+ leftState,
+ leftTimer,
+ isLeft = true,
+ ctx
+ )
+ }
+
+ if (stateCleaningEnabled && rightTimer.value == timestamp) {
+ expireOutTimeRow(
+ timestamp,
+ rightState,
+ rightTimer,
+ isLeft = false,
+ ctx
+ )
}
}
- protected def getNewExpiredTime(curProcessTime: Long, oldExpiredTime: Long): Long = {
+ def getNewExpiredTime(curProcessTime: Long, oldExpiredTime: Long): Long = {
if (stateCleaningEnabled && curProcessTime + minRetentionTime > oldExpiredTime) {
curProcessTime + maxRetentionTime
} else {
@@ -159,14 +184,52 @@ abstract class NonWindowJoin(
}
/**
+ * Removes records which are expired from the state. Register a new timer if the state still
+ * holds records after the clean-up.
+ */
+ def expireOutTimeRow(
+ curTime: Long,
+ rowMapState: MapState[Row, JTuple2[Long, Long]],
+ timerState: ValueState[Long],
+ isLeft: Boolean,
+ ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
+
+ val rowMapIter = rowMapState.iterator()
+ var validTimestamp: Boolean = false
+
+ while (rowMapIter.hasNext) {
+ val mapEntry = rowMapIter.next()
+ val recordExpiredTime = mapEntry.getValue.f1
+ if (recordExpiredTime <= curTime) {
+ rowMapIter.remove()
+ } else {
+ // we found a timestamp that is still valid
+ validTimestamp = true
+ }
+ }
+
+ // If the state has non-expired timestamps, register a new timer.
+ // Otherwise clean the complete state for this input.
+ if (validTimestamp) {
+ val cleanupTime = curTime + maxRetentionTime
+ ctx.timerService.registerProcessingTimeTimer(cleanupTime)
+ timerState.update(cleanupTime)
+ } else {
+ timerState.clear()
+ rowMapState.clear()
+ }
+ }
+
+ /**
* Puts or Retract an element from the input stream into state and search the other state to
* output records meet the condition. Records will be expired in state if state retention time
* has been specified.
*/
- protected def processElement(
+ def processElement(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
isLeft: Boolean): Unit
@@ -177,12 +240,14 @@ abstract class NonWindowJoin(
*
* @param value The input CRow
* @param ctx The ctx to register timer or get current time
+ * @param timerState The state to record last timer
* @param currentSideState The state to hold current side stream element
* @return The row number and expired time for current input row
*/
- protected def updateCurrentSide(
+ def updateCurrentSide(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]]): JTuple2[Long, Long] = {
val inputRow = value.row
@@ -196,7 +261,10 @@ abstract class NonWindowJoin(
cntAndExpiredTime.f1 = getNewExpiredTime(curProcessTime, cntAndExpiredTime.f1)
// update timer if necessary
- processCleanupTimer(ctx, curProcessTime)
+ if (stateCleaningEnabled && timerState.value() == 0) {
+ timerState.update(cntAndExpiredTime.f1)
+ ctx.timerService().registerProcessingTimeTimer(cntAndExpiredTime.f1)
+ }
// update current side stream state
if (!value.change) {
@@ -214,7 +282,7 @@ abstract class NonWindowJoin(
cntAndExpiredTime
}
- protected def callJoinFunction(
+ def callJoinFunction(
inputRow: Row,
inputRowFromLeft: Boolean,
otherSideRow: Row,
@@ -226,4 +294,8 @@ abstract class NonWindowJoin(
joinFunction.join(otherSideRow, inputRow, cRowWrapper)
}
}
+
+ override def close(): Unit = {
+ FunctionUtils.closeFunction(joinFunction)
+ }
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
index 5995fb8..b4f14e4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
@@ -69,12 +69,13 @@ class NonWindowLeftRightJoin(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val inputRow = value.row
- updateCurrentSide(value, ctx, currentSideState)
+ updateCurrentSide(value, ctx, timerState, currentSideState)
cRowWrapper.reset()
cRowWrapper.setCollector(out)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
index a3e25f9..33517cc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
@@ -71,13 +71,14 @@ class NonWindowLeftRightJoinWithNonEquiPredicates(
value: CRow,
ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
out: Collector[CRow],
+ timerState: ValueState[Long],
currentSideState: MapState[Row, JTuple2[Long, Long]],
otherSideState: MapState[Row, JTuple2[Long, Long]],
recordFromLeft: Boolean): Unit = {
val currentJoinCntState = getJoinCntState(joinCntState, recordFromLeft)
val inputRow = value.row
- val cntAndExpiredTime = updateCurrentSide(value, ctx, currentSideState)
+ val cntAndExpiredTime = updateCurrentSide(value, ctx, timerState, currentSideState)
if (!value.change && cntAndExpiredTime.f0 <= 0 && recordFromLeft == isLeftJoin) {
currentJoinCntState.remove(inputRow)
}
@@ -100,21 +101,17 @@ class NonWindowLeftRightJoinWithNonEquiPredicates(
}
/**
- * Called when a processing timer trigger.
- * Expire left/right expired records and expired joinCnt state.
+ * Removes records which are expired from state. Register a new timer if the state still
+ * holds records after the clean-up. Also, clear joinCnt map state when clear rowMapState.
*/
- override def onTimer(
- timestamp: Long,
- ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
- out: Collector[CRow]): Unit = {
+ override def expireOutTimeRow(
+ curTime: Long,
+ rowMapState: MapState[Row, JTuple2[Long, Long]],
+ timerState: ValueState[Long],
+ isLeft: Boolean,
+ ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
- // expired timer has already been removed, delete state directly.
- if (stateCleaningEnabled) {
- cleanupState(
- leftState,
- rightState,
- getJoinCntState(joinCntState, isLeftJoin))
- }
+ expireOutTimeRow(curTime, rowMapState, timerState, isLeft, joinCntState, ctx)
}
}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
index 9fadbb0..0018a16 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
@@ -73,7 +73,7 @@ abstract class NonWindowOuterJoin(
* @param otherSideState the other side state
* @return the number of matched rows
*/
- protected def preservedJoin(
+ def preservedJoin(
inputRow: Row,
inputRowFromLeft: Boolean,
otherSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
@@ -106,7 +106,7 @@ abstract class NonWindowOuterJoin(
* RowWrapper has been reset before we call retractJoin and we also assume that the current
* change of cRowWrapper is equal to value.change.
*/
- protected def retractJoin(
+ def retractJoin(
value: CRow,
inputRowFromLeft: Boolean,
currentSideState: MapState[Row, JTuple2[Long, Long]],
@@ -152,8 +152,7 @@ abstract class NonWindowOuterJoin(
* Return approximate number of records in corresponding state. Only check if record number is
* 0, 1 or bigger.
*/
- protected def approxiRecordNumInState(
- currentSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
+ def approxiRecordNumInState(currentSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
var recordNum = 0L
val it = currentSideState.iterator()
while(it.hasNext && recordNum < 2) {
@@ -165,7 +164,7 @@ abstract class NonWindowOuterJoin(
/**
* Append input row with default null value if there is no match and Collect.
*/
- protected def collectAppendNull(
+ def collectAppendNull(
inputRow: Row,
inputFromLeft: Boolean,
out: Collector[Row]): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
index 0eb6a81..8fe2f4f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
@@ -21,6 +21,7 @@ import org.apache.flink.api.common.state._
import org.apache.flink.api.common.typeinfo.TypeInformation
import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.table.api.{StreamQueryConfig, Types}
import org.apache.flink.table.runtime.types.CRow
import org.apache.flink.types.Row
@@ -82,7 +83,7 @@ import org.apache.flink.types.Row
* unmatched or vice versa. The RowWrapper has been reset before we call retractJoin and we
* also assume that the current change of cRowWrapper is equal to value.change.
*/
- protected def retractJoinWithNonEquiPreds(
+ def retractJoinWithNonEquiPreds(
value: CRow,
inputRowFromLeft: Boolean,
otherSideState: MapState[Row, JTuple2[Long, Long]],
@@ -131,6 +132,48 @@ import org.apache.flink.types.Row
}
/**
+ * Removes records which are expired from state. Registers a new timer if the state still
+ * holds records after the clean-up. Also, clear joinCnt map state when clear rowMapState.
+ */
+ def expireOutTimeRow(
+ curTime: Long,
+ rowMapState: MapState[Row, JTuple2[Long, Long]],
+ timerState: ValueState[Long],
+ isLeft: Boolean,
+ joinCntState: Array[MapState[Row, Long]],
+ ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
+
+ val currentJoinCntState = getJoinCntState(joinCntState, isLeft)
+ val rowMapIter = rowMapState.iterator()
+ var validTimestamp: Boolean = false
+
+ while (rowMapIter.hasNext) {
+ val mapEntry = rowMapIter.next()
+ val recordExpiredTime = mapEntry.getValue.f1
+ if (recordExpiredTime <= curTime) {
+ rowMapIter.remove()
+ currentJoinCntState.remove(mapEntry.getKey)
+ } else {
+ // we found a timestamp that is still valid
+ validTimestamp = true
+ }
+ }
+ // If the state has non-expired timestamps, register a new timer.
+ // Otherwise clean the complete state for this input.
+ if (validTimestamp) {
+ val cleanupTime = curTime + maxRetentionTime
+ ctx.timerService.registerProcessingTimeTimer(cleanupTime)
+ timerState.update(cleanupTime)
+ } else {
+ timerState.clear()
+ rowMapState.clear()
+ if (isLeft == isLeftJoin) {
+ currentJoinCntState.clear()
+ }
+ }
+ }
+
+ /**
* Get left or right join cnt state.
*
* @param joinCntState the join cnt state array, index 0 is left join cnt state, index 1
@@ -138,7 +181,7 @@ import org.apache.flink.types.Row
* @param isLeftCntState the flag whether get the left join cnt state
* @return the corresponding join cnt state
*/
- protected def getJoinCntState(
+ def getJoinCntState(
joinCntState: Array[MapState[Row, Long]],
isLeftCntState: Boolean)
: MapState[Row, Long] = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
index 6ae5e63..3c18449 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
@@ -71,10 +71,6 @@ class StateCleaningCountTrigger(queryConfig: StreamQueryConfig, maxCount: Long)
val cleanupTime = currentTime + maxRetentionTime
// register timer and remember clean-up time
ctx.registerProcessingTimeTimer(cleanupTime)
- // delete expired timer
- if (curCleanupTime != null) {
- ctx.deleteProcessingTimeTimer(curCleanupTime)
- }
ctx.getPartitionedState(cleanupStateDesc).update(cleanupTime)
}
@@ -97,9 +93,12 @@ class StateCleaningCountTrigger(queryConfig: StreamQueryConfig, maxCount: Long)
ctx: TriggerContext): TriggerResult = {
if (stateCleaningEnabled) {
- // do clear directly, since we have already deleted expired timer
- clear(window, ctx)
- return TriggerResult.FIRE_AND_PURGE
+ val cleanupTime = ctx.getPartitionedState(cleanupStateDesc).value()
+ // check that the triggered timer is the last registered processing time timer.
+ if (null != cleanupTime && time == cleanupTime) {
+ clear(window, ctx)
+ return TriggerResult.FIRE_AND_PURGE
+ }
}
TriggerResult.CONTINUE
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
index 4619c75..bd19be8 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
@@ -21,18 +21,21 @@ import java.lang.{Integer => JInt, Long => JLong}
import java.util.concurrent.ConcurrentLinkedQueue
import org.apache.flink.api.common.time.Time
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
import org.apache.flink.api.java.operators.join.JoinType
+import org.apache.flink.api.java.typeutils.RowTypeInfo
import org.apache.flink.streaming.api.operators.co.KeyedCoProcessOperator
import org.apache.flink.streaming.api.watermark.Watermark
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
import org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness
-import org.apache.flink.table.api.Types
-import org.apache.flink.table.runtime.harness.HarnessTestBase.{TestStreamQueryConfig, TupleRowKeySelector}
+import org.apache.flink.table.api.{StreamQueryConfig, Types}
+import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, RowResultSortComparatorWithWatermarks, TestStreamQueryConfig, TupleRowKeySelector}
import org.apache.flink.table.runtime.join._
import org.apache.flink.table.runtime.operators.KeyedCoProcessOperatorWithWatermarkDelay
-import org.apache.flink.table.runtime.types.CRow
-import org.junit.Assert.assertEquals
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.types.Row
+import org.junit.Assert.{assertEquals, assertTrue}
import org.junit.Test
/**
@@ -827,6 +830,14 @@ class JoinHarnessTest extends HarnessTestBase {
@Test
def testNonWindowInnerJoin() {
+ val joinReturnType = CRowTypeInfo(new RowTypeInfo(
+ Array[TypeInformation[_]](
+ INT_TYPE_INFO,
+ STRING_TYPE_INFO,
+ INT_TYPE_INFO,
+ STRING_TYPE_INFO),
+ Array("a", "b", "c", "d")))
+
val joinProcessFunc = new NonWindowInnerJoin(
rowType,
rowType,
@@ -868,32 +879,35 @@ class JoinHarnessTest extends HarnessTestBase {
// right stream input and output normally
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi1")))
- // lkeys(1, 2) rkeys(1) timer_key_time(1:5, 2:6)
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(6, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello1")))
- // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
- assertEquals(6, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(8, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
- // expired stream record with key value of 1
+ // expired left stream record with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi2")))
- // lkeys(2) rkeys(1, 2) timer_key_time(1:9, 2:6)
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(6, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
- // expired all left stream records
+ // expired all left stream record
testHarness.setProcessingTime(6)
- // lkeys() rkeys(1) timer_key_time(1:9)
+ assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+
+ // expired right stream record with key value of 2
+ testHarness.setProcessingTime(8)
assertEquals(2, testHarness.numKeyedStateEntries())
assertEquals(1, testHarness.numProcessingTimeTimers())
- // expired all stream records
- testHarness.setProcessingTime(9)
+ testHarness.setProcessingTime(10)
+ assertTrue(testHarness.numKeyedStateEntries() > 0)
+ // expired all right stream record
+ testHarness.setProcessingTime(11)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -961,37 +975,32 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
- assertEquals(4, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello1")))
- // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
- // expired stream records with key value of 1
+ // expired left stream record with key value of 1
testHarness.setProcessingTime(5)
- // lkeys(2) rkeys(2) timer_key_time(2:6)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi2")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi2")))
- // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
- assertEquals(4, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
- // expired all stream records
+ // expired all left stream record
testHarness.setProcessingTime(6)
- // lkeys() rkeys() timer_key_time(1:9)
- assertEquals(1, testHarness.numKeyedStateEntries())
- assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(3, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all data
- testHarness.setProcessingTime(9)
+ // expired right stream record with key value of 2
+ testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1058,36 +1067,32 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
- assertEquals(4, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello1")))
- // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
- // expired stream records with key value of 1
+ // expired left stream record with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi2")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi2")))
- // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
- assertEquals(4, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
- // expired stream records with key value of 2
+ // expired all left stream record
testHarness.setProcessingTime(6)
- // lkeys() rkeys() timer_key_time(1:9)
- assertEquals(1, testHarness.numKeyedStateEntries())
- assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(3, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all data
- testHarness.setProcessingTime(9)
+ // expired right stream record with key value of 2
+ testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1155,7 +1160,7 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "bbb")))
assertEquals(1, testHarness.numProcessingTimeTimers())
- // lkeys(1) rkeys() timer_key_time(1:5)
+ // 1 left timer(5), 1 left key(1), 1 join cnt
assertEquals(3, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
testHarness.processElement1(new StreamRecord(
@@ -1163,8 +1168,7 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "bbb")))
assertEquals(2, testHarness.numProcessingTimeTimers())
- // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
- // l_join_cnt_keys(1, 2)
+ // 2 left timer(5,6), 2 left key(1,2), 2 join cnt
assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1173,19 +1177,17 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "bbb")))
- // lkeys(1, 2) rkeys(1) timer_key_time(1:5, 2:6)
- // l_join_cnt_keys(1, 2)
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,6), 2 left keys(1,2), 2 join cnt, 1 right timer(7), 1 right key(1)
+ assertEquals(8, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "ccc")))
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "Hello")))
- // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
- // l_join_cnt_keys(1, 2)
- assertEquals(8, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,6), 2 left keys(1,2), 2 join cnt, 2 right timer(7,8), 2 right key(1,2)
+ assertEquals(10, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
@@ -1195,29 +1197,22 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "Hi2")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // lkeys(1, 2) rkeys(2) timer_key_time(1:8, 2:6)
- // l_join_cnt_keys(1, 2)
- assertEquals(7, testHarness.numKeyedStateEntries())
+ // expired left stream record with key value of 1
testHarness.setProcessingTime(5)
- // [1]. this will clean up left stream records with expired time of 5
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "Hi3")))
- // [2]. there are no elements can be connected, since be cleaned by [1]
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "Hi3")))
- // lkeys(1, 2) rkeys(2) timer_key_time(1:8, 2:6)
- // l_join_cnt_keys(1, 2)
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 1 left timer(6), 1 left keys(2), 1 join cnt, 2 right timer(7,8), 1 right key(2)
+ assertEquals(6, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
- // expired all records with key value of 2
+ // expired all left stream record
testHarness.setProcessingTime(6)
- // lkeys(1) rkeys() timer_key_time(1:8)
- // l_join_cnt_keys(1)
assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all data
+ // expired right stream record with key value of 2
testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1258,12 +1253,6 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "bbb", 1: JInt, "Hi1")))
expectedOutput.add(new StreamRecord(
CRow(1: JInt, "bbb", null: JInt, null)))
- // processing time of 5
- // timer of 8, we use only one timer state now
- expectedOutput.add(new StreamRecord(
- CRow(false, 1: JInt, "bbb", null: JInt, null)))
- expectedOutput.add(new StreamRecord(
- CRow(1: JInt, "bbb", 1: JInt, "Hi3")))
verify(expectedOutput, result)
testHarness.close()
@@ -1316,36 +1305,32 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // lkeys() rkeys(1, 2) timer_key_time(1:5, 2:6)
- assertEquals(4, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "Hello1")))
- // lkeys(2) rkeys(1, 2) timer_key_time(1:5, 2:6)
- assertEquals(5, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(7, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
- // expired stream records with key value of 1
+ // expired right stream record with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "Hi2")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi2")))
- // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
- assertEquals(4, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
- // expired stream records with key value of 2
+ // expired all right stream record
testHarness.setProcessingTime(6)
- // lkeys() rkeys() timer_key_time(1:9)
- assertEquals(1, testHarness.numKeyedStateEntries())
- assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(3, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all data
- testHarness.setProcessingTime(9)
+ // expired left stream record with key value of 2
+ testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1413,17 +1398,15 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "bbb")))
assertEquals(1, testHarness.numProcessingTimeTimers())
- // lkeys() rkeys(1) timer_key_time(1:5)
- // r_join_cnt_keys(1)
+ // 1 right timer(5), 1 right key(1), 1 join cnt
assertEquals(3, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "bbb")))
- // lkeys() rkeys(1, 2) timer_key_time(1:5, 2:6)
- // r_join_cnt_keys(1, 2)
assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 2 right timer(5,6), 2 right key(1,2), 2 join cnt
assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1432,19 +1415,17 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "Hi1")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "bbb")))
- // lkeys(1) rkeys(1, 2) timer_key_time(1:5, 2:6)
- // r_join_cnt_keys(1, 2)
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 2 right timer(5,6), 2 right keys(1,2), 2 join cnt, 1 left timer(7), 1 left key(1)
+ assertEquals(8, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "ccc")))
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "Hello")))
- // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
- // r_join_cnt_keys(1, 2)
- assertEquals(8, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 2 right timer(5,6), 2 right keys(1,2), 2 join cnt, 2 left timer(7,8), 2 left key(1,2)
+ assertEquals(10, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "aaa")))
@@ -1454,27 +1435,22 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "Hi2")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi1")))
- // lkeys(2) rkeys(1, 2) timer_key_time(1:8, 2:6)
- // r_join_cnt_keys(1, 2)
- assertEquals(7, testHarness.numKeyedStateEntries())
+ // expired right stream record with key value of 1
testHarness.setProcessingTime(5)
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "Hi3")))
testHarness.processElement1(new StreamRecord(
CRow(false, 1: JInt, "Hi3")))
- // lkeys(2) rkeys(1, 2) timer_key_time(1:8, 2:6)
- // r_join_cnt_keys(1, 2)
- assertEquals(7, testHarness.numKeyedStateEntries())
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 1 right timer(6), 1 right keys(2), 1 join cnt, 2 left timer(7,8), 1 left key(2)
+ assertEquals(6, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
- // expired all stream records with key value of 2
- // lkeys() rkeys(1) timer_key_time(1:8)
- // r_join_cnt_keys(1)
+ // expired all right stream record
testHarness.setProcessingTime(6)
assertEquals(3, testHarness.numKeyedStateEntries())
- assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
- // expired all data
+ // expired left stream record with key value of 2
testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numKeyedStateEntries())
assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1515,12 +1491,6 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "Hi1", 1: JInt, "bbb")))
expectedOutput.add(new StreamRecord(
CRow(null: JInt, null, 1: JInt, "bbb")))
- // processing time of 5
- // timer of 8, we use only one timer state now
- expectedOutput.add(new StreamRecord(
- CRow(false, null: JInt, null, 1: JInt, "bbb")))
- expectedOutput.add(new StreamRecord(
- CRow(1: JInt, "Hi3", 1: JInt, "bbb")))
verify(expectedOutput, result)
testHarness.close()
@@ -1554,8 +1524,8 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "bbb")))
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "ccc")))
- // lkeys(1) rkeys() timer_key_time(1:5)
assertEquals(1, testHarness.numProcessingTimeTimers())
+ // 1 left timer(5), 1 left key(1)
assertEquals(2, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
@@ -1564,7 +1534,8 @@ class JoinHarnessTest extends HarnessTestBase {
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "ccc")))
assertEquals(2, testHarness.numProcessingTimeTimers())
- // lkeys(1) rkeys(2) timer_key_time(1:5, 2:6)
+ // 1 left timer(5), 1 left key(1)
+ // 1 right timer(6), 1 right key(1)
assertEquals(4, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1572,16 +1543,18 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(2: JInt, "aaa")))
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "ddd")))
- // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,7), 2 left key(1,2)
+ // 1 right timer(6), 1 right key(1)
+ assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "ddd")))
- // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(6, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,7), 2 left key(1,2)
+ // 2 right timer(6,7), 2 right key(1,2)
+ assertEquals(8, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
@@ -1592,26 +1565,28 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(false, 1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "ddd")))
- // lkeys(1) rkeys(2) timer_key_time(1:8, 2:6)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(4, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,7), 1 left key(1)
+ // 2 right timer(6,7), 1 right key(2)
+ assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(5)
- assertEquals(2, testHarness.numProcessingTimeTimers())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
+ // 1 left timer(7)
+ // 2 right timer(6,7), 1 right key(2)
assertEquals(4, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(6)
- // lkeys(1) rkeys() timer_key_time(1:8)
- assertEquals(1, testHarness.numProcessingTimeTimers())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 1 left timer(7)
+ // 2 right timer(7)
assertEquals(2, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(7)
- assertEquals(1, testHarness.numProcessingTimeTimers())
- assertEquals(2, testHarness.numKeyedStateEntries())
-
- testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numProcessingTimeTimers())
assertEquals(0, testHarness.numKeyedStateEntries())
+
+ testHarness.setProcessingTime(8)
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "bbb")))
testHarness.processElement2(new StreamRecord(
@@ -1718,9 +1693,8 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(1: JInt, "bbb")))
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "ccc")))
- // lkeys(1) rkeys() timer_key_time(1:5)
- // l_join_cnt_keys(1) r_join_cnt_keys()
assertEquals(1, testHarness.numProcessingTimeTimers())
+ // 1 left timer(5), 1 left key(1), 1 left joincnt key(1)
assertEquals(3, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(2)
@@ -1728,9 +1702,9 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(2: JInt, "bbb")))
testHarness.processElement2(new StreamRecord(
CRow(2: JInt, "ccc")))
- // lkeys(1) rkeys(2) timer_key_time(1:5, 2:6)
- // l_join_cnt_keys(1) r_join_cnt_keys(2)
assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 1 left timer(5), 1 left key(1), 1 left joincnt key(1)
+ // 1 right timer(6), 1 right key(1), 1 right joincnt key(1)
assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(3)
@@ -1738,46 +1712,46 @@ class JoinHarnessTest extends HarnessTestBase {
CRow(2: JInt, "aaa")))
testHarness.processElement1(new StreamRecord(
CRow(2: JInt, "ddd")))
- // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
- // l_join_cnt_keys(1, 2) r_join_cnt_keys(2)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(8, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
+ // 1 right timer(6), 1 right key(1), 1 right joincnt key(1)
+ assertEquals(9, testHarness.numKeyedStateEntries())
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(1: JInt, "ddd")))
- // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
- // l_join_cnt_keys(1, 2) r_join_cnt_keys(1, 2)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(10, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
+ // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
+ assertEquals(12, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(4)
testHarness.processElement1(new StreamRecord(
CRow(false, 2: JInt, "aaa")))
testHarness.processElement2(new StreamRecord(
CRow(false, 1: JInt, "ddd")))
- // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:8, 2:6)
- // l_join_cnt_keys(1, 2) r_join_cnt_keys(1, 2)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(10, testHarness.numKeyedStateEntries())
+ assertEquals(4, testHarness.numProcessingTimeTimers())
+ // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
+ // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
+ assertEquals(12, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(5)
- assertEquals(2, testHarness.numProcessingTimeTimers())
- assertEquals(10, testHarness.numKeyedStateEntries())
+ assertEquals(3, testHarness.numProcessingTimeTimers())
+ // 1 left timer(7), 1 left key(2), 1 left joincnt key(2)
+ // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
+ assertEquals(9, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(6)
- // lkeys(1) rkeys(1) timer_key_time(1:8)
- // l_join_cnt_keys(1) r_join_cnt_keys(1)
- assertEquals(1, testHarness.numProcessingTimeTimers())
- assertEquals(5, testHarness.numKeyedStateEntries())
+ assertEquals(2, testHarness.numProcessingTimeTimers())
+ // 1 left timer(7), 1 left key(2), 1 left joincnt key(2)
+ // 1 right timer(7), 1 right key(2), 1 right joincnt key(2)
+ assertEquals(6, testHarness.numKeyedStateEntries())
testHarness.setProcessingTime(7)
- assertEquals(1, testHarness.numProcessingTimeTimers())
- assertEquals(5, testHarness.numKeyedStateEntries())
-
- testHarness.setProcessingTime(8)
assertEquals(0, testHarness.numProcessingTimeTimers())
assertEquals(0, testHarness.numKeyedStateEntries())
+
+ testHarness.setProcessingTime(8)
testHarness.processElement1(new StreamRecord(
CRow(1: JInt, "bbb")))
testHarness.processElement2(new StreamRecord(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
index 25395be..7f9c0ef 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
@@ -80,8 +80,8 @@ class StateCleaningCountTriggerHarnessTest {
TriggerResult.CONTINUE,
testHarness.processElement(new StreamRecord(1), GlobalWindow.get))
- // have one timer 7002
- assertEquals(1, testHarness.numProcessingTimeTimers)
+ // have two timers 6001 and 7002
+ assertEquals(2, testHarness.numProcessingTimeTimers)
assertEquals(0, testHarness.numEventTimeTimers)
assertEquals(2, testHarness.numStateEntries)
assertEquals(2, testHarness.numStateEntries(GlobalWindow.get))
@@ -116,6 +116,9 @@ class StateCleaningCountTriggerHarnessTest {
// try to trigger onProcessingTime method via 7002, and all states are cleared
val timesIt = testHarness.advanceProcessingTime(7002).iterator()
+ assertEquals(
+ TriggerResult.CONTINUE,
+ timesIt.next().f1)
assertEquals(
TriggerResult.FIRE_AND_PURGE,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
index 1c02889..fe90a5f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
@@ -110,7 +110,7 @@ private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig)
out: Collector[String]): Unit = {
val curTime = ctx.timerService().currentProcessingTime()
- processCleanupTimer(ctx, curTime)
+ registerProcessingCleanupTimer(ctx, curTime)
state.update(value._2)
}
@@ -119,12 +119,8 @@ private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig)
ctx: KeyedProcessFunction[String, (String, String), String]#OnTimerContext,
out: Collector[String]): Unit = {
- if (stateCleaningEnabled) {
- val cleanupTime = cleanupTimeState.value()
- if (null != cleanupTime && timestamp == cleanupTime) {
- // clean up
- cleanupState(state)
- }
+ if (needToCleanupState(timestamp)) {
+ cleanupState(state)
}
}
}
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
index 6c0ca1a..519b03f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
@@ -110,7 +110,7 @@ private class MockedProcessFunction(queryConfig: StreamQueryConfig)
out: Collector[String]): Unit = {
val curTime = ctx.timerService().currentProcessingTime()
- processCleanupTimer(ctx, curTime)
+ registerProcessingCleanupTimer(ctx, curTime)
state.update(value._2)
}
@@ -119,7 +119,7 @@ private class MockedProcessFunction(queryConfig: StreamQueryConfig)
ctx: ProcessFunction[(String, String), String]#OnTimerContext,
out: Collector[String]): Unit = {
- if (stateCleaningEnabled) {
+ if (needToCleanupState(timestamp)) {
cleanupState(state)
}
}