You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by ji...@apache.org on 2018/12/11 07:07:08 UTC

[flink] branch release-1.7 updated: In order to maintain state compatibility revert FLINK-10543.

This is an automated email from the ASF dual-hosted git repository.

jincheng pushed a commit to branch release-1.7
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.7 by this push:
     new d396633  In order to maintain state compatibility revert FLINK-10543.
d396633 is described below

commit d3966332d4fe37fd22f92c758a6a34097ed16ac6
Author: sunjincheng121 <su...@gmail.com>
AuthorDate: Tue Dec 11 14:43:40 2018 +0800

    In order to maintain state compatibility revert FLINK-10543.
    
    This reverts commit 0fa9ec030d6b102f4d24f0c7f8b58c0fab97fff6.
---
 .../table/runtime/aggregate/CleanupState.scala     |  57 ----
 .../CoProcessFunctionWithCleanupState.scala        |  69 -----
 .../aggregate/GroupAggProcessFunction.scala        |   4 +-
 .../KeyedProcessFunctionWithCleanupState.scala     |  38 ++-
 .../aggregate/ProcTimeBoundedRangeOver.scala       |  29 +-
 .../aggregate/ProcTimeBoundedRowsOver.scala        |   4 +-
 .../runtime/aggregate/ProcTimeUnboundedOver.scala  |   4 +-
 .../ProcessFunctionWithCleanupState.scala          |  41 ++-
 .../aggregate/RowTimeBoundedRangeOver.scala        |  20 +-
 .../runtime/aggregate/RowTimeBoundedRowsOver.scala |   8 +-
 .../runtime/aggregate/RowTimeUnboundedOver.scala   |   8 +-
 .../table/runtime/join/NonWindowFullJoin.scala     |   3 +-
 .../NonWindowFullJoinWithNonEquiPredicates.scala   |  23 +-
 .../table/runtime/join/NonWindowInnerJoin.scala    |   3 +-
 .../flink/table/runtime/join/NonWindowJoin.scala   | 104 +++++--
 .../runtime/join/NonWindowLeftRightJoin.scala      |   3 +-
 ...nWindowLeftRightJoinWithNonEquiPredicates.scala |  25 +-
 .../table/runtime/join/NonWindowOuterJoin.scala    |   9 +-
 .../NonWindowOuterJoinWithNonEquiPredicates.scala  |  47 ++-
 .../triggers/StateCleaningCountTrigger.scala       |  13 +-
 .../table/runtime/harness/JoinHarnessTest.scala    | 320 ++++++++++-----------
 .../StateCleaningCountTriggerHarnessTest.scala     |   7 +-
 .../KeyedProcessFunctionWithCleanupStateTest.scala |  10 +-
 .../ProcessFunctionWithCleanupStateTest.scala      |   4 +-
 24 files changed, 430 insertions(+), 423 deletions(-)

diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala
deleted file mode 100644
index d9c8e2c..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CleanupState.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.runtime.aggregate
-
-import org.apache.flink.api.common.state.ValueState
-import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.apache.flink.streaming.api.functions.co.CoProcessFunction
-import java.lang.{Long => JLong}
-
-import org.apache.flink.streaming.api.TimerService
-
-/**
-  * Base class for clean up state, both for [[ProcessFunction]] and [[CoProcessFunction]].
-  */
-trait CleanupState {
-
-  def registerProcessingCleanupTimer(
-      cleanupTimeState: ValueState[JLong],
-      currentTime: Long,
-      minRetentionTime: Long,
-      maxRetentionTime: Long,
-      timerService: TimerService): Unit = {
-
-    // last registered timer
-    val curCleanupTime = cleanupTimeState.value()
-
-    // check if a cleanup timer is registered and
-    // that the current cleanup timer won't delete state we need to keep
-    if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
-      // we need to register a new (later) timer
-      val cleanupTime = currentTime + maxRetentionTime
-      // register timer and remember clean-up time
-      timerService.registerProcessingTimeTimer(cleanupTime)
-      // delete expired timer
-      if (curCleanupTime != null) {
-        timerService.deleteProcessingTimeTimer(curCleanupTime)
-      }
-      cleanupTimeState.update(cleanupTime)
-    }
-  }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala
deleted file mode 100644
index 0c76636..0000000
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/CoProcessFunctionWithCleanupState.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.table.runtime.aggregate
-
-import java.lang.{Long => JLong}
-
-import org.apache.flink.api.common.state.{State, ValueState, ValueStateDescriptor}
-import org.apache.flink.streaming.api.TimeDomain
-import org.apache.flink.streaming.api.functions.co.CoProcessFunction
-import org.apache.flink.table.api.{StreamQueryConfig, Types}
-
-abstract class CoProcessFunctionWithCleanupState[IN1, IN2, OUT](queryConfig: StreamQueryConfig)
-  extends CoProcessFunction[IN1, IN2, OUT]
-  with CleanupState {
-
-  protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
-  protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
-  protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
-
-  // holds the latest registered cleanup timer
-  private var cleanupTimeState: ValueState[JLong] = _
-
-  protected def initCleanupTimeState(stateName: String) {
-    if (stateCleaningEnabled) {
-      val cleanupTimeDescriptor: ValueStateDescriptor[JLong] =
-        new ValueStateDescriptor[JLong](stateName, Types.LONG)
-      cleanupTimeState = getRuntimeContext.getState(cleanupTimeDescriptor)
-    }
-  }
-
-  protected def processCleanupTimer(
-    ctx: CoProcessFunction[IN1, IN2, OUT]#Context,
-    currentTime: Long): Unit = {
-    if (stateCleaningEnabled) {
-      registerProcessingCleanupTimer(
-        cleanupTimeState,
-        currentTime,
-        minRetentionTime,
-        maxRetentionTime,
-        ctx.timerService()
-      )
-    }
-  }
-
-  protected def isProcessingTimeTimer(ctx: OnTimerContext): Boolean = {
-    ctx.timeDomain() == TimeDomain.PROCESSING_TIME
-  }
-
-  protected def cleanupState(states: State*): Unit = {
-    // clear all state
-    states.foreach(_.clear())
-    this.cleanupTimeState.clear()
-  }
-}
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
index 2d72e6d..c59efe2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/GroupAggProcessFunction.scala
@@ -86,7 +86,7 @@ class GroupAggProcessFunction(
 
     val currentTime = ctx.timerService().currentProcessingTime()
     // register state-cleanup timer
-    processCleanupTimer(ctx, currentTime)
+    registerProcessingCleanupTimer(ctx, currentTime)
 
     val input = inputC.row
 
@@ -172,7 +172,7 @@ class GroupAggProcessFunction(
       ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
       out: Collector[CRow]): Unit = {
 
-    if (stateCleaningEnabled) {
+    if (needToCleanupState(timestamp)) {
       cleanupState(state, cntState)
       function.cleanup()
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
index edf5c2c..4d6840a 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/KeyedProcessFunctionWithCleanupState.scala
@@ -25,15 +25,13 @@ import org.apache.flink.streaming.api.functions.{KeyedProcessFunction, ProcessFu
 import org.apache.flink.table.api.{StreamQueryConfig, Types}
 
 abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: StreamQueryConfig)
-  extends KeyedProcessFunction[K, I, O]
-  with CleanupState {
-
+  extends KeyedProcessFunction[K, I, O] {
   protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
   protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
   protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
 
   // holds the latest registered cleanup timer
-  protected var cleanupTimeState: ValueState[JLong] = _
+  private var cleanupTimeState: ValueState[JLong] = _
 
   protected def initCleanupTimeState(stateName: String) {
     if (stateCleaningEnabled) {
@@ -43,17 +41,23 @@ abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: Stream
     }
   }
 
-  protected def processCleanupTimer(
+  protected def registerProcessingCleanupTimer(
     ctx: KeyedProcessFunction[K, I, O]#Context,
     currentTime: Long): Unit = {
     if (stateCleaningEnabled) {
-      registerProcessingCleanupTimer(
-        cleanupTimeState,
-        currentTime,
-        minRetentionTime,
-        maxRetentionTime,
-        ctx.timerService()
-      )
+
+      // last registered timer
+      val curCleanupTime = cleanupTimeState.value()
+
+      // check if a cleanup timer is registered and
+      // that the current cleanup timer won't delete state we need to keep
+      if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
+        // we need to register a new (later) timer
+        val cleanupTime = currentTime + maxRetentionTime
+        // register timer and remember clean-up time
+        ctx.timerService().registerProcessingTimeTimer(cleanupTime)
+        cleanupTimeState.update(cleanupTime)
+      }
     }
   }
 
@@ -61,6 +65,16 @@ abstract class KeyedProcessFunctionWithCleanupState[K, I, O](queryConfig: Stream
     ctx.timeDomain() == TimeDomain.PROCESSING_TIME
   }
 
+  protected def needToCleanupState(timestamp: Long): Boolean = {
+    if (stateCleaningEnabled) {
+      val cleanupTime = cleanupTimeState.value()
+      // check that the triggered timer is the last registered processing time timer.
+      null != cleanupTime && timestamp == cleanupTime
+    } else {
+      false
+    }
+  }
+
   protected def cleanupState(states: State*): Unit = {
     // clear all state
     states.foreach(_.clear())
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
index 6126dc7..591b942 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRangeOver.scala
@@ -95,7 +95,7 @@ class ProcTimeBoundedRangeOver(
 
     val currentTime = ctx.timerService.currentProcessingTime
     // register state-cleanup timer
-    processCleanupTimer(ctx, currentTime)
+    registerProcessingCleanupTimer(ctx, currentTime)
 
     // buffer the event incoming event
 
@@ -117,14 +117,11 @@ class ProcTimeBoundedRangeOver(
     ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
     out: Collector[CRow]): Unit = {
 
-    if (stateCleaningEnabled) {
-      val cleanupTime = cleanupTimeState.value()
-      if (null != cleanupTime && timestamp == cleanupTime) {
-        // clean up and return
-        cleanupState(rowMapState, accumulatorState)
-        function.cleanup()
-        return
-      }
+    if (needToCleanupState(timestamp)) {
+      // clean up and return
+      cleanupState(rowMapState, accumulatorState)
+      function.cleanup()
+      return
     }
 
     // remove timestamp set outside of ProcessFunction.
@@ -134,10 +131,11 @@ class ProcTimeBoundedRangeOver(
     // that have registered this time trigger 1 ms ago
 
     val currentTime = timestamp - 1
+    var i = 0
     // get the list of elements of current proctime
     val currentElements = rowMapState.get(currentTime)
 
-    // Expired clean-up timers pass the needToCleanupState check.
+    // Expired clean-up timers pass the needToCleanupState() check.
     // Perform a null check to verify that we have data to process.
     if (null == currentElements) {
       return
@@ -158,6 +156,7 @@ class ProcTimeBoundedRangeOver(
     // and eliminate them. Multiple elements could have been received at the same timestamp
     // the removal of old elements happens only once per proctime as onTimer is called only once
     val iter = rowMapState.iterator
+    val markToRemove = new ArrayList[Long]()
     while (iter.hasNext) {
       val entry = iter.next()
       val elementKey = entry.getKey
@@ -170,9 +169,17 @@ class ProcTimeBoundedRangeOver(
           function.retract(accumulators, retractRow)
           iRemove += 1
         }
-        iter.remove()
+        // mark element for later removal not to modify the iterator over MapState
+        markToRemove.add(elementKey)
       }
     }
+    // need to remove in 2 steps not to have concurrent access errors via iterator to the MapState
+    i = 0
+    while (i < markToRemove.size()) {
+      rowMapState.remove(markToRemove.get(i))
+      i += 1
+    }
+
 
     // add current elements to aggregator. Multiple elements might
     // have arrived in the same proctime
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
index fa58ac5..ccddaa5 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeBoundedRowsOver.scala
@@ -110,7 +110,7 @@ class ProcTimeBoundedRowsOver(
     val currentTime = ctx.timerService.currentProcessingTime
 
     // register state-cleanup timer
-    processCleanupTimer(ctx, currentTime)
+    registerProcessingCleanupTimer(ctx, currentTime)
 
     // initialize state for the processed element
     var accumulators = accumulatorState.value
@@ -187,7 +187,7 @@ class ProcTimeBoundedRowsOver(
     ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
     out: Collector[CRow]): Unit = {
 
-    if (stateCleaningEnabled) {
+    if (needToCleanupState(timestamp)) {
       cleanupState(rowMapState, accumulatorState, counterState, smallestTsState)
       function.cleanup()
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
index ce1a959..6e4c510 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcTimeUnboundedOver.scala
@@ -71,7 +71,7 @@ class ProcTimeUnboundedOver(
     out: Collector[CRow]): Unit = {
 
     // register state-cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
 
     val input = inputC.row
 
@@ -95,7 +95,7 @@ class ProcTimeUnboundedOver(
     ctx: ProcessFunction[CRow, CRow]#OnTimerContext,
     out: Collector[CRow]): Unit = {
 
-    if (stateCleaningEnabled) {
+    if (needToCleanupState(timestamp)) {
       cleanupState(state)
       function.cleanup()
     }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
index 7263de7..292fd3b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/ProcessFunctionWithCleanupState.scala
@@ -26,35 +26,40 @@ import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.apache.flink.table.api.{StreamQueryConfig, Types}
 
 abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryConfig)
-  extends ProcessFunction[IN, OUT]
-  with CleanupState {
+  extends ProcessFunction[IN, OUT]{
 
   protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
   protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
   protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
 
   // holds the latest registered cleanup timer
-  protected var cleanupTimeState: ValueState[JLong] = _
+  private var cleanupTimeState: ValueState[JLong] = _
 
   protected def initCleanupTimeState(stateName: String) {
     if (stateCleaningEnabled) {
-      val cleanupTimeDescriptor: ValueStateDescriptor[JLong] =
+      val inputCntDescriptor: ValueStateDescriptor[JLong] =
         new ValueStateDescriptor[JLong](stateName, Types.LONG)
-      cleanupTimeState = getRuntimeContext.getState(cleanupTimeDescriptor)
+      cleanupTimeState = getRuntimeContext.getState(inputCntDescriptor)
     }
   }
 
-  protected def processCleanupTimer(
+  protected def registerProcessingCleanupTimer(
     ctx: ProcessFunction[IN, OUT]#Context,
     currentTime: Long): Unit = {
     if (stateCleaningEnabled) {
-      registerProcessingCleanupTimer(
-        cleanupTimeState,
-        currentTime,
-        minRetentionTime,
-        maxRetentionTime,
-        ctx.timerService()
-      )
+
+      // last registered timer
+      val curCleanupTime = cleanupTimeState.value()
+
+      // check if a cleanup timer is registered and
+      // that the current cleanup timer won't delete state we need to keep
+      if (curCleanupTime == null || (currentTime + minRetentionTime) > curCleanupTime) {
+        // we need to register a new (later) timer
+        val cleanupTime = currentTime + maxRetentionTime
+        // register timer and remember clean-up time
+        ctx.timerService().registerProcessingTimeTimer(cleanupTime)
+        cleanupTimeState.update(cleanupTime)
+      }
     }
   }
 
@@ -62,6 +67,16 @@ abstract class ProcessFunctionWithCleanupState[IN,OUT](queryConfig: StreamQueryC
     ctx.timeDomain() == TimeDomain.PROCESSING_TIME
   }
 
+  protected def needToCleanupState(timestamp: Long): Boolean = {
+    if (stateCleaningEnabled) {
+      val cleanupTime = cleanupTimeState.value()
+      // check that the triggered timer is the last registered processing time timer.
+      null != cleanupTime && timestamp == cleanupTime
+    } else {
+      false
+    }
+  }
+
   protected def cleanupState(states: State*): Unit = {
     // clear all state
     states.foreach(_.clear())
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
index 7c509d6..b13acdf 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRangeOver.scala
@@ -114,7 +114,7 @@ class RowTimeBoundedRangeOver(
     val input = inputC.row
 
     // register state-cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
 
     // triggering timestamp for trigger calculation
     val triggeringTs = input.getField(rowTimeIdx).asInstanceOf[Long]
@@ -143,7 +143,7 @@ class RowTimeBoundedRangeOver(
     out: Collector[CRow]): Unit = {
 
     if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
-      if (stateCleaningEnabled) {
+      if (needToCleanupState(timestamp)) {
 
         val keysIt = dataState.keys.iterator()
         val lastProcessedTime = lastTriggeringTsState.value
@@ -164,7 +164,7 @@ class RowTimeBoundedRangeOver(
           // There are records left to process because a watermark has not been received yet.
           // This would only happen if the input stream has stopped. So we don't need to clean up.
           // We leave the state as it is and schedule a new cleanup timer
-          processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+          registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
         }
       }
       return
@@ -188,6 +188,9 @@ class RowTimeBoundedRangeOver(
         aggregatesIndex = 0
       }
 
+      // keep up timestamps of retract data
+      val retractTsList: JList[Long] = new JArrayList[Long]
+
       // do retraction
       val iter = dataState.iterator()
       while (iter.hasNext) {
@@ -202,7 +205,7 @@ class RowTimeBoundedRangeOver(
             function.retract(accumulators, retractRow)
             dataListIndex += 1
           }
-          iter.remove()
+          retractTsList.add(dataTs)
         }
       }
 
@@ -227,13 +230,20 @@ class RowTimeBoundedRangeOver(
         dataListIndex += 1
       }
 
+      // remove the data that has been retracted
+      dataListIndex = 0
+      while (dataListIndex < retractTsList.size) {
+        dataState.remove(retractTsList.get(dataListIndex))
+        dataListIndex += 1
+      }
+
       // update state
       accumulatorState.update(accumulators)
     }
     lastTriggeringTsState.update(timestamp)
 
     // update cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
   }
 
   override def close(): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
index d01a499..e120d6b 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeBoundedRowsOver.scala
@@ -123,7 +123,7 @@ class RowTimeBoundedRowsOver(
     val input = inputC.row
 
     // register state-cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
 
     // triggering timestamp for trigger calculation
     val triggeringTs = input.getField(rowTimeIdx).asInstanceOf[Long]
@@ -152,7 +152,7 @@ class RowTimeBoundedRowsOver(
     out: Collector[CRow]): Unit = {
 
     if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
-      if (stateCleaningEnabled) {
+      if (needToCleanupState(timestamp)) {
 
         val keysIt = dataState.keys.iterator()
         val lastProcessedTime = lastTriggeringTsState.value
@@ -173,7 +173,7 @@ class RowTimeBoundedRowsOver(
           // There are records left to process because a watermark has not been received yet.
           // This would only happen if the input stream has stopped. So we don't need to clean up.
           // We leave the state as it is and schedule a new cleanup timer
-          processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+          registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
         }
       }
       return
@@ -263,7 +263,7 @@ class RowTimeBoundedRowsOver(
     lastTriggeringTsState.update(timestamp)
 
     // update cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
   }
 
   override def close(): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
index 690d0d0..181c768 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/aggregate/RowTimeUnboundedOver.scala
@@ -108,7 +108,7 @@ abstract class RowTimeUnboundedOver(
     val input = inputC.row
 
     // register state-cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
 
     val timestamp = input.getField(rowTimeIdx).asInstanceOf[Long]
     val curWatermark = ctx.timerService().currentWatermark()
@@ -143,7 +143,7 @@ abstract class RowTimeUnboundedOver(
       out: Collector[CRow]): Unit = {
 
     if (isProcessingTimeTimer(ctx.asInstanceOf[OnTimerContext])) {
-      if (stateCleaningEnabled) {
+      if (needToCleanupState(timestamp)) {
 
         // we check whether there are still records which have not been processed yet
         val noRecordsToProcess = !rowMapState.keys.iterator().hasNext
@@ -155,7 +155,7 @@ abstract class RowTimeUnboundedOver(
           // There are records left to process because a watermark has not been received yet.
           // This would only happen if the input stream has stopped. So we don't need to clean up.
           // We leave the state as it is and schedule a new cleanup timer
-          processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+          registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
         }
       }
       return
@@ -207,7 +207,7 @@ abstract class RowTimeUnboundedOver(
     }
 
     // update cleanup timer
-    processCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
+    registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime())
   }
 
   /**
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
index 5b1069f..57c60f1 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoin.scala
@@ -66,12 +66,13 @@ class NonWindowFullJoin(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow],
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]],
       otherSideState: MapState[Row, JTuple2[Long, Long]],
       recordFromLeft: Boolean): Unit = {
 
     val inputRow = value.row
-    updateCurrentSide(value, ctx, currentSideState)
+    updateCurrentSide(value, ctx, timerState, currentSideState)
 
     cRowWrapper.reset()
     cRowWrapper.setCollector(out)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
index 0166eef..9c27eb4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowFullJoinWithNonEquiPredicates.scala
@@ -68,13 +68,14 @@ class NonWindowFullJoinWithNonEquiPredicates(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow],
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]],
       otherSideState: MapState[Row, JTuple2[Long, Long]],
       recordFromLeft: Boolean): Unit = {
 
     val currentJoinCntState = getJoinCntState(joinCntState, recordFromLeft)
     val inputRow = value.row
-    val cntAndExpiredTime = updateCurrentSide(value, ctx, currentSideState)
+    val cntAndExpiredTime = updateCurrentSide(value, ctx, timerState, currentSideState)
     if (!value.change && cntAndExpiredTime.f0 <= 0) {
       currentJoinCntState.remove(inputRow)
     }
@@ -98,18 +99,18 @@ class NonWindowFullJoinWithNonEquiPredicates(
   }
 
   /**
-    * Called when a processing timer trigger.
-    * Expire left/right expired records and expired joinCnt state.
+    * Removes records which are expired from left state. Register a new timer if the state still
+    * holds records after the clean-up. Also, clear leftJoinCnt map state when clear left
+    * rowMapState.
     */
-  override def onTimer(
-      timestamp: Long,
-      ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
-      out: Collector[CRow]): Unit = {
+  override def expireOutTimeRow(
+      curTime: Long,
+      rowMapState: MapState[Row, JTuple2[Long, Long]],
+      timerState: ValueState[Long],
+      isLeft: Boolean,
+      ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
 
-    // expired timer has already been removed, delete state directly.
-    if (stateCleaningEnabled) {
-      cleanupState(leftState, rightState, joinCntState(0), joinCntState(1))
-    }
+    expireOutTimeRow(curTime, rowMapState, timerState, isLeft, joinCntState, ctx)
   }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
index 91a7507..2e5832c 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowInnerJoin.scala
@@ -63,12 +63,13 @@ class NonWindowInnerJoin(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow],
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]],
       otherSideState: MapState[Row, JTuple2[Long, Long]],
       isLeft: Boolean): Unit = {
 
     val inputRow = value.row
-    updateCurrentSide(value, ctx, currentSideState)
+    updateCurrentSide(value, ctx, timerState, currentSideState)
 
     cRowWrapper.setCollector(out)
     cRowWrapper.setChange(value.change)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
index e15cbfa..c59f4c2 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowJoin.scala
@@ -19,7 +19,7 @@ package org.apache.flink.table.runtime.join
 
 import org.apache.flink.api.common.functions.FlatJoinFunction
 import org.apache.flink.api.common.functions.util.FunctionUtils
-import org.apache.flink.api.common.state.{MapState, MapStateDescriptor}
+import org.apache.flink.api.common.state.{MapState, MapStateDescriptor, ValueState, ValueStateDescriptor}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.api.java.typeutils.TupleTypeInfo
@@ -27,7 +27,6 @@ import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.co.CoProcessFunction
 import org.apache.flink.table.api.{StreamQueryConfig, Types}
 import org.apache.flink.table.codegen.Compiler
-import org.apache.flink.table.runtime.aggregate.CoProcessFunctionWithCleanupState
 import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.table.typeutils.TypeCheckUtils._
 import org.apache.flink.table.util.Logging
@@ -49,7 +48,7 @@ abstract class NonWindowJoin(
     genJoinFuncName: String,
     genJoinFuncCode: String,
     queryConfig: StreamQueryConfig)
-  extends CoProcessFunctionWithCleanupState[CRow, CRow, CRow](queryConfig)
+  extends CoProcessFunction[CRow, CRow, CRow]
   with Compiler[FlatJoinFunction[Row, Row, Row]]
   with Logging {
 
@@ -63,6 +62,15 @@ abstract class NonWindowJoin(
   protected var rightState: MapState[Row, JTuple2[Long, Long]] = _
   protected var cRowWrapper: CRowWrappingMultiOutputCollector = _
 
+  protected val minRetentionTime: Long = queryConfig.getMinIdleStateRetentionTime
+  protected val maxRetentionTime: Long = queryConfig.getMaxIdleStateRetentionTime
+  protected val stateCleaningEnabled: Boolean = minRetentionTime > 1
+
+  // state to record last timer of left stream, 0 means no timer
+  protected var leftTimer: ValueState[Long] = _
+  // state to record last timer of right stream, 0 means no timer
+  protected var rightTimer: ValueState[Long] = _
+
   // other condition function
   protected var joinFunction: FlatJoinFunction[Row, Row, Row] = _
 
@@ -70,8 +78,7 @@ abstract class NonWindowJoin(
   protected var curProcessTime: Long = _
 
   override def open(parameters: Configuration): Unit = {
-    LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n " +
-                s"Code:\n$genJoinFuncCode")
+    LOG.debug(s"Compiling JoinFunction: $genJoinFuncName \n\n Code:\n$genJoinFuncCode")
     val clazz = compile(
       getRuntimeContext.getUserCodeClassLoader,
       genJoinFuncName,
@@ -93,7 +100,10 @@ abstract class NonWindowJoin(
     rightState = getRuntimeContext.getMapState(rightStateDescriptor)
 
     // initialize timer state
-    initCleanupTimeState("NonWindowJoinCleanupTime")
+    val valueStateDescriptor1 = new ValueStateDescriptor[Long]("timervaluestate1", classOf[Long])
+    leftTimer = getRuntimeContext.getState(valueStateDescriptor1)
+    val valueStateDescriptor2 = new ValueStateDescriptor[Long]("timervaluestate2", classOf[Long])
+    rightTimer = getRuntimeContext.getState(valueStateDescriptor2)
 
     cRowWrapper = new CRowWrappingMultiOutputCollector()
     LOG.debug("Instantiating NonWindowJoin.")
@@ -112,7 +122,7 @@ abstract class NonWindowJoin(
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow]): Unit = {
 
-    processElement(valueC, ctx, out, leftState, rightState, isLeft = true)
+    processElement(valueC, ctx, out, leftTimer, leftState, rightState, isLeft = true)
   }
 
   /**
@@ -128,7 +138,7 @@ abstract class NonWindowJoin(
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow]): Unit = {
 
-    processElement(valueC, ctx, out, rightState, leftState, isLeft = false)
+    processElement(valueC, ctx, out, rightTimer, rightState, leftState, isLeft = false)
   }
 
   /**
@@ -144,13 +154,28 @@ abstract class NonWindowJoin(
       ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
       out: Collector[CRow]): Unit = {
 
-    // expired timer has already been removed, delete state directly.
-    if (stateCleaningEnabled) {
-      cleanupState(leftState, rightState)
+    if (stateCleaningEnabled && leftTimer.value == timestamp) {
+      expireOutTimeRow(
+        timestamp,
+        leftState,
+        leftTimer,
+        isLeft = true,
+        ctx
+      )
+    }
+
+    if (stateCleaningEnabled && rightTimer.value == timestamp) {
+      expireOutTimeRow(
+        timestamp,
+        rightState,
+        rightTimer,
+        isLeft = false,
+        ctx
+      )
     }
   }
 
-  protected def getNewExpiredTime(curProcessTime: Long, oldExpiredTime: Long): Long = {
+  def getNewExpiredTime(curProcessTime: Long, oldExpiredTime: Long): Long = {
     if (stateCleaningEnabled && curProcessTime + minRetentionTime > oldExpiredTime) {
       curProcessTime + maxRetentionTime
     } else {
@@ -159,14 +184,52 @@ abstract class NonWindowJoin(
   }
 
   /**
+    * Removes records which are expired from the state. Register a new timer if the state still
+    * holds records after the clean-up.
+    */
+  def expireOutTimeRow(
+      curTime: Long,
+      rowMapState: MapState[Row, JTuple2[Long, Long]],
+      timerState: ValueState[Long],
+      isLeft: Boolean,
+      ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
+
+    val rowMapIter = rowMapState.iterator()
+    var validTimestamp: Boolean = false
+
+    while (rowMapIter.hasNext) {
+      val mapEntry = rowMapIter.next()
+      val recordExpiredTime = mapEntry.getValue.f1
+      if (recordExpiredTime <= curTime) {
+        rowMapIter.remove()
+      } else {
+        // we found a timestamp that is still valid
+        validTimestamp = true
+      }
+    }
+
+    // If the state has non-expired timestamps, register a new timer.
+    // Otherwise clean the complete state for this input.
+    if (validTimestamp) {
+      val cleanupTime = curTime + maxRetentionTime
+      ctx.timerService.registerProcessingTimeTimer(cleanupTime)
+      timerState.update(cleanupTime)
+    } else {
+      timerState.clear()
+      rowMapState.clear()
+    }
+  }
+
+  /**
     * Puts or Retract an element from the input stream into state and search the other state to
     * output records meet the condition. Records will be expired in state if state retention time
     * has been specified.
     */
-  protected def processElement(
+  def processElement(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow],
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]],
       otherSideState: MapState[Row, JTuple2[Long, Long]],
       isLeft: Boolean): Unit
@@ -177,12 +240,14 @@ abstract class NonWindowJoin(
     *
     * @param value            The input CRow
     * @param ctx              The ctx to register timer or get current time
+    * @param timerState       The state to record last timer
     * @param currentSideState The state to hold current side stream element
     * @return The row number and expired time for current input row
     */
-  protected def updateCurrentSide(
+  def updateCurrentSide(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]]): JTuple2[Long, Long] = {
 
     val inputRow = value.row
@@ -196,7 +261,10 @@ abstract class NonWindowJoin(
 
     cntAndExpiredTime.f1 = getNewExpiredTime(curProcessTime, cntAndExpiredTime.f1)
     // update timer if necessary
-    processCleanupTimer(ctx, curProcessTime)
+    if (stateCleaningEnabled && timerState.value() == 0) {
+      timerState.update(cntAndExpiredTime.f1)
+      ctx.timerService().registerProcessingTimeTimer(cntAndExpiredTime.f1)
+    }
 
     // update current side stream state
     if (!value.change) {
@@ -214,7 +282,7 @@ abstract class NonWindowJoin(
     cntAndExpiredTime
   }
 
-  protected def callJoinFunction(
+  def callJoinFunction(
       inputRow: Row,
       inputRowFromLeft: Boolean,
       otherSideRow: Row,
@@ -226,4 +294,8 @@ abstract class NonWindowJoin(
       joinFunction.join(otherSideRow, inputRow, cRowWrapper)
     }
   }
+
+  override def close(): Unit = {
+    FunctionUtils.closeFunction(joinFunction)
+  }
 }
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
index 5995fb8..b4f14e4 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoin.scala
@@ -69,12 +69,13 @@ class NonWindowLeftRightJoin(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow],
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]],
       otherSideState: MapState[Row, JTuple2[Long, Long]],
       recordFromLeft: Boolean): Unit = {
 
     val inputRow = value.row
-    updateCurrentSide(value, ctx, currentSideState)
+    updateCurrentSide(value, ctx, timerState, currentSideState)
 
     cRowWrapper.reset()
     cRowWrapper.setCollector(out)
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
index a3e25f9..33517cc 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowLeftRightJoinWithNonEquiPredicates.scala
@@ -71,13 +71,14 @@ class NonWindowLeftRightJoinWithNonEquiPredicates(
       value: CRow,
       ctx: CoProcessFunction[CRow, CRow, CRow]#Context,
       out: Collector[CRow],
+      timerState: ValueState[Long],
       currentSideState: MapState[Row, JTuple2[Long, Long]],
       otherSideState: MapState[Row, JTuple2[Long, Long]],
       recordFromLeft: Boolean): Unit = {
 
     val currentJoinCntState = getJoinCntState(joinCntState, recordFromLeft)
     val inputRow = value.row
-    val cntAndExpiredTime = updateCurrentSide(value, ctx, currentSideState)
+    val cntAndExpiredTime = updateCurrentSide(value, ctx, timerState, currentSideState)
     if (!value.change && cntAndExpiredTime.f0 <= 0 && recordFromLeft == isLeftJoin) {
       currentJoinCntState.remove(inputRow)
     }
@@ -100,21 +101,17 @@ class NonWindowLeftRightJoinWithNonEquiPredicates(
   }
 
   /**
-    * Called when a processing timer trigger.
-    * Expire left/right expired records and expired joinCnt state.
+    * Removes records which are expired from state. Register a new timer if the state still
+    * holds records after the clean-up. Also, clear joinCnt map state when clear rowMapState.
     */
-  override def onTimer(
-      timestamp: Long,
-      ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext,
-      out: Collector[CRow]): Unit = {
+  override def expireOutTimeRow(
+      curTime: Long,
+      rowMapState: MapState[Row, JTuple2[Long, Long]],
+      timerState: ValueState[Long],
+      isLeft: Boolean,
+      ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
 
-    // expired timer has already been removed, delete state directly.
-    if (stateCleaningEnabled) {
-      cleanupState(
-        leftState,
-        rightState,
-        getJoinCntState(joinCntState, isLeftJoin))
-    }
+    expireOutTimeRow(curTime, rowMapState, timerState, isLeft, joinCntState, ctx)
   }
 }
 
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
index 9fadbb0..0018a16 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoin.scala
@@ -73,7 +73,7 @@ abstract class NonWindowOuterJoin(
     * @param otherSideState   the other side state
     * @return the number of matched rows
     */
-  protected def preservedJoin(
+  def preservedJoin(
       inputRow: Row,
       inputRowFromLeft: Boolean,
       otherSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
@@ -106,7 +106,7 @@ abstract class NonWindowOuterJoin(
     * RowWrapper has been reset before we call retractJoin and we also assume that the current
     * change of cRowWrapper is equal to value.change.
     */
-  protected def retractJoin(
+  def retractJoin(
       value: CRow,
       inputRowFromLeft: Boolean,
       currentSideState: MapState[Row, JTuple2[Long, Long]],
@@ -152,8 +152,7 @@ abstract class NonWindowOuterJoin(
     * Return approximate number of records in corresponding state. Only check if record number is
     * 0, 1 or bigger.
     */
-  protected def approxiRecordNumInState(
-      currentSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
+  def approxiRecordNumInState(currentSideState: MapState[Row, JTuple2[Long, Long]]): Long = {
     var recordNum = 0L
     val it = currentSideState.iterator()
     while(it.hasNext && recordNum < 2) {
@@ -165,7 +164,7 @@ abstract class NonWindowOuterJoin(
   /**
     * Append input row with default null value if there is no match and Collect.
     */
-  protected def collectAppendNull(
+  def collectAppendNull(
       inputRow: Row,
       inputFromLeft: Boolean,
       out: Collector[Row]): Unit = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
index 0eb6a81..8fe2f4f 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/join/NonWindowOuterJoinWithNonEquiPredicates.scala
@@ -21,6 +21,7 @@ import org.apache.flink.api.common.state._
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.tuple.{Tuple2 => JTuple2}
 import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.co.CoProcessFunction
 import org.apache.flink.table.api.{StreamQueryConfig, Types}
 import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.types.Row
@@ -82,7 +83,7 @@ import org.apache.flink.types.Row
     * unmatched or vice versa. The RowWrapper has been reset before we call retractJoin and we
     * also assume that the current change of cRowWrapper is equal to value.change.
     */
-  protected def retractJoinWithNonEquiPreds(
+  def retractJoinWithNonEquiPreds(
       value: CRow,
       inputRowFromLeft: Boolean,
       otherSideState: MapState[Row, JTuple2[Long, Long]],
@@ -131,6 +132,48 @@ import org.apache.flink.types.Row
   }
 
   /**
+    * Removes records which are expired from state. Registers a new timer if the state still
+    * holds records after the clean-up. Also, clear joinCnt map state when clear rowMapState.
+    */
+  def expireOutTimeRow(
+      curTime: Long,
+      rowMapState: MapState[Row, JTuple2[Long, Long]],
+      timerState: ValueState[Long],
+      isLeft: Boolean,
+      joinCntState: Array[MapState[Row, Long]],
+      ctx: CoProcessFunction[CRow, CRow, CRow]#OnTimerContext): Unit = {
+
+    val currentJoinCntState = getJoinCntState(joinCntState, isLeft)
+    val rowMapIter = rowMapState.iterator()
+    var validTimestamp: Boolean = false
+
+    while (rowMapIter.hasNext) {
+      val mapEntry = rowMapIter.next()
+      val recordExpiredTime = mapEntry.getValue.f1
+      if (recordExpiredTime <= curTime) {
+        rowMapIter.remove()
+        currentJoinCntState.remove(mapEntry.getKey)
+      } else {
+        // we found a timestamp that is still valid
+        validTimestamp = true
+      }
+    }
+    // If the state has non-expired timestamps, register a new timer.
+    // Otherwise clean the complete state for this input.
+    if (validTimestamp) {
+      val cleanupTime = curTime + maxRetentionTime
+      ctx.timerService.registerProcessingTimeTimer(cleanupTime)
+      timerState.update(cleanupTime)
+    } else {
+      timerState.clear()
+      rowMapState.clear()
+      if (isLeft == isLeftJoin) {
+        currentJoinCntState.clear()
+      }
+    }
+  }
+
+  /**
     * Get left or right join cnt state.
     *
     * @param joinCntState    the join cnt state array, index 0 is left join cnt state, index 1
@@ -138,7 +181,7 @@ import org.apache.flink.types.Row
     * @param isLeftCntState the flag whether get the left join cnt state
     * @return the corresponding join cnt state
     */
-  protected def getJoinCntState(
+  def getJoinCntState(
       joinCntState: Array[MapState[Row, Long]],
       isLeftCntState: Boolean)
     : MapState[Row, Long] = {
diff --git a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
index 6ae5e63..3c18449 100644
--- a/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
+++ b/flink-libraries/flink-table/src/main/scala/org/apache/flink/table/runtime/triggers/StateCleaningCountTrigger.scala
@@ -71,10 +71,6 @@ class StateCleaningCountTrigger(queryConfig: StreamQueryConfig, maxCount: Long)
         val cleanupTime = currentTime + maxRetentionTime
         // register timer and remember clean-up time
         ctx.registerProcessingTimeTimer(cleanupTime)
-        // delete expired timer
-        if (curCleanupTime != null) {
-          ctx.deleteProcessingTimeTimer(curCleanupTime)
-        }
 
         ctx.getPartitionedState(cleanupStateDesc).update(cleanupTime)
       }
@@ -97,9 +93,12 @@ class StateCleaningCountTrigger(queryConfig: StreamQueryConfig, maxCount: Long)
       ctx: TriggerContext): TriggerResult = {
 
     if (stateCleaningEnabled) {
-      // do clear directly, since we have already deleted expired timer
-      clear(window, ctx)
-      return TriggerResult.FIRE_AND_PURGE
+      val cleanupTime = ctx.getPartitionedState(cleanupStateDesc).value()
+      // check that the triggered timer is the last registered processing time timer.
+      if (null != cleanupTime && time == cleanupTime) {
+        clear(window, ctx)
+        return TriggerResult.FIRE_AND_PURGE
+      }
     }
     TriggerResult.CONTINUE
   }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
index 4619c75..bd19be8 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/JoinHarnessTest.scala
@@ -21,18 +21,21 @@ import java.lang.{Integer => JInt, Long => JLong}
 import java.util.concurrent.ConcurrentLinkedQueue
 
 import org.apache.flink.api.common.time.Time
-import org.apache.flink.api.common.typeinfo.BasicTypeInfo
+import org.apache.flink.api.common.typeinfo.BasicTypeInfo._
+import org.apache.flink.api.common.typeinfo.{BasicTypeInfo, TypeInformation}
 import org.apache.flink.api.java.operators.join.JoinType
+import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.streaming.api.operators.co.KeyedCoProcessOperator
 import org.apache.flink.streaming.api.watermark.Watermark
 import org.apache.flink.streaming.runtime.streamrecord.StreamRecord
 import org.apache.flink.streaming.util.KeyedTwoInputStreamOperatorTestHarness
-import org.apache.flink.table.api.Types
-import org.apache.flink.table.runtime.harness.HarnessTestBase.{TestStreamQueryConfig, TupleRowKeySelector}
+import org.apache.flink.table.api.{StreamQueryConfig, Types}
+import org.apache.flink.table.runtime.harness.HarnessTestBase.{RowResultSortComparator, RowResultSortComparatorWithWatermarks, TestStreamQueryConfig, TupleRowKeySelector}
 import org.apache.flink.table.runtime.join._
 import org.apache.flink.table.runtime.operators.KeyedCoProcessOperatorWithWatermarkDelay
-import org.apache.flink.table.runtime.types.CRow
-import org.junit.Assert.assertEquals
+import org.apache.flink.table.runtime.types.{CRow, CRowTypeInfo}
+import org.apache.flink.types.Row
+import org.junit.Assert.{assertEquals, assertTrue}
 import org.junit.Test
 
 /**
@@ -827,6 +830,14 @@ class JoinHarnessTest extends HarnessTestBase {
   @Test
   def testNonWindowInnerJoin() {
 
+    val joinReturnType = CRowTypeInfo(new RowTypeInfo(
+      Array[TypeInformation[_]](
+        INT_TYPE_INFO,
+        STRING_TYPE_INFO,
+        INT_TYPE_INFO,
+        STRING_TYPE_INFO),
+      Array("a", "b", "c", "d")))
+
     val joinProcessFunc = new NonWindowInnerJoin(
       rowType,
       rowType,
@@ -868,32 +879,35 @@ class JoinHarnessTest extends HarnessTestBase {
     // right stream input and output normally
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "Hi1")))
-    // lkeys(1, 2) rkeys(1) timer_key_time(1:5, 2:6)
-    assertEquals(5, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(6, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
     testHarness.setProcessingTime(4)
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "Hello1")))
-    // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    assertEquals(6, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(8, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
 
-    // expired stream record with key value of 1
+    // expired left stream record with key value of 1
     testHarness.setProcessingTime(5)
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "Hi2")))
-    // lkeys(2) rkeys(1, 2) timer_key_time(1:9, 2:6)
-    assertEquals(5, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(6, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
 
-    // expired all left stream records
+    // expired all left stream record
     testHarness.setProcessingTime(6)
-    // lkeys() rkeys(1) timer_key_time(1:9)
+    assertEquals(4, testHarness.numKeyedStateEntries())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
+
+    // expired right stream record with key value of 2
+    testHarness.setProcessingTime(8)
     assertEquals(2, testHarness.numKeyedStateEntries())
     assertEquals(1, testHarness.numProcessingTimeTimers())
 
-    // expired all stream records
-    testHarness.setProcessingTime(9)
+    testHarness.setProcessingTime(10)
+    assertTrue(testHarness.numKeyedStateEntries() > 0)
+    // expired all right stream record
+    testHarness.setProcessingTime(11)
     assertEquals(0, testHarness.numKeyedStateEntries())
     assertEquals(0, testHarness.numProcessingTimeTimers())
 
@@ -961,37 +975,32 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "Hi1")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "Hi1")))
-    // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
-    assertEquals(4, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
     testHarness.setProcessingTime(4)
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "Hello1")))
-    // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
-    assertEquals(5, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(7, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
 
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "aaa")))
-    // expired stream records with key value of 1
+    // expired left stream record with key value of 1
     testHarness.setProcessingTime(5)
-    // lkeys(2) rkeys(2) timer_key_time(2:6)
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "Hi2")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "Hi2")))
-    // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
-    assertEquals(4, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
 
-    // expired all stream records
+    // expired all left stream record
     testHarness.setProcessingTime(6)
-    // lkeys() rkeys() timer_key_time(1:9)
-    assertEquals(1, testHarness.numKeyedStateEntries())
-    assertEquals(1, testHarness.numProcessingTimeTimers())
+    assertEquals(3, testHarness.numKeyedStateEntries())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
 
-    // expired all data
-    testHarness.setProcessingTime(9)
+    // expired right stream record with key value of 2
+    testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numKeyedStateEntries())
     assertEquals(0, testHarness.numProcessingTimeTimers())
 
@@ -1058,36 +1067,32 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "Hi1")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "Hi1")))
-    // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
-    assertEquals(4, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
     testHarness.setProcessingTime(4)
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "Hello1")))
-    // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
-    assertEquals(5, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(7, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
 
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "aaa")))
-    // expired stream records with key value of 1
+    // expired left stream record with key value of 1
     testHarness.setProcessingTime(5)
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "Hi2")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "Hi2")))
-    // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
-    assertEquals(4, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
 
-    // expired stream records with key value of 2
+    // expired all left stream record
     testHarness.setProcessingTime(6)
-    // lkeys() rkeys() timer_key_time(1:9)
-    assertEquals(1, testHarness.numKeyedStateEntries())
-    assertEquals(1, testHarness.numProcessingTimeTimers())
+    assertEquals(3, testHarness.numKeyedStateEntries())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
 
-    // expired all data
-    testHarness.setProcessingTime(9)
+    // expired right stream record with key value of 2
+    testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numKeyedStateEntries())
     assertEquals(0, testHarness.numProcessingTimeTimers())
 
@@ -1155,7 +1160,7 @@ class JoinHarnessTest extends HarnessTestBase {
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "bbb")))
     assertEquals(1, testHarness.numProcessingTimeTimers())
-    // lkeys(1) rkeys() timer_key_time(1:5)
+    // 1 left timer(5), 1 left key(1), 1 join cnt
     assertEquals(3, testHarness.numKeyedStateEntries())
     testHarness.setProcessingTime(2)
     testHarness.processElement1(new StreamRecord(
@@ -1163,8 +1168,7 @@ class JoinHarnessTest extends HarnessTestBase {
     testHarness.processElement1(new StreamRecord(
       CRow(2: JInt, "bbb")))
     assertEquals(2, testHarness.numProcessingTimeTimers())
-    // lkeys(1, 2) rkeys() timer_key_time(1:5, 2:6)
-    // l_join_cnt_keys(1, 2)
+    // 2 left timer(5,6), 2 left key(1,2), 2 join cnt
     assertEquals(6, testHarness.numKeyedStateEntries())
     testHarness.setProcessingTime(3)
 
@@ -1173,19 +1177,17 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "Hi1")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "bbb")))
-    // lkeys(1, 2) rkeys(1) timer_key_time(1:5, 2:6)
-    // l_join_cnt_keys(1, 2)
-    assertEquals(7, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,6), 2 left keys(1,2), 2 join cnt, 1 right timer(7), 1 right key(1)
+    assertEquals(8, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
     testHarness.setProcessingTime(4)
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "ccc")))
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "Hello")))
-    // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    // l_join_cnt_keys(1, 2)
-    assertEquals(8, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,6), 2 left keys(1,2), 2 join cnt, 2 right timer(7,8), 2 right key(1,2)
+    assertEquals(10, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
 
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "aaa")))
@@ -1195,29 +1197,22 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(false, 1: JInt, "Hi2")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "Hi1")))
-    // lkeys(1, 2) rkeys(2) timer_key_time(1:8, 2:6)
-    // l_join_cnt_keys(1, 2)
-    assertEquals(7, testHarness.numKeyedStateEntries())
+    // expired left stream record with key value of 1
     testHarness.setProcessingTime(5)
-    // [1]. this will clean up left stream records with expired time of 5
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "Hi3")))
-    // [2]. there are no elements can be connected, since be cleaned by [1]
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "Hi3")))
-    // lkeys(1, 2) rkeys(2) timer_key_time(1:8, 2:6)
-    // l_join_cnt_keys(1, 2)
-    assertEquals(7, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 1 left timer(6), 1 left keys(2), 1 join cnt, 2 right timer(7,8), 1 right key(2)
+    assertEquals(6, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
 
-    // expired all records with key value of 2
+    // expired all left stream record
     testHarness.setProcessingTime(6)
-    // lkeys(1) rkeys() timer_key_time(1:8)
-    // l_join_cnt_keys(1)
     assertEquals(3, testHarness.numKeyedStateEntries())
-    assertEquals(1, testHarness.numProcessingTimeTimers())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
 
-    // expired all data
+    // expired right stream record with key value of 2
     testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numKeyedStateEntries())
     assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1258,12 +1253,6 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(false, 1: JInt, "bbb", 1: JInt, "Hi1")))
     expectedOutput.add(new StreamRecord(
       CRow(1: JInt, "bbb", null: JInt, null)))
-    // processing time of 5
-    // timer of 8, we use only one timer state now
-    expectedOutput.add(new StreamRecord(
-      CRow(false, 1: JInt, "bbb", null: JInt, null)))
-    expectedOutput.add(new StreamRecord(
-      CRow(1: JInt, "bbb", 1: JInt, "Hi3")))
     verify(expectedOutput, result)
 
     testHarness.close()
@@ -1316,36 +1305,32 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "Hi1")))
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "Hi1")))
-    // lkeys() rkeys(1, 2) timer_key_time(1:5, 2:6)
-    assertEquals(4, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
     testHarness.setProcessingTime(4)
     testHarness.processElement1(new StreamRecord(
       CRow(2: JInt, "Hello1")))
-    // lkeys(2) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    assertEquals(5, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(7, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
 
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "aaa")))
-    // expired stream records with key value of 1
+    // expired right stream record with key value of 1
     testHarness.setProcessingTime(5)
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "Hi2")))
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "Hi2")))
-    // lkeys(2) rkeys(2) timer_key_time(1:9, 2:6)
-    assertEquals(4, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
 
-    // expired stream records with key value of 2
+    // expired all right stream record
     testHarness.setProcessingTime(6)
-    // lkeys() rkeys() timer_key_time(1:9)
-    assertEquals(1, testHarness.numKeyedStateEntries())
-    assertEquals(1, testHarness.numProcessingTimeTimers())
+    assertEquals(3, testHarness.numKeyedStateEntries())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
 
-    // expired all data
-    testHarness.setProcessingTime(9)
+    // expired left stream record with key value of 2
+    testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numKeyedStateEntries())
     assertEquals(0, testHarness.numProcessingTimeTimers())
 
@@ -1413,17 +1398,15 @@ class JoinHarnessTest extends HarnessTestBase {
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "bbb")))
     assertEquals(1, testHarness.numProcessingTimeTimers())
-    // lkeys() rkeys(1) timer_key_time(1:5)
-    // r_join_cnt_keys(1)
+    // 1 right timer(5), 1 right key(1), 1 join cnt
     assertEquals(3, testHarness.numKeyedStateEntries())
     testHarness.setProcessingTime(2)
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "aaa")))
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "bbb")))
-    // lkeys() rkeys(1, 2) timer_key_time(1:5, 2:6)
-    // r_join_cnt_keys(1, 2)
     assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 2 right timer(5,6), 2 right key(1,2), 2 join cnt
     assertEquals(6, testHarness.numKeyedStateEntries())
     testHarness.setProcessingTime(3)
 
@@ -1432,19 +1415,17 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "Hi1")))
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "bbb")))
-    // lkeys(1) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    // r_join_cnt_keys(1, 2)
-    assertEquals(7, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 2 right timer(5,6), 2 right keys(1,2), 2 join cnt, 1 left timer(7), 1 left key(1)
+    assertEquals(8, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
     testHarness.setProcessingTime(4)
     testHarness.processElement1(new StreamRecord(
       CRow(2: JInt, "ccc")))
     testHarness.processElement1(new StreamRecord(
       CRow(2: JInt, "Hello")))
-    // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    // r_join_cnt_keys(1, 2)
-    assertEquals(8, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 2 right timer(5,6), 2 right keys(1,2), 2 join cnt, 2 left timer(7,8), 2 left key(1,2)
+    assertEquals(10, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
 
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "aaa")))
@@ -1454,27 +1435,22 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(false, 1: JInt, "Hi2")))
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "Hi1")))
-    // lkeys(2) rkeys(1, 2) timer_key_time(1:8, 2:6)
-    // r_join_cnt_keys(1, 2)
-    assertEquals(7, testHarness.numKeyedStateEntries())
+    // expired right stream record with key value of 1
     testHarness.setProcessingTime(5)
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "Hi3")))
     testHarness.processElement1(new StreamRecord(
       CRow(false, 1: JInt, "Hi3")))
-    // lkeys(2) rkeys(1, 2) timer_key_time(1:8, 2:6)
-    // r_join_cnt_keys(1, 2)
-    assertEquals(7, testHarness.numKeyedStateEntries())
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 1 right timer(6), 1 right keys(2), 1 join cnt, 2 left timer(7,8), 1 left key(2)
+    assertEquals(6, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
 
-    // expired all stream records with key value of 2
-    // lkeys() rkeys(1) timer_key_time(1:8)
-    // r_join_cnt_keys(1)
+    // expired all right stream record
     testHarness.setProcessingTime(6)
     assertEquals(3, testHarness.numKeyedStateEntries())
-    assertEquals(1, testHarness.numProcessingTimeTimers())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
 
-    // expired all data
+    // expired left stream record with key value of 2
     testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numKeyedStateEntries())
     assertEquals(0, testHarness.numProcessingTimeTimers())
@@ -1515,12 +1491,6 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(false, 1: JInt, "Hi1", 1: JInt, "bbb")))
     expectedOutput.add(new StreamRecord(
       CRow(null: JInt, null, 1: JInt, "bbb")))
-    // processing time of 5
-    // timer of 8, we use only one timer state now
-    expectedOutput.add(new StreamRecord(
-      CRow(false, null: JInt, null, 1: JInt, "bbb")))
-    expectedOutput.add(new StreamRecord(
-      CRow(1: JInt, "Hi3", 1: JInt, "bbb")))
     verify(expectedOutput, result)
 
     testHarness.close()
@@ -1554,8 +1524,8 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "bbb")))
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "ccc")))
-    // lkeys(1) rkeys() timer_key_time(1:5)
     assertEquals(1, testHarness.numProcessingTimeTimers())
+    // 1 left timer(5), 1 left key(1)
     assertEquals(2, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(2)
@@ -1564,7 +1534,8 @@ class JoinHarnessTest extends HarnessTestBase {
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "ccc")))
     assertEquals(2, testHarness.numProcessingTimeTimers())
-    // lkeys(1) rkeys(2) timer_key_time(1:5, 2:6)
+    // 1 left timer(5), 1 left key(1)
+    // 1 right timer(6), 1 right key(1)
     assertEquals(4, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(3)
@@ -1572,16 +1543,18 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(2: JInt, "aaa")))
     testHarness.processElement1(new StreamRecord(
       CRow(2: JInt, "ddd")))
-    // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,7), 2 left key(1,2)
+    // 1 right timer(6), 1 right key(1)
+    assertEquals(6, testHarness.numKeyedStateEntries())
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "aaa")))
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "ddd")))
-    // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(6, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,7), 2 left key(1,2)
+    // 2 right timer(6,7), 2 right key(1,2)
+    assertEquals(8, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(4)
     testHarness.processElement1(new StreamRecord(
@@ -1592,26 +1565,28 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(false, 1: JInt, "aaa")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "ddd")))
-    // lkeys(1) rkeys(2) timer_key_time(1:8, 2:6)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(4, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,7), 1 left key(1)
+    // 2 right timer(6,7), 1 right key(2)
+    assertEquals(6, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(5)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
+    // 1 left timer(7)
+    // 2 right timer(6,7), 1 right key(2)
     assertEquals(4, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(6)
-    // lkeys(1) rkeys() timer_key_time(1:8)
-    assertEquals(1, testHarness.numProcessingTimeTimers())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 1 left timer(7)
+    // 2 right timer(7)
     assertEquals(2, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(7)
-    assertEquals(1, testHarness.numProcessingTimeTimers())
-    assertEquals(2, testHarness.numKeyedStateEntries())
-
-    testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numProcessingTimeTimers())
     assertEquals(0, testHarness.numKeyedStateEntries())
+
+    testHarness.setProcessingTime(8)
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "bbb")))
     testHarness.processElement2(new StreamRecord(
@@ -1718,9 +1693,8 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(1: JInt, "bbb")))
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "ccc")))
-    // lkeys(1) rkeys() timer_key_time(1:5)
-    // l_join_cnt_keys(1) r_join_cnt_keys()
     assertEquals(1, testHarness.numProcessingTimeTimers())
+    // 1 left timer(5), 1 left key(1), 1 left joincnt key(1)
     assertEquals(3, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(2)
@@ -1728,9 +1702,9 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(2: JInt, "bbb")))
     testHarness.processElement2(new StreamRecord(
       CRow(2: JInt, "ccc")))
-    // lkeys(1) rkeys(2) timer_key_time(1:5, 2:6)
-    // l_join_cnt_keys(1) r_join_cnt_keys(2)
     assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 1 left timer(5), 1 left key(1), 1 left joincnt key(1)
+    // 1 right timer(6), 1 right key(1), 1 right joincnt key(1)
     assertEquals(6, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(3)
@@ -1738,46 +1712,46 @@ class JoinHarnessTest extends HarnessTestBase {
       CRow(2: JInt, "aaa")))
     testHarness.processElement1(new StreamRecord(
       CRow(2: JInt, "ddd")))
-    // lkeys(1, 2) rkeys(2) timer_key_time(1:5, 2:6)
-    // l_join_cnt_keys(1, 2) r_join_cnt_keys(2)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(8, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
+    // 1 right timer(6), 1 right key(1), 1 right joincnt key(1)
+    assertEquals(9, testHarness.numKeyedStateEntries())
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "aaa")))
     testHarness.processElement2(new StreamRecord(
       CRow(1: JInt, "ddd")))
-    // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:5, 2:6)
-    // l_join_cnt_keys(1, 2) r_join_cnt_keys(1, 2)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(10, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
+    // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
+    assertEquals(12, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(4)
     testHarness.processElement1(new StreamRecord(
       CRow(false, 2: JInt, "aaa")))
     testHarness.processElement2(new StreamRecord(
       CRow(false, 1: JInt, "ddd")))
-    // lkeys(1, 2) rkeys(1, 2) timer_key_time(1:8, 2:6)
-    // l_join_cnt_keys(1, 2) r_join_cnt_keys(1, 2)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(10, testHarness.numKeyedStateEntries())
+    assertEquals(4, testHarness.numProcessingTimeTimers())
+    // 2 left timer(5,7), 2 left key(1,2), 2 left joincnt key(1,2)
+    // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
+    assertEquals(12, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(5)
-    assertEquals(2, testHarness.numProcessingTimeTimers())
-    assertEquals(10, testHarness.numKeyedStateEntries())
+    assertEquals(3, testHarness.numProcessingTimeTimers())
+    // 1 left timer(7), 1 left key(2), 1 left joincnt key(2)
+    // 2 right timer(6,7), 2 right key(1,2), 2 right joincnt key(1,2)
+    assertEquals(9, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(6)
-    // lkeys(1) rkeys(1) timer_key_time(1:8)
-    // l_join_cnt_keys(1) r_join_cnt_keys(1)
-    assertEquals(1, testHarness.numProcessingTimeTimers())
-    assertEquals(5, testHarness.numKeyedStateEntries())
+    assertEquals(2, testHarness.numProcessingTimeTimers())
+    // 1 left timer(7), 1 left key(2), 1 left joincnt key(2)
+    // 1 right timer(7), 1 right key(2), 1 right joincnt key(2)
+    assertEquals(6, testHarness.numKeyedStateEntries())
 
     testHarness.setProcessingTime(7)
-    assertEquals(1, testHarness.numProcessingTimeTimers())
-    assertEquals(5, testHarness.numKeyedStateEntries())
-
-    testHarness.setProcessingTime(8)
     assertEquals(0, testHarness.numProcessingTimeTimers())
     assertEquals(0, testHarness.numKeyedStateEntries())
+
+    testHarness.setProcessingTime(8)
     testHarness.processElement1(new StreamRecord(
       CRow(1: JInt, "bbb")))
     testHarness.processElement2(new StreamRecord(
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
index 25395be..7f9c0ef 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/harness/StateCleaningCountTriggerHarnessTest.scala
@@ -80,8 +80,8 @@ class StateCleaningCountTriggerHarnessTest {
       TriggerResult.CONTINUE,
       testHarness.processElement(new StreamRecord(1), GlobalWindow.get))
 
-    // have one timer 7002
-    assertEquals(1, testHarness.numProcessingTimeTimers)
+    // have two timers 6001 and 7002
+    assertEquals(2, testHarness.numProcessingTimeTimers)
     assertEquals(0, testHarness.numEventTimeTimers)
     assertEquals(2, testHarness.numStateEntries)
     assertEquals(2, testHarness.numStateEntries(GlobalWindow.get))
@@ -116,6 +116,9 @@ class StateCleaningCountTriggerHarnessTest {
 
     // try to trigger onProcessingTime method via 7002, and all states are cleared
     val timesIt = testHarness.advanceProcessingTime(7002).iterator()
+    assertEquals(
+      TriggerResult.CONTINUE,
+      timesIt.next().f1)
 
     assertEquals(
       TriggerResult.FIRE_AND_PURGE,
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
index 1c02889..fe90a5f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/KeyedProcessFunctionWithCleanupStateTest.scala
@@ -110,7 +110,7 @@ private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig)
       out: Collector[String]): Unit = {
 
     val curTime = ctx.timerService().currentProcessingTime()
-    processCleanupTimer(ctx, curTime)
+    registerProcessingCleanupTimer(ctx, curTime)
     state.update(value._2)
   }
 
@@ -119,12 +119,8 @@ private class MockedKeyedProcessFunction(queryConfig: StreamQueryConfig)
       ctx: KeyedProcessFunction[String, (String, String), String]#OnTimerContext,
       out: Collector[String]): Unit = {
 
-    if (stateCleaningEnabled) {
-      val cleanupTime = cleanupTimeState.value()
-      if (null != cleanupTime && timestamp == cleanupTime) {
-        // clean up
-        cleanupState(state)
-      }
+    if (needToCleanupState(timestamp)) {
+      cleanupState(state)
     }
   }
 }
diff --git a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
index 6c0ca1a..519b03f 100644
--- a/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
+++ b/flink-libraries/flink-table/src/test/scala/org/apache/flink/table/runtime/operators/ProcessFunctionWithCleanupStateTest.scala
@@ -110,7 +110,7 @@ private class MockedProcessFunction(queryConfig: StreamQueryConfig)
       out: Collector[String]): Unit = {
 
     val curTime = ctx.timerService().currentProcessingTime()
-    processCleanupTimer(ctx, curTime)
+    registerProcessingCleanupTimer(ctx, curTime)
     state.update(value._2)
   }
 
@@ -119,7 +119,7 @@ private class MockedProcessFunction(queryConfig: StreamQueryConfig)
       ctx: ProcessFunction[(String, String), String]#OnTimerContext,
       out: Collector[String]): Unit = {
 
-    if (stateCleaningEnabled) {
+    if (needToCleanupState(timestamp)) {
       cleanupState(state)
     }
   }