[jira] [Commented] (FLINK-15719) Exceptions when using scala types directly with the State Process API

["Ying Z (Jira)" <ji...@apache.org>]

    [ https://issues.apache.org/jira/browse/FLINK-15719?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17173617#comment-17173617 ] 

Ying Z commented on FLINK-15719:
--------------------------------

Hi, [~tzulitai] , I want to help to modify the doc [1] to make it less error-prone, is it ok? Here is my test code of keyed state in scala lang:
 # stateful process function to generate state
 # inputs: 1 2 3 4 5 6

 
{code:java}
// code placeholder
import org.apache.flink.api.common.state.{ListState, ListStateDescriptor, ValueState, ValueStateDescriptor}
import org.apache.flink.api.common.typeinfo.Types
import org.apache.flink.configuration.Configuration
import org.apache.flink.runtime.state.filesystem.FsStateBackend
import org.apache.flink.streaming.api.environment.CheckpointConfig.ExternalizedCheckpointCleanup
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.util.Collector

class StatefulFunctionWithTime extends KeyedProcessFunction[Int, Int, Void] {
  var state: ValueState[Int] = _
  var updateTimes: ListState[Long] = _

  @throws[Exception]
  override def open(parameters: Configuration): Unit = {
    val stateDescriptor = new ValueStateDescriptor("state", createTypeInformation[Int])
//    val stateDescriptor = new ValueStateDescriptor("state", Types.INT)
    state = getRuntimeContext().getState(stateDescriptor)

    val updateDescriptor = new ListStateDescriptor("times", createTypeInformation[Long])
//    val updateDescriptor = new ListStateDescriptor("times", Types.LONG)
    updateTimes = getRuntimeContext().getListState(updateDescriptor)
  }

  @throws[Exception]
  override def processElement(value: Int, ctx: KeyedProcessFunction[ Int, Int, Void ]#Context, out: Collector[Void]): Unit = {
    state.update(value + 1)
    updateTimes.add(System.currentTimeMillis)
  }
}

object KeyedStateSample extends App {
  val env = StreamExecutionEnvironment.getExecutionEnvironment
  val fsStateBackend = new FsStateBackend("file:///tmp/chk_dir")
  env.setStateBackend(fsStateBackend)
  env.enableCheckpointing(60000)
  env.getCheckpointConfig.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)

  env.socketTextStream("127.0.0.1", 8010)
    .map(_.toInt)
    .keyBy(i => i)
    .process(new StatefulFunctionWithTime)
    .uid("my-uid")

  env.execute()
}
{code}
 
 # read the state generated by code above, which outputs:
 # KeyedState(3,4,List(1596878053283))
KeyedState(5,6,List(1596878055023))
KeyedState(2,3,List(1596878052359))
KeyedState(4,5,List(1596878054098))
KeyedState(6,7,List(1596878056151))
KeyedState(1,2,List(1596878051332))

 
{code:java}
// code placeholder
import org.apache.flink.api.common.state.{ListState, ListStateDescriptor, ValueState, ValueStateDescriptor}
import org.apache.flink.api.java.ExecutionEnvironment
import org.apache.flink.api.scala._
import org.apache.flink.configuration.Configuration
import org.apache.flink.runtime.state.memory.MemoryStateBackend
import org.apache.flink.state.api.Savepoint
import org.apache.flink.state.api.functions.KeyedStateReaderFunction
import org.apache.flink.util.Collector

import scala.collection.JavaConverters._

/**
 * Description:
 */
object TestReadState extends App {
  val bEnv      = ExecutionEnvironment.getExecutionEnvironment
  val savepoint = Savepoint.load(bEnv, "file:///tmp/chk_dir/f988137ef1df4597bebc596ef7c76626/chk-2", new MemoryStateBackend)
  val keyedState = savepoint.readKeyedState("my-uid", new ReaderFunction)
  keyedState.print()

  case class KeyedState(key: Int, value: Int, times: List[Long])
  class ReaderFunction extends KeyedStateReaderFunction[java.lang.Integer, KeyedState] {
    var state: ValueState[Int] = _
    var updateTimes: ListState[Long] = _

    @throws[Exception]
    override def open(parameters: Configuration): Unit = {
      val stateDescriptor = new ValueStateDescriptor("state", createTypeInformation[Int])
      state = getRuntimeContext().getState(stateDescriptor)

      val updateDescriptor = new ListStateDescriptor("times", createTypeInformation[Long])
      updateTimes = getRuntimeContext().getListState(updateDescriptor)
    }

    override def readKey(key: java.lang.Integer,
                         ctx: KeyedStateReaderFunction.Context,
                         out: Collector[KeyedState]): Unit = {
      val data = KeyedState(
        key,
        state.value(),
        updateTimes.get().asScala.toList)
      out.collect(data)
    }
  }
}
{code}
 

 

1. https://ci.apache.org/projects/flink/flink-docs-master/dev/libs/state_processor_api.html#keyed-state

> Exceptions when using scala types directly with the State Process API
> ---------------------------------------------------------------------
>
>                 Key: FLINK-15719
>                 URL: https://issues.apache.org/jira/browse/FLINK-15719
>             Project: Flink
>          Issue Type: Bug
>          Components: API / State Processor
>    Affects Versions: 1.9.1
>            Reporter: Ying Z
>            Assignee: Tzu-Li (Gordon) Tai
>            Priority: Major
>
> I followed these steps to generate and read states:
>  # implements the example[1] `CountWindowAverage` in Scala(exactly same), and run jobA => that makes good.
>  # execute `flink cancel -s ${JobID}` => savepoints was generated as expected.
>  # implements the example[2] `StatefulFunctionWithTime` in Scala(code below), and run jobB => failed, exceptions shows that "Caused by: org.apache.flink.util.StateMigrationException: The new key serializer must be compatible."
> ReaderFunction code as below:
> {code:java}
> // code placeholder
>   class ReaderFunction extends KeyedStateReaderFunction[Long, (Long, Long)] {
>     var countState: ValueState[(Long, Long)] = _
>     override def open(parameters: Configuration): Unit = {
>       val stateDescriptor = new ValueStateDescriptor("average", createTypeInformation[(Long, Long)])
>       countState = getRuntimeContext().getState(stateDescriptor)
>     }    override def readKey(key: Long, ctx: KeyedStateReaderFunction.Context, out: Collector[(Long, Long)]): Unit = {
>       out.collect(countState.value())
>     }
>   }
> {code}
> 1: [https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/stream/state/state.html#using-managed-keyed-state] 
> 2: [https://ci.apache.org/projects/flink/flink-docs-release-1.9/dev/libs/state_processor_api.html#keyed-state] 



--
This message was sent by Atlassian Jira
(v8.3.4#803005)