You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by "fhueske (via GitHub)" <gi...@apache.org> on 2025/01/08 14:55:33 UTC
Re: [PR] [FLINK-19059]: Support non-time retract mode for OverAggregate operator [flink]
fhueske commented on code in PR #25753:
URL: https://github.com/apache/flink/pull/25753#discussion_r1907085683
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
Review Comment:
This code can also be put into a helper method `insertNewRow(input)` which is reused above.
The helper method should IMO also include the next lines:
```
sortedKeyState.update(sortedKeyList);
idState.update(id);
```
so that all state updates related to adding the new row are in a single helper method
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
Review Comment:
```suggestion
long curKey = iterator.next().getLong(keyIdx);
```
?
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
Review Comment:
move these lines into a helper method that handles inserting a new row into the various state objects.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
Review Comment:
```suggestion
// If previous accumulator is same as the new accumulator
```
##########
flink-table/flink-table-runtime/src/test/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunctionTest.java:
##########
@@ -0,0 +1,467 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.streaming.api.operators.KeyedProcessOperator;
+import org.apache.flink.streaming.api.watermark.Watermark;
+import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.StringData;
+import org.apache.flink.table.data.binary.BinaryRowData;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordComparator;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.types.RowKind;
+
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.flink.table.runtime.util.StreamRecordUtils.insertRecord;
+import static org.apache.flink.table.runtime.util.StreamRecordUtils.updateAfterRecord;
+import static org.apache.flink.table.runtime.util.StreamRecordUtils.updateBeforeRecord;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link NonTimeUnboundedPrecedingFunction}. */
+public class NonTimeUnboundedPrecedingFunctionTest extends RowTimeOverWindowTestBase {
+
+ private static final int SORT_KEY_IDX = 1;
+
+ private static final FieldGetter SORT_KEY_FIELD_GETTER =
+ RowData.createFieldGetter(DataTypes.BIGINT().getLogicalType(), SORT_KEY_IDX);
+
+ private static final GeneratedRecordComparator GENERATED_SORT_KEY_COMPARATOR =
+ new GeneratedRecordComparator("", "", new Object[0]) {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public RecordComparator newInstance(ClassLoader classLoader) {
+ return new LongRecordComparator(SORT_KEY_IDX);
+ }
+ };
+
+ private static final GeneratedRecordEqualiser GENERATED_RECORD_EQUALISER =
+ new GeneratedRecordEqualiser("", "", new Object[0]) {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public RecordEqualiser newInstance(ClassLoader classLoader) {
+ // return new RowDataRecordEqualiser();
+ return new TestRecordEqualiser();
+ }
+ };
+
+ /** Custom test comparator for comparing numbers. */
+ public static class LongRecordComparator implements RecordComparator {
+
+ private int pos;
+
+ public LongRecordComparator(int pos) {
+ this.pos = pos;
+ }
+
+ @Override
+ public int compare(RowData o1, RowData o2) {
+ boolean null0At1 = o1.isNullAt(pos);
+ boolean null0At2 = o2.isNullAt(pos);
+ int cmp0 =
+ null0At1 && null0At2
+ ? 0
+ : (null0At1
+ ? -1
+ : (null0At2
+ ? 1
+ : Long.compare(o1.getLong(pos), o2.getLong(pos))));
+ if (cmp0 != 0) {
+ return cmp0;
+ }
+ return 0;
+ }
+ }
+
+ /** Custom test row equaliser for comparing rows. */
+ public static class TestRecordEqualiser implements RecordEqualiser {
+
+ private static final long serialVersionUID = -6706336100425614942L;
+
+ @Override
+ public boolean equals(RowData row1, RowData row2) {
+ if (row1 instanceof BinaryRowData && row2 instanceof BinaryRowData) {
+ return row1.equals(row2);
+ } else if (row1 instanceof GenericRowData && row2 instanceof GenericRowData) {
+ return row1.getString(0).equals(row2.getString(0))
+ && row1.getLong(1) == row2.getLong(1)
+ && row1.getLong(2) == row2.getLong(2);
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
+
+ @Test
+ public void testInsertOnlyRecordsWithCustomSortKey() throws Exception {
+ NonTimeUnboundedPrecedingFunction<RowData> function =
+ new NonTimeUnboundedPrecedingFunction<RowData>(
+ 0,
+ 2000,
+ aggsHandleFunction,
Review Comment:
`aggsHandleFunction` uses a `SUM` function. We should either change this to `AVG` or add a new `avgAggsHandleFunction` for our tests.
In contrast to the `SUM` function, the `AVG` function uses different types for the accumulator and result value.
Using `AVG` should have caught all issues in which accumulators and values were used interchangeably.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ if (isRetracted) {
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+
+ currFunction.accumulate(curValue);
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ RowData curAcc = accMapState.get(curKey.getLong(keyIdx));
+ currFunction.setAccumulators(curAcc);
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+
+ valueMapState.remove(curKey.getLong(keyIdx));
+ accMapState.remove(curKey.getLong(keyIdx));
+ if (iterator.hasPrevious()) {
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ RowData prevAcc = accMapState.get(prev.getLong(keyIdx));
+ currFunction.setAccumulators(prevAcc);
Review Comment:
could reuse the helper method suggested above: `setAccOfPreviousRow(iterator)`
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
Review Comment:
couldn't we insert `inputKey` here instead of creating a new object?
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
Review Comment:
```suggestion
accMapState.put(id, currFunction.getAccumulators());
```
I think we need to store the accumulators here and not the value. For some functions, these are the same (count, sum, max, min) but for some they are not (avg, ...).
It would be good to have some tests with agg functions for which the accumulator is different from the value.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
Review Comment:
I just realized that the order by logic is more involved than we thought.
If multiple rows have the same sort key, they all have the same aggregated value.
See this simple example from MySQL as an example:
```
mysql> select num, val, sum(val) over (order by num) oversum from tt;
+------+------+---------+
| num | val | oversum |
+------+------+---------+
| 1 | 1 | 6 |
| 1 | 2 | 6 |
| 1 | 3 | 6 |
| 2 | 3 | 9 |
| 3 | 4 | 13 |
+------+------+---------+
```
We could address this by storing not a single id per sort key, but a list of ids for each sort key.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
Review Comment:
We might also want to compare the *values* that were computed from the previous and new accumulators.
If these are the same, we don't need to update the current row, because it did not change.
However, we should not `break` in this case but just `continue`.
The reasoning is that the same accumulator will always result in the same computed value. However, different accumulators might result in the same computed value.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
Review Comment:
```suggestion
Long prevKey = sortedKeyList.get(iterator.previousIndex()).getLong(keyIdx);
RowData prevAcc = accMapState.get(prevKey);
currFunction.setAccumulators(prevAcc);
```
put this into a helper method `setAccOfPreviousRow(iterator)` and reuse it above (and in the remove code path)?
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
Review Comment:
Why don't set the `id` already here and insert this later into the list?
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
Review Comment:
```suggestion
accMapState.put(curKey.getLong(keyIdx), currFunction.getAccumulators());
```
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ if (isRetracted) {
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+
+ currFunction.accumulate(curValue);
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
Review Comment:
Before we compare the full row, we can compare just on the sort key which is already deserialized.
This means, we don't need to deserialize any rows before we reach rows with the same sort key
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
Review Comment:
we are comparing here the previous *accumulator* with the new *value* which is not correct.
we need to compare the previous accumulator with the current accumulator, i.e.,
```
prevAcc.equals(currFunction.getAccumulators())
```
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ if (isRetracted) {
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+
+ currFunction.accumulate(curValue);
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ RowData curAcc = accMapState.get(curKey.getLong(keyIdx));
+ currFunction.setAccumulators(curAcc);
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+
+ valueMapState.remove(curKey.getLong(keyIdx));
+ accMapState.remove(curKey.getLong(keyIdx));
+ if (iterator.hasPrevious()) {
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ RowData prevAcc = accMapState.get(prev.getLong(keyIdx));
+ currFunction.setAccumulators(prevAcc);
+ } else {
+ currFunction.setAccumulators(currFunction.createAccumulators());
+ }
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
Review Comment:
this could be moved up to the statements that update the `valueMapState` and `accMapState`.
That would keep all related state updates together.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
Review Comment:
We need to compute the previous *value* from the previous *accumulator*. They might not be the same (and also have different data types, e.g., for `AVG`). In your tests this is probably working because for some functions `accumulator == value` (count, sum, min, max).
For that we either need a second `function` or use `currFunction` and rotate its accumulators.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
Review Comment:
The logic in this loop can be reused in the `remove` method (after the retracted row was removed from the state).
We should move the code into a helper method `updateSubsequentRows(iterator)`.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
Review Comment:
In fact, I'm not sure if we handle this case correctly for the time-ordered over aggregates.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
Review Comment:
```suggestion
RowData curKey = iterator.next().getLong(keyIdx);
```
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ if (isRetracted) {
Review Comment:
We should use the helper method `updateSubsequentRows(iterator)` suggested above and reuse the logic.
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
Review Comment:
IMO, the code would be a bit easier to follow if the method was similarly structured as the `insert` method, i.e.,
* use the iterator to find the row to retract, remove the row from the list and map states and retract the row
* update the following accumulators and emit the update records
##########
flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/operators/over/NonTimeUnboundedPrecedingFunction.java:
##########
@@ -0,0 +1,591 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.runtime.operators.over;
+
+import org.apache.flink.api.common.functions.OpenContext;
+import org.apache.flink.api.common.state.MapState;
+import org.apache.flink.api.common.state.MapStateDescriptor;
+import org.apache.flink.api.common.state.ValueState;
+import org.apache.flink.api.common.state.ValueStateDescriptor;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.ListTypeInfo;
+import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.RowData.FieldGetter;
+import org.apache.flink.table.data.utils.JoinedRowData;
+import org.apache.flink.table.runtime.dataview.PerKeyStateDataViewStore;
+import org.apache.flink.table.runtime.functions.KeyedProcessFunctionWithCleanupState;
+import org.apache.flink.table.runtime.generated.AggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedAggsHandleFunction;
+import org.apache.flink.table.runtime.generated.GeneratedRecordComparator;
+import org.apache.flink.table.runtime.generated.GeneratedRecordEqualiser;
+import org.apache.flink.table.runtime.generated.RecordEqualiser;
+import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.flink.types.RowKind;
+import org.apache.flink.util.Collector;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.ListIterator;
+
+/** A basic implementation to support non-time unbounded over aggregate with retract mode. */
+public class NonTimeUnboundedPrecedingFunction<K>
+ extends KeyedProcessFunctionWithCleanupState<K, RowData, RowData> {
+ private static final long serialVersionUID = 1L;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonTimeUnboundedPrecedingFunction.class);
+
+ private final int keyIdx;
+ private final int sortKeyIdx;
+ private final FieldGetter sortKeyFieldGetter;
+
+ private final GeneratedAggsHandleFunction genAggsHandler;
+ private final GeneratedRecordEqualiser generatedRecordEqualiser;
+ private final GeneratedRecordComparator generatedRecordComparator;
+ private final GeneratedRecordComparator keyGeneratedRecordComparator;
+
+ // The util to compare two rows based on the sort attribute.
+ private transient Comparator<RowData> sortKeyComparator;
+ private transient Comparator<RowData> newSortKeyComparator;
+ // The record equaliser used to equal RowData.
+ private transient RecordEqualiser equaliser;
+
+ private final LogicalType[] accTypes;
+ private final LogicalType[] inputFieldTypes;
+ protected transient JoinedRowData output;
+
+ // state to hold the accumulators of the aggregations
+ private transient ValueState<RowData> accState;
+ // state to hold rows until state ttl expires
+ private transient ValueState<List<RowData>> inputState;
+
+ // state to hold the Long ID counter
+ private transient ValueState<Long> idState;
+
+ // state to hold a list of sorted keys with an artificial id
+ // The artificial id acts as the key in the valueMapState
+ private transient ValueState<List<RowData>> sortedKeyState;
+ // state to hold rows until state ttl expires
+ private transient MapState<Long, RowData> valueMapState;
+ // state to hold row ID and its associated accumulator
+ private transient MapState<Long, RowData> accMapState;
+
+ protected transient AggsHandleFunction currFunction;
+ protected transient AggsHandleFunction prevFunction;
+
+ public NonTimeUnboundedPrecedingFunction(
+ long minRetentionTime,
+ long maxRetentionTime,
+ GeneratedAggsHandleFunction genAggsHandler,
+ GeneratedRecordEqualiser genRecordEqualiser,
+ GeneratedRecordComparator genRecordComparator,
+ GeneratedRecordComparator keyGenRecordComparator,
+ LogicalType[] accTypes,
+ LogicalType[] inputFieldTypes,
+ FieldGetter sortKeyFieldGetter,
+ int sortKeyIdx) {
+ super(minRetentionTime, maxRetentionTime);
+ this.genAggsHandler = genAggsHandler;
+ this.generatedRecordEqualiser = genRecordEqualiser;
+ this.generatedRecordComparator = genRecordComparator;
+ this.keyGeneratedRecordComparator = keyGenRecordComparator;
+ this.accTypes = accTypes;
+ this.inputFieldTypes = inputFieldTypes;
+ this.sortKeyFieldGetter = sortKeyFieldGetter;
+ this.sortKeyIdx = sortKeyIdx;
+ this.keyIdx = 0;
+ }
+
+ @Override
+ public void open(OpenContext openContext) throws Exception {
+ // Initialize agg functions
+ currFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ currFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ prevFunction = genAggsHandler.newInstance(getRuntimeContext().getUserCodeClassLoader());
+ prevFunction.open(new PerKeyStateDataViewStore(getRuntimeContext()));
+
+ // Initialize output record
+ output = new JoinedRowData();
+
+ // Initialize record equaliser
+ equaliser =
+ generatedRecordEqualiser.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize sort comparator
+ sortKeyComparator =
+ generatedRecordComparator.newInstance(getRuntimeContext().getUserCodeClassLoader());
+
+ newSortKeyComparator =
+ keyGeneratedRecordComparator.newInstance(
+ getRuntimeContext().getUserCodeClassLoader());
+
+ // Initialize accumulator state
+ InternalTypeInfo<RowData> accTypeInfo = InternalTypeInfo.ofFields(accTypes);
+ ValueStateDescriptor<RowData> accStateDesc =
+ new ValueStateDescriptor<RowData>("accState", accTypeInfo);
+ accState = getRuntimeContext().getState(accStateDesc);
+
+ // Input elements are all binary rows as they came from network
+ InternalTypeInfo<RowData> inputType = InternalTypeInfo.ofFields(inputFieldTypes);
+ ListTypeInfo<RowData> rowListTypeInfo = new ListTypeInfo<RowData>(inputType);
+
+ ValueStateDescriptor<List<RowData>> inputStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("inputState", rowListTypeInfo);
+
+ // Initialize state which maintains records in sorted(ASC) order
+ inputState = getRuntimeContext().getState(inputStateDescriptor);
+
+ ValueStateDescriptor<Long> idStateDescriptor =
+ new ValueStateDescriptor<Long>("idState", Long.class);
+ idState = getRuntimeContext().getState(idStateDescriptor);
+
+ LogicalType idType = DataTypes.BIGINT().getLogicalType();
+ LogicalType sortKeyType = inputType.toRowType().getTypeAt(sortKeyIdx);
+ RowType sortedRow = RowType.of(idType, sortKeyType);
+ LogicalType[] sortedRowTypes = sortedRow.getChildren().toArray(new LogicalType[0]);
+ InternalTypeInfo<RowData> sortedKeyRowType = InternalTypeInfo.ofFields(sortedRowTypes);
+ ListTypeInfo<RowData> sortedKeyTypeInfo = new ListTypeInfo<>(sortedKeyRowType);
+ // Initialize state which maintains a sorted list of pair(ID, sortKey)
+ ValueStateDescriptor<List<RowData>> sortedKeyStateDescriptor =
+ new ValueStateDescriptor<List<RowData>>("sortedKeyState", sortedKeyTypeInfo);
+ sortedKeyState = getRuntimeContext().getState(sortedKeyStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> valueStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("valueState", Types.LONG, inputType);
+ valueMapState = getRuntimeContext().getMapState(valueStateDescriptor);
+
+ MapStateDescriptor<Long, RowData> accStateDescriptor =
+ new MapStateDescriptor<Long, RowData>("accMapState", Types.LONG, accTypeInfo);
+ accMapState = getRuntimeContext().getMapState(accStateDescriptor);
+
+ initCleanupTimeState("NonTimeUnboundedPrecedingFunctionCleanupTime");
+ }
+
+ /**
+ * Puts an element from the input stream into state. Emits the aggregated value for the newly
+ * inserted element. For append stream emits updates(UB, UA) for all elements which are present
+ * after the newly inserted element. For retract stream emits an UB for the element and
+ * thereafter emits updates(UB, UA) for all elements which are present after the retracted
+ * element.
+ *
+ * @param input The input value.
+ * @param ctx A {@link Context} that allows querying the timestamp of the element and getting
+ * TimerService for registering timers and querying the time. The context is only valid
+ * during the invocation of this method, do not store it.
+ * @param out The collector for returning result values.
+ * @throws Exception
+ */
+ @Override
+ public void processElement(
+ RowData input,
+ KeyedProcessFunction<K, RowData, RowData>.Context ctx,
+ Collector<RowData> out)
+ throws Exception {
+ // register state-cleanup timer
+ registerProcessingCleanupTimer(ctx, ctx.timerService().currentProcessingTime());
+
+ // get last accumulator
+ RowData lastAccumulatorCurr = accState.value();
+ if (lastAccumulatorCurr == null) {
+ // initialize accumulator
+ lastAccumulatorCurr = currFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ currFunction.setAccumulators(lastAccumulatorCurr);
+
+ // get last accumulator
+ RowData lastAccumulatorPrev = accState.value();
+ if (lastAccumulatorPrev == null) {
+ // initialize accumulator
+ lastAccumulatorPrev = prevFunction.createAccumulators();
+ }
+ // set accumulator in function context first
+ prevFunction.setAccumulators(lastAccumulatorPrev);
+
+ RowKind rowKind = input.getRowKind();
+
+ if (rowKind == RowKind.INSERT || rowKind == RowKind.UPDATE_AFTER) {
+ // insertIntoSortedList(input, out);
+ // insertIntoSortedListWithValueMap(input, out);
+ insertIntoSortedListWithValueMapAndAccumulatorMap(input, out);
+ } else if (rowKind == RowKind.DELETE || rowKind == RowKind.UPDATE_BEFORE) {
+ // removeFromSortedList(input, out);
+ // removeFromSortedListWithValueMap(input, out);
+ removeFromSortedListWithValueMapAndAccumulatorMap(input, out);
+ }
+
+ // Reset acc state since we can have out of order inserts into the ordered list
+ currFunction.resetAccumulators();
+ prevFunction.resetAccumulators();
+ currFunction.cleanup();
+ prevFunction.cleanup();
+ }
+
+ private void insertIntoSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ List<RowData> rowList = inputState.value();
+ if (rowList == null) {
+ rowList = new ArrayList<>();
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = rowData.getRowKind();
+ rowData.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ if (sortKeyComparator.compare(curRow, rowData) > 0) {
+ iterator.previous();
+ iterator.add(rowData);
+ isInserted = true;
+ break;
+ }
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(rowData);
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(rowData);
+
+ // Update state with the newly inserted row
+ inputState.update(rowList);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ isInserted = true;
+ id++;
+ break;
+ }
+ // Can also add the accKey to the sortedKeyList to avoid reading from the valueMapState
+ RowData curRow = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ id++;
+ }
+
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ }
+ }
+
+ private void insertIntoSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ if (sortedKeyList == null) {
+ sortedKeyList = new ArrayList<>();
+ }
+ Long id = idState.value();
+ if (id == null) {
+ id = 0L;
+ }
+ boolean isInserted = false;
+ RowKind origRowKind = input.getRowKind();
+ input.setRowKind(RowKind.INSERT);
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ RowData inputKey = GenericRowData.of(-1L, sortKeyFieldGetter.getFieldOrNull(input));
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next(); // (ID, sortKey)
+ if (newSortKeyComparator.compare(curKey, inputKey) > 0) {
+ iterator.previous();
+ RowData prev = sortedKeyList.get(iterator.previousIndex());
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ Long prevKey = prev.getLong(keyIdx);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ currFunction.accumulate(input);
+ accMapState.put(id, currFunction.getValue());
+ isInserted = true;
+ id++;
+ break;
+ }
+ }
+
+ // Add to the end of the list
+ if (!isInserted) {
+ if (iterator.hasPrevious()) {
+ RowData prev = iterator.previous();
+ Long prevKey = prev.getLong(0);
+ RowData prevAcc = accMapState.get(prevKey);
+ currFunction.setAccumulators(prevAcc);
+ iterator.next();
+ }
+ // Only accumulate rowData with currFunction
+ currFunction.accumulate(input);
+ iterator.add(GenericRowData.of(id, sortKeyFieldGetter.getFieldOrNull(input)));
+ valueMapState.put(id, input);
+ accMapState.put(id, currFunction.getAccumulators());
+ id++;
+ }
+
+ // Update sorted key state with the newly inserted row's key
+ sortedKeyState.update(sortedKeyList);
+ idState.update(id);
+
+ // prepare output row
+ output.setRowKind(origRowKind);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+
+ // Emit updated agg value for all records after newly inserted row
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
+ // If previously accumulated value is same as the newly accumulated value
+ // then no need to send updates downstream since nothing has changed
+ break;
+ }
+ // Generate UPDATE_BEFORE
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevAcc);
+ out.collect(output);
+ // Generate UPDATE_AFTER
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+
+ accMapState.put(curKey.getLong(keyIdx), currFunction.getValue());
+ }
+ }
+
+ private void removeFromSortedList(RowData rowData, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ rowData.setRowKind(RowKind.INSERT);
+ List<RowData> rowList = inputState.value();
+ ListIterator<RowData> iterator = rowList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curRow = iterator.next();
+ currFunction.accumulate(curRow);
+ prevFunction.accumulate(curRow);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curRow, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curRow, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curRow, rowData)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(rowData, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ currFunction.retract(curRow);
+ isRetracted = true;
+ }
+ }
+
+ // Update state without the retracted row
+ inputState.update(rowList);
+ }
+
+ private void removeFromSortedListWithValueMap(RowData input, Collector<RowData> out)
+ throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ currFunction.accumulate(curValue);
+ prevFunction.accumulate(curValue);
+ if (isRetracted) {
+ // Emit updated agg value for all records after retraction
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(curValue, prevFunction.getValue());
+ out.collect(output);
+
+ output.setRowKind(RowKind.UPDATE_AFTER);
+ output.replace(curValue, currFunction.getValue());
+ out.collect(output);
+ } else if (equaliser.equals(curValue, input)) {
+ // Retract record
+ output.setRowKind(RowKind.UPDATE_BEFORE);
+ output.replace(input, currFunction.getValue());
+ out.collect(output);
+ iterator.remove();
+ valueMapState.remove(curKey.getLong(keyIdx));
+ currFunction.retract(curValue);
+ isRetracted = true;
+ }
+ }
+
+ // Update sorted key state without the retracted row
+ sortedKeyState.update(sortedKeyList);
+ }
+
+ private void removeFromSortedListWithValueMapAndAccumulatorMap(
+ RowData input, Collector<RowData> out) throws Exception {
+ boolean isRetracted = false;
+ input.setRowKind(RowKind.INSERT);
+ List<RowData> sortedKeyList = sortedKeyState.value();
+ ListIterator<RowData> iterator = sortedKeyList.listIterator();
+
+ while (iterator.hasNext()) {
+ RowData curKey = iterator.next();
+ RowData curValue = valueMapState.get(curKey.getLong(keyIdx));
+ if (isRetracted) {
+ RowData prevAcc = accMapState.get(curKey.getLong(keyIdx));
+ if (prevAcc.equals(currFunction.getValue())) {
Review Comment:
We should compare the previous accumulator with the new accumulator (not the new value).
For the new value, we also need to `accumulate()` the current value first.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org