You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by li...@apache.org on 2023/04/25 06:18:39 UTC

[flink-ml] branch master updated: [FLINK-31887] Upgrade Flink version of Flink ML to 1.16.1

This is an automated email from the ASF dual-hosted git repository.

lindong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-ml.git


The following commit(s) were added to refs/heads/master by this push:
     new 5b193dee [FLINK-31887] Upgrade Flink version of Flink ML to 1.16.1
5b193dee is described below

commit 5b193deedff2aa2be96fbfb5304f812caace9e12
Author: JiangXin <ji...@gmail.com>
AuthorDate: Tue Apr 25 14:18:33 2023 +0800

    [FLINK-31887] Upgrade Flink version of Flink ML to 1.16.1
    
    This closes #234.
---
 docs/content/docs/operators/feature/minhashlsh.md  |  3 +-
 .../try-flink-ml/java/build-your-own-project.md    |  2 +-
 docs/content/docs/try-flink-ml/java/quick-start.md |  2 +-
 .../docs/try-flink-ml/python/quick-start.md        |  2 +-
 flink-ml-benchmark/README.md                       |  2 +-
 .../flink/iteration/operator/OperatorUtils.java    | 55 +++++++++++++---------
 .../coordinator/HeadOperatorCoordinator.java       | 31 ++++++------
 .../perround/AbstractPerRoundWrapperOperator.java  | 17 +++++--
 .../proxy/state/ProxyKeyedStateBackend.java        |  4 +-
 .../flink/iteration/operator/HeadOperatorTest.java |  6 ++-
 .../coordinator/HeadOperatorCoordinatorTest.java   |  9 ++--
 .../apache/flink/ml/classification/KnnTest.java    |  4 +-
 .../org/apache/flink/ml/feature/BinarizerTest.java |  7 ++-
 .../apache/flink/ml/feature/BucketizerTest.java    |  3 +-
 .../flink/ml/feature/CountVectorizerTest.java      |  3 +-
 .../org/apache/flink/ml/feature/ImputerTest.java   |  3 +-
 .../apache/flink/ml/feature/InteractionTest.java   |  3 +-
 .../flink/ml/feature/KBinsDiscretizerTest.java     |  3 +-
 .../apache/flink/ml/feature/MaxAbsScalerTest.java  |  4 +-
 .../apache/flink/ml/feature/MinHashLSHTest.java    |  8 ++--
 .../apache/flink/ml/feature/MinMaxScalerTest.java  |  3 +-
 .../apache/flink/ml/feature/NormalizerTest.java    |  3 +-
 .../flink/ml/feature/PolynomialExpansionTest.java  |  3 +-
 .../flink/ml/feature/RandomSplitterTest.java       |  3 +-
 .../apache/flink/ml/feature/RobustScalerTest.java  |  3 +-
 .../ml/feature/VarianceThresholdSelectorTest.java  |  3 +-
 .../apache/flink/ml/feature/VectorIndexerTest.java |  3 +-
 .../feature/stringindexer/StringIndexerTest.java   |  3 +-
 .../org/apache/flink/ml/stats/ChiSqTestTest.java   |  3 +-
 flink-ml-python/dev/dev-requirements.txt           | 10 ++--
 flink-ml-python/pom.xml                            |  2 +-
 .../examples/ml/feature/minhashlsh_example.py      |  3 +-
 .../pyflink/ml/classification/tests/test_knn.py    |  4 +-
 .../pyflink/ml/feature/tests/test_minhashlsh.py    | 13 ++---
 flink-ml-python/pyflink/ml/linalg.py               | 12 ++---
 flink-ml-python/pyflink/ml/tests/test_pipeline.py  |  3 +-
 flink-ml-python/setup.py                           |  7 ++-
 pom.xml                                            |  2 +-
 38 files changed, 153 insertions(+), 101 deletions(-)

diff --git a/docs/content/docs/operators/feature/minhashlsh.md b/docs/content/docs/operators/feature/minhashlsh.md
index 5c3aee92..22ce0b88 100644
--- a/docs/content/docs/operators/feature/minhashlsh.md
+++ b/docs/content/docs/operators/feature/minhashlsh.md
@@ -221,6 +221,7 @@ public class MinHashLSHExample {
 from pyflink.common import Types
 from pyflink.datastream import StreamExecutionEnvironment
 from pyflink.table import StreamTableEnvironment
+from pyflink.table.expressions import col
 
 from pyflink.ml.linalg import Vectors, SparseVectorTypeInfo
 from pyflink.ml.feature.lsh import MinHashLSH
@@ -268,7 +269,7 @@ for result in t_env.to_data_stream(output).execute_and_collect():
 
 # Finds approximate nearest neighbors of the key.
 key = Vectors.sparse(6, [1, 3], [1., 1.])
-output = model.approx_nearest_neighbors(data_a, key, 2).select("id, distCol")
+output = model.approx_nearest_neighbors(data_a, key, 2).select(col("id"), col("distCol"))
 for result in t_env.to_data_stream(output).execute_and_collect():
     id_value = result[field_names.index("id")]
     dist_value = result[-1]
diff --git a/docs/content/docs/try-flink-ml/java/build-your-own-project.md b/docs/content/docs/try-flink-ml/java/build-your-own-project.md
index 713bb924..adebe30e 100644
--- a/docs/content/docs/try-flink-ml/java/build-your-own-project.md
+++ b/docs/content/docs/try-flink-ml/java/build-your-own-project.md
@@ -80,7 +80,7 @@ the basic skeleton of a project, with some necessary Flink dependencies.
 $ mvn archetype:generate \
     -DarchetypeGroupId=org.apache.flink \
     -DarchetypeArtifactId=flink-quickstart-java \
-    -DarchetypeVersion=1.15.1 \
+    -DarchetypeVersion=1.16.1 \
     -DgroupId=kmeans-example \
     -DartifactId=kmeans-example \
     -Dversion=0.1 \
diff --git a/docs/content/docs/try-flink-ml/java/quick-start.md b/docs/content/docs/try-flink-ml/java/quick-start.md
index 725df28d..f92cb71b 100644
--- a/docs/content/docs/try-flink-ml/java/quick-start.md
+++ b/docs/content/docs/try-flink-ml/java/quick-start.md
@@ -50,7 +50,7 @@ $ java -version
 
 ## Download Flink
 
-[Download 1.15 or a higher version of
+[Download 1.16 or a higher version of
 Flink](https://flink.apache.org/downloads.html), then extract the archive:
 
 ```shell
diff --git a/docs/content/docs/try-flink-ml/python/quick-start.md b/docs/content/docs/try-flink-ml/python/quick-start.md
index d53ebc40..4ee84df0 100644
--- a/docs/content/docs/try-flink-ml/python/quick-start.md
+++ b/docs/content/docs/try-flink-ml/python/quick-start.md
@@ -248,7 +248,7 @@ $ java -version
 
 ### Download Flink
 
-[Download 1.15 or a higher version of
+[Download 1.16 or a higher version of
 Flink](https://flink.apache.org/downloads.html), then extract the archive:
 
 ```shell
diff --git a/flink-ml-benchmark/README.md b/flink-ml-benchmark/README.md
index 69e49397..b46244ee 100644
--- a/flink-ml-benchmark/README.md
+++ b/flink-ml-benchmark/README.md
@@ -7,7 +7,7 @@ stages in a Linux/MacOS environment.
 
 ### Install Flink
 
-Please make sure Flink 1.15 or higher version has been installed in your local
+Please make sure Flink 1.16 or higher version has been installed in your local
 environment. You can refer to the [local
 installation](https://nightlies.apache.org/flink/flink-docs-master/docs/try-flink/local_installation/)
 instruction on Flink's document website for how to achieve this.
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java
index 3d67b7fa..7b81e524 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java
@@ -47,7 +47,6 @@ import java.util.Arrays;
 import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.Executor;
-import java.util.stream.Stream;
 
 import static org.apache.flink.util.Preconditions.checkState;
 
@@ -136,31 +135,20 @@ public class OperatorUtils {
         wrappedConfig.setTypeSerializerOut(
                 ((IterationRecordSerializer<?>) typeSerializerOut).getInnerSerializer());
 
-        Stream.concat(
-                        config.getChainedOutputs(cl).stream(),
-                        config.getNonChainedOutputs(cl).stream())
+        config.getChainedOutputs(cl)
                 .forEach(
-                        edge -> {
-                            OutputTag<?> outputTag = edge.getOutputTag();
-                            if (outputTag != null) {
-                                TypeSerializer<?> typeSerializerSideOut =
-                                        config.getTypeSerializerSideOut(outputTag, cl);
-                                checkState(
-                                        typeSerializerSideOut instanceof IterationRecordSerializer,
-                                        "The serializer of side output with tag[%s] should be IterationRecordSerializer but it is %s.",
-                                        outputTag,
-                                        typeSerializerSideOut);
-                                wrappedConfig.setTypeSerializerSideOut(
-                                        new OutputTag<>(
-                                                outputTag.getId(),
-                                                ((IterationRecordTypeInfo<?>)
-                                                                outputTag.getTypeInfo())
-                                                        .getInnerTypeInfo()),
-                                        ((IterationRecordSerializer) typeSerializerSideOut)
-                                                .getInnerSerializer());
-                            }
+                        chainedOutput -> {
+                            OutputTag<?> outputTag = chainedOutput.getOutputTag();
+                            setTypeSerializerSideOut(outputTag, config, wrappedConfig, cl);
+                        });
+        config.getOperatorNonChainedOutputs(cl)
+                .forEach(
+                        nonChainedOutput -> {
+                            OutputTag<?> outputTag = nonChainedOutput.getOutputTag();
+                            setTypeSerializerSideOut(outputTag, config, wrappedConfig, cl);
                         });
 
+        wrappedConfig.serializeAllConfigs();
         return wrappedConfig;
     }
 
@@ -185,4 +173,25 @@ public class OperatorUtils {
                                 operatorId,
                                 UUID.randomUUID().toString()));
     }
+
+    private static void setTypeSerializerSideOut(
+            OutputTag<?> outputTag,
+            StreamConfig config,
+            StreamConfig wrappedConfig,
+            ClassLoader cl) {
+        if (outputTag == null) {
+            return;
+        }
+        TypeSerializer<?> typeSerializerSideOut = config.getTypeSerializerSideOut(outputTag, cl);
+        checkState(
+                typeSerializerSideOut instanceof IterationRecordSerializer,
+                "The serializer of side output with tag[%s] should be IterationRecordSerializer but it is %s.",
+                outputTag,
+                typeSerializerSideOut);
+        wrappedConfig.setTypeSerializerSideOut(
+                new OutputTag<>(
+                        outputTag.getId(),
+                        ((IterationRecordTypeInfo<?>) outputTag.getTypeInfo()).getInnerTypeInfo()),
+                ((IterationRecordSerializer) typeSerializerSideOut).getInnerSerializer());
+    }
 }
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java
index 1b6b4d9c..2494c5f7 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java
@@ -58,11 +58,6 @@ public class HeadOperatorCoordinator implements OperatorCoordinator, SharedProgr
     @Override
     public void start() {}
 
-    @Override
-    public void subtaskReady(int subtaskIndex, SubtaskGateway subtaskGateway) {
-        this.subtaskGateways[subtaskIndex] = subtaskGateway;
-    }
-
     @Override
     public void resetToCheckpoint(long checkpointId, @Nullable byte[] bytes) {
         for (int i = 0; i < context.currentParallelism(); ++i) {
@@ -71,19 +66,15 @@ public class HeadOperatorCoordinator implements OperatorCoordinator, SharedProgr
     }
 
     @Override
-    public void subtaskFailed(int subtaskIndex, @Nullable Throwable throwable) {
-        sharedProgressAligner.removeProgressInfo(context.getOperatorId(), subtaskIndex);
-    }
-
-    @Override
-    public void handleEventFromOperator(int subtaskIndex, OperatorEvent operatorEvent) {
-        if (operatorEvent instanceof SubtaskAlignedEvent) {
+    public void handleEventFromOperator(int subtask, int attemptNumber, OperatorEvent event)
+            throws Exception {
+        if (event instanceof SubtaskAlignedEvent) {
             sharedProgressAligner.reportSubtaskProgress(
-                    context.getOperatorId(), subtaskIndex, (SubtaskAlignedEvent) operatorEvent);
-        } else if (operatorEvent instanceof TerminatingOnInitializeEvent) {
+                    context.getOperatorId(), subtask, (SubtaskAlignedEvent) event);
+        } else if (event instanceof TerminatingOnInitializeEvent) {
             sharedProgressAligner.notifyGloballyTerminating();
         } else {
-            throw new UnsupportedOperationException("Not supported event: " + operatorEvent);
+            throw new UnsupportedOperationException("Not supported event: " + event);
         }
     }
 
@@ -116,6 +107,16 @@ public class HeadOperatorCoordinator implements OperatorCoordinator, SharedProgr
     @Override
     public void subtaskReset(int i, long l) {}
 
+    @Override
+    public void executionAttemptFailed(int subtask, int attemptNumber, @Nullable Throwable reason) {
+        sharedProgressAligner.removeProgressInfo(context.getOperatorId(), subtask);
+    }
+
+    @Override
+    public void executionAttemptReady(int subtask, int attemptNumber, SubtaskGateway gateway) {
+        this.subtaskGateways[subtask] = gateway;
+    }
+
     /** The factory of {@link HeadOperatorCoordinator}. */
     public static class HeadOperatorCoordinatorProvider implements Provider {
 
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java
index 4ae88858..d6728764 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java
@@ -610,6 +610,10 @@ public abstract class AbstractPerRoundWrapperOperator<T, S extends StreamOperato
                             keyedStateBackend, HeapKeyedStateBackend.class, "registeredKVStates")
                     .entrySet()
                     .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+            ReflectionUtils.<Map<String, ?>>getFieldValue(
+                            keyedStateBackend, HeapKeyedStateBackend.class, "createdKVStates")
+                    .entrySet()
+                    .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
             ReflectionUtils.<Map<String, ?>>getFieldValue(
                             keyedStateBackend,
                             AbstractKeyedStateBackend.class,
@@ -640,12 +644,17 @@ public abstract class AbstractPerRoundWrapperOperator<T, S extends StreamOperato
                             });
             kvStateInformation.entrySet().removeIf(entry -> entry.getKey().startsWith(roundPrefix));
 
-            Map<String, ?> field =
-                    ReflectionUtils.getFieldValue(
+            ReflectionUtils.<Map<String, ?>>getFieldValue(
+                            keyedStateBackend, RocksDBKeyedStateBackend.class, "createdKVStates")
+                    .entrySet()
+                    .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+            ReflectionUtils.<Map<String, ?>>getFieldValue(
                             keyedStateBackend,
                             AbstractKeyedStateBackend.class,
-                            "keyValueStatesByName");
-            field.entrySet().removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+                            "keyValueStatesByName")
+                    .entrySet()
+                    .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+
         } else {
             LOG.warn("Unable to cleanup the keyed state {}", keyedStateBackend);
         }
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java
index bc51f957..0fb13d09 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java
@@ -127,7 +127,7 @@ public class ProxyKeyedStateBackend<K> implements CheckpointableKeyedStateBacken
 
     @Nonnull
     @Override
-    public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
+    public <N, SV, SEV, S extends State, IS extends S> IS createOrUpdateInternalState(
             @Nonnull TypeSerializer<N> namespaceSerializer,
             @Nonnull StateDescriptor<S, SV> stateDesc,
             @Nonnull
@@ -135,7 +135,7 @@ public class ProxyKeyedStateBackend<K> implements CheckpointableKeyedStateBacken
                             snapshotTransformFactory)
             throws Exception {
         StateDescriptor<S, ?> newDescriptor = createNewDescriptor(stateDesc);
-        return wrappedBackend.createInternalState(
+        return wrappedBackend.createOrUpdateInternalState(
                 namespaceSerializer, newDescriptor, snapshotTransformFactory);
     }
 
diff --git a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java
index 17778933..fc9bd6b4 100644
--- a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java
+++ b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java
@@ -824,7 +824,11 @@ public class HeadOperatorTest extends TestLogger {
                                         RecordingOperatorEventGateway::new),
                                 operatorId)
                         .build()) {
-            return runnable.apply(harness);
+            try {
+                return runnable.apply(harness);
+            } finally {
+                RecordingHeadOperatorFactory.latestHeadOperator.close();
+            }
         }
     }
 
diff --git a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java
index b77d2727..63d1e085 100644
--- a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java
+++ b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java
@@ -41,7 +41,7 @@ import static org.junit.Assert.assertEquals;
 public class HeadOperatorCoordinatorTest extends TestLogger {
 
     @Test(timeout = 60000L)
-    public void testForwardEvents() throws InterruptedException {
+    public void testForwardEvents() throws Exception {
         IterationID iterationId = new IterationID();
         List<OperatorID> operatorIds = Arrays.asList(new OperatorID(), new OperatorID());
         List<Integer> parallelisms = Arrays.asList(2, 3);
@@ -88,19 +88,20 @@ public class HeadOperatorCoordinatorTest extends TestLogger {
             EventReceivingTasks receivingTasks,
             int parallelism) {
         for (int i = 0; i < parallelism; i++) {
-            coordinator.subtaskReady(i, receivingTasks.createGatewayForSubtask(i));
+            coordinator.executionAttemptReady(i, 0, receivingTasks.createGatewayForSubtask(i, 0));
         }
     }
 
     private void receiveEvent(
             List<HeadOperatorCoordinator> coordinators,
             List<Integer> parallelisms,
-            BiFunction<Integer, Integer, List<OperatorEvent>> eventFactory) {
+            BiFunction<Integer, Integer, List<OperatorEvent>> eventFactory)
+            throws Exception {
         for (int i = 0; i < coordinators.size(); ++i) {
             for (int j = 0; j < parallelisms.get(i); ++j) {
                 List<OperatorEvent> events = eventFactory.apply(i, j);
                 for (OperatorEvent event : events) {
-                    coordinators.get(i).handleEventFromOperator(j, event);
+                    coordinators.get(i).handleEventFromOperator(j, 0, event);
                 }
             }
         }
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java
index 483dc34d..a44ae5d4 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java
@@ -151,8 +151,8 @@ public class KnnTest extends AbstractTestBase {
                         .setFeaturesCol("test_features")
                         .setK(4)
                         .setPredictionCol("test_prediction");
-        KnnModel model = knn.fit(trainData.as("test_features, test_label"));
-        Table output = model.transform(predictData.as("test_features, test_label"))[0];
+        KnnModel model = knn.fit(trainData.as("test_features", "test_label"));
+        Table output = model.transform(predictData.as("test_features", "test_label"))[0];
         assertEquals(
                 Arrays.asList("test_features", "test_label", "test_prediction"),
                 output.getResolvedSchema().getColumnNames());
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java
index eb06b0c6..86341768 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -99,8 +100,10 @@ public class BinarizerTest extends AbstractTestBase {
         }
         doubleValues.sort(Double::compare);
         assertArrayEquals(EXPECTED_VALUE_OUTPUT, doubleValues.toArray());
-        compareResultCollections(EXPECTED_DENSE_OUTPUT, denseVectorValues, TestUtils::compare);
-        compareResultCollections(EXPECTED_SPARSE_OUTPUT, sparseVectorValues, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(
+                EXPECTED_DENSE_OUTPUT, denseVectorValues, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(
+                EXPECTED_SPARSE_OUTPUT, sparseVectorValues, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java
index 2c49dfa6..66adde8e 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -97,7 +98,7 @@ public class BucketizerTest extends AbstractTestBase {
             result.add(outRow);
         }
 
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expectedResult, result, Comparator.comparingInt(r -> ((Integer) r.getField(0))));
     }
 
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java
index 32d58ccc..531a4eb0 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -110,7 +111,7 @@ public class CountVectorizerTest extends AbstractTestBase {
                                 (MapFunction<Row, SparseVector>)
                                         row -> (SparseVector) row.getField(outputCol));
         List<SparseVector> result = IteratorUtils.toList(stream.executeAndCollect());
-        compareResultCollections(expected, result, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
index d8805a70..206695cd 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -138,7 +139,7 @@ public class ImputerTest extends AbstractTestBase {
                                     return outputRow;
                                 })
                         .collect(Collectors.toList());
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expected,
                 result,
                 (row1, row2) -> {
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java
index 0b0cc995..dc108531 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -107,7 +108,7 @@ public class InteractionTest extends AbstractTestBase {
                 resultVec.add(row.getFieldAs(outputCol));
             }
         }
-        compareResultCollections(expectedData, resultVec, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expectedData, resultVec, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java
index e6959baa..4c70baf8 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -133,7 +134,7 @@ public class KBinsDiscretizerTest extends AbstractTestBase {
         List<Row> collectedResult =
                 IteratorUtils.toList(
                         tEnv.toDataStream(output.select($(predictionCol))).executeAndCollect());
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expectedOutput,
                 collectedResult,
                 (o1, o2) ->
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java
index 6b8edf05..f7eeba7c 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -46,7 +47,6 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 
-import static org.apache.flink.test.util.TestBaseUtils.compareResultCollections;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
@@ -135,7 +135,7 @@ public class MaxAbsScalerTest {
                                 VectorTypeInfo.INSTANCE);
 
         List<Vector> result = IteratorUtils.toList(stream.executeAndCollect());
-        compareResultCollections(expectedData, result, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expectedData, result, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
index d92f062d..862c190d 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
@@ -35,6 +35,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -103,7 +104,7 @@ public class MinHashLSHTest extends AbstractTestBase {
         StreamTableEnvironment tEnv =
                 (StreamTableEnvironment) ((TableImpl) output).getTableEnvironment();
         List<Row> results = IteratorUtils.toList(tEnv.toDataStream(output).executeAndCollect());
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expected,
                 results,
                 (d0, d1) -> {
@@ -379,7 +380,8 @@ public class MinHashLSHTest extends AbstractTestBase {
         Table output =
                 lshModel.approxNearestNeighbors(inputTable, key, 2).select($("id"), $("distCol"));
         List<Row> results = IteratorUtils.toList(output.execute().collect());
-        compareResultCollections(expected, results, Comparator.comparing(r -> r.getFieldAs(0)));
+        TestBaseUtils.compareResultCollections(
+                expected, results, Comparator.comparing(r -> r.getFieldAs(0)));
     }
 
     @Test
@@ -418,7 +420,7 @@ public class MinHashLSHTest extends AbstractTestBase {
 
         Table output = lshModel.approxSimilarityJoin(dataA, dataB, .6, "id");
         List<Row> results = IteratorUtils.toList(output.execute().collect());
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expected,
                 results,
                 Comparator.<Row>comparingInt(r -> r.getFieldAs(0))
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java
index 32451635..8481b4f3 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -95,7 +96,7 @@ public class MinMaxScalerTest extends AbstractTestBase {
                                 (MapFunction<Row, DenseVector>)
                                         row -> (DenseVector) row.getField(outputCol));
         List<DenseVector> result = IteratorUtils.toList(stream.executeAndCollect());
-        compareResultCollections(expected, result, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java
index 13e28c68..cf8c5c61 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -109,7 +110,7 @@ public class NormalizerTest extends AbstractTestBase {
                 resultVec.add(row.getFieldAs(outputCol));
             }
         }
-        compareResultCollections(expectedData, resultVec, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expectedData, resultVec, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java
index 5454eabf..be6fa7dc 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -100,7 +101,7 @@ public class PolynomialExpansionTest extends AbstractTestBase {
                 resultVec.add(row.getFieldAs(outputCol));
             }
         }
-        compareResultCollections(expectedData, resultVec, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expectedData, resultVec, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java
index 75ad42ab..4c482b7f 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -152,7 +153,7 @@ public class RandomSplitterTest extends AbstractTestBase {
             results.addAll(result);
         }
         assertEquals(expectedData.size(), results.size());
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expectedData, results, Comparator.comparingLong(row -> row.getFieldAs(0)));
     }
 }
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
index e8179024..2e58b4d1 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -104,7 +105,7 @@ public class RobustScalerTest extends AbstractTestBase {
                                 (MapFunction<Row, DenseVector>)
                                         row -> (DenseVector) row.getField(outputCol));
         List<DenseVector> result = IteratorUtils.toList(stream.executeAndCollect());
-        compareResultCollections(expected, result, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
index 43a893ce..194b247d 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -107,7 +108,7 @@ public class VarianceThresholdSelectorTest extends AbstractTestBase {
                                 (MapFunction<Row, Vector>) row -> (Vector) row.getField(outputCol),
                                 VectorTypeInfo.INSTANCE);
         List<Vector> result = IteratorUtils.toList(stream.executeAndCollect());
-        compareResultCollections(expected, result, TestUtils::compare);
+        TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
     }
 
     @Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java
index de69f4d5..5d01de29 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.api.Expressions;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -277,7 +278,7 @@ public class VectorIndexerTest extends AbstractTestBase {
                 IteratorUtils.toList(
                         tEnv.toDataStream(output.select(Expressions.$(outputCol)))
                                 .executeAndCollect());
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expectedOutput,
                 collectedResult,
                 Comparator.comparingInt(o -> (o.getField(0)).hashCode()));
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java
index 9f4a5ae8..6105a3b0 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -421,7 +422,7 @@ public class StringIndexerTest extends AbstractTestBase {
     }
 
     static void verifyPredictionResult(List<Row> expected, List<Row> result) {
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expected,
                 result,
                 (row1, row2) -> {
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java
index 87a3f376..e1dbd0d3 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.api.internal.TableImpl;
 import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
 
 import org.apache.commons.collections.IteratorUtils;
@@ -120,7 +121,7 @@ public class ChiSqTestTest extends AbstractTestBase {
 
         List<Row> result = IteratorUtils.toList(outputDataStream.executeAndCollect());
 
-        compareResultCollections(
+        TestBaseUtils.compareResultCollections(
                 expected,
                 result,
                 (row1, row2) -> {
diff --git a/flink-ml-python/dev/dev-requirements.txt b/flink-ml-python/dev/dev-requirements.txt
index 68315a66..b8ac56c8 100755
--- a/flink-ml-python/dev/dev-requirements.txt
+++ b/flink-ml-python/dev/dev-requirements.txt
@@ -14,11 +14,13 @@
 # limitations under the License.
 setuptools>=18.0
 wheel
-apache-flink==1.15.1
-pandas>=1.0,<1.2.0
+apache-flink==1.16.1
+pandas>=1.3.0,<1.4.0; python_version >= '3.7'
+pandas>=1.0,<1.2.0; python_version < '3.7'
 jsonpickle==2.0.0
-cloudpickle==1.2.2
+cloudpickle==2.1.0
 pytest==4.4.1
 flake8==4.0.1
 mypy==0.910
-numpy>=1.14.3,<1.20
\ No newline at end of file
+numpy>=1.21.4,<1.22.0; python_version >= '3.7'
+numpy>=1.14.3,<1.20; python_version < '3.7'
\ No newline at end of file
diff --git a/flink-ml-python/pom.xml b/flink-ml-python/pom.xml
index 5a24c2f2..3e442fec 100644
--- a/flink-ml-python/pom.xml
+++ b/flink-ml-python/pom.xml
@@ -39,7 +39,7 @@ under the License.
 
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-python_2.12</artifactId>
+            <artifactId>flink-python</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
diff --git a/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py b/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py
index 29b1a4d0..a054f232 100644
--- a/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py
+++ b/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py
@@ -22,6 +22,7 @@
 from pyflink.common import Types
 from pyflink.datastream import StreamExecutionEnvironment
 from pyflink.table import StreamTableEnvironment
+from pyflink.table.expressions import col
 
 from pyflink.ml.linalg import Vectors, SparseVectorTypeInfo
 from pyflink.ml.feature.lsh import MinHashLSH
@@ -69,7 +70,7 @@ for result in t_env.to_data_stream(output).execute_and_collect():
 
 # Finds approximate nearest neighbors of the key.
 key = Vectors.sparse(6, [1, 3], [1., 1.])
-output = model.approx_nearest_neighbors(data_a, key, 2).select("id, distCol")
+output = model.approx_nearest_neighbors(data_a, key, 2).select(col("id"), col("distCol"))
 for result in t_env.to_data_stream(output).execute_and_collect():
     id_value = result[field_names.index("id")]
     dist_value = result[-1]
diff --git a/flink-ml-python/pyflink/ml/classification/tests/test_knn.py b/flink-ml-python/pyflink/ml/classification/tests/test_knn.py
index 32390952..44ce8d87 100644
--- a/flink-ml-python/pyflink/ml/classification/tests/test_knn.py
+++ b/flink-ml-python/pyflink/ml/classification/tests/test_knn.py
@@ -89,8 +89,8 @@ class KNNTest(PyFlinkMLTestCase):
             .set_k(4) \
             .set_prediction_col('test_prediction')
 
-        model = knn.fit(self.train_data.alias('test_features, test_label'))
-        output = model.transform(self.predict_data.alias('test_features, test_label'))[0]
+        model = knn.fit(self.train_data.alias('test_features', 'test_label'))
+        output = model.transform(self.predict_data.alias('test_features', 'test_label'))[0]
         self.assertEqual(output.get_schema().get_field_names(),
                          ['test_features',
                           'test_label',
diff --git a/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py b/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py
index 5463a1f7..9fed97c9 100644
--- a/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py
+++ b/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py
@@ -26,6 +26,7 @@ from pyflink.ml.wrapper import JavaWithParams
 from pyflink.ml.feature.lsh import MinHashLSH, MinHashLSHModel
 from pyflink.ml.tests.test_utils import PyFlinkMLTestCase
 from pyflink.table import Table
+from pyflink.table.expressions import col
 
 
 class MinHashLSHTest(PyFlinkMLTestCase):
@@ -103,7 +104,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
             .set_num_hash_tables(5) \
             .set_num_hash_functions_per_table(3)
         model = lsh.fit(self.data)
-        output = model.transform(self.data)[0].select("hashes")
+        output = model.transform(self.data)[0].select(col("hashes"))
         self.verify_output_hashes(output, self.expected)
 
     def test_estimator_save_load_transform(self):
@@ -117,7 +118,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
         lsh.save(path)
         lsh = MinHashLSH.load(self.t_env, path)
         model = lsh.fit(self.data)
-        output = model.transform(self.data)[0].select(lsh.output_col)
+        output = model.transform(self.data)[0].select(col(lsh.output_col))
         self.verify_output_hashes(output, self.expected)
 
     def test_model_save_load_transform(self):
@@ -132,7 +133,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
         model.save(path)
         self.env.execute('save_model')
         model = MinHashLSHModel.load(self.t_env, path)
-        output = model.transform(self.data)[0].select(lsh.output_col)
+        output = model.transform(self.data)[0].select(col(lsh.output_col))
         self.verify_output_hashes(output, self.expected)
 
     def test_get_model_data(self):
@@ -167,7 +168,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
         model_data_table = model_a.get_model_data()[0]
         model_b: MinHashLSHModel = MinHashLSHModel().set_model_data(model_data_table)
         self.update_existing_params(model_b, model_a)
-        output = model_b.transform(self.data)[0].select(lsh.output_col)
+        output = model_b.transform(self.data)[0].select(col(lsh.output_col))
         self.verify_output_hashes(output, self.expected)
 
     def test_approx_nearest_neighbors(self):
@@ -184,7 +185,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
 
         model: MinHashLSHModel = lsh.fit(self.data)
         key = Vectors.sparse(6, [1, 3], [1., 1.])
-        output = model.approx_nearest_neighbors(self.data, key, 2).select("id, distCol")
+        output = model.approx_nearest_neighbors(self.data, key, 2).select(col("id"), col("distCol"))
         actual_result = [r for r in self.t_env.to_data_stream(output).execute_and_collect()]
         actual_result.sort(key=lambda r: r[0])
         self.assertEqual(expected, actual_result)
@@ -203,7 +204,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
 
         model: MinHashLSHModel = lsh.fit(self.data)
         key = Vectors.dense([0., 1., 0., 1., 0., 0.])
-        output = model.approx_nearest_neighbors(self.data, key, 2).select("id, distCol")
+        output = model.approx_nearest_neighbors(self.data, key, 2).select(col("id"), col("distCol"))
         actual_result = [r for r in self.t_env.to_data_stream(output).execute_and_collect()]
         actual_result.sort(key=lambda r: r[0])
         self.assertEqual(expected, actual_result)
diff --git a/flink-ml-python/pyflink/ml/linalg.py b/flink-ml-python/pyflink/ml/linalg.py
index 739d5593..46dd3464 100644
--- a/flink-ml-python/pyflink/ml/linalg.py
+++ b/flink-ml-python/pyflink/ml/linalg.py
@@ -506,12 +506,12 @@ class SparseVector(Vector):
             cur_len = len(self._indices)
             indices = np.zeros(cur_len + 1, dtype=np.int32)
             values = np.zeros(cur_len + 1, dtype=np.float64)
-            indices[0:idx] = self._indices[0:idx]
-            values[0:idx] = self._values[0:idx]
+            indices[0:idx] = self._indices[0:idx]  # type: ignore
+            values[0:idx] = self._values[0:idx]  # type: ignore
             indices[idx] = i
             values[idx] = value
-            indices[idx + 1:] = self._indices[idx:]
-            values[idx + 1:] = self._values[idx]
+            indices[idx + 1:] = self._indices[idx:]  # type: ignore
+            values[idx + 1:] = self._values[idx]  # type: ignore
             self._indices = indices
             self._values = values
 
@@ -555,7 +555,7 @@ class SparseVector(Vector):
             self_cmind = np.in1d(self._indices, other._indices, assume_unique=True)
             self_values = self._values[self_cmind]
             if self_values.size == 0:
-                return np.float_(0.0)
+                return np.float_(0.0)  # type: ignore
             else:
                 other_cmind = np.in1d(other._indices, self._indices, assume_unique=True)
                 return np.dot(self_values, other._values[other_cmind])
@@ -623,7 +623,7 @@ class SparseVector(Vector):
             while j < len(other._indices):
                 result += other._values[j] * other._values[j]
                 j += 1
-            return np.float_(result)
+            return np.float_(result)  # type: ignore
         else:
             if isinstance(other, (array.array, np.ndarray, list, tuple, range)):
                 return self.squared_distance(DenseVector(other))
diff --git a/flink-ml-python/pyflink/ml/tests/test_pipeline.py b/flink-ml-python/pyflink/ml/tests/test_pipeline.py
index cd5dffda..57b0b983 100644
--- a/flink-ml-python/pyflink/ml/tests/test_pipeline.py
+++ b/flink-ml-python/pyflink/ml/tests/test_pipeline.py
@@ -18,6 +18,7 @@
 import os
 from typing import Dict, Any, List
 
+from pyflink.table.expressions import col
 from pyflink.table import Table, StreamTableEnvironment
 
 from pyflink.ml.api import Model
@@ -80,7 +81,7 @@ class PipelineTest(PyFlinkMLTestCase):
 class Add10Model(Model):
     def transform(self, *inputs: Table) -> List[Table]:
         assert len(inputs) == 1
-        return [inputs[0].select("a + 10 as a")]
+        return [inputs[0].select(col("a") + 10).alias("a")]
 
     def save(self, path: str) -> None:
         from pyflink.ml.util import read_write_utils
diff --git a/flink-ml-python/setup.py b/flink-ml-python/setup.py
index 6c67c9fd..b263f5ed 100644
--- a/flink-ml-python/setup.py
+++ b/flink-ml-python/setup.py
@@ -119,8 +119,11 @@ try:
         author='Apache Software Foundation',
         author_email='dev@flink.apache.org',
         python_requires='>=3.6',
-        install_requires=['apache-flink==1.15.1', 'pandas>=1.0,<1.2.0', 'jsonpickle==2.0.0',
-                          'cloudpickle==1.2.2', 'numpy>=1.14.3,<1.20'],
+        install_requires=['apache-flink==1.16.1', 'jsonpickle==2.0.0', 'cloudpickle==2.1.0',
+                          'pandas>=1.0,<1.2.0; python_full_version < "3.7"',
+                          'pandas>=1.3.0,<1.4.0; python_full_version >= "3.7"',
+                          'numpy>=1.14.3,<1.20; python_full_version < "3.7"',
+                          'numpy>=1.21.4,<1.22.0; python_full_version >= "3.7"'],
         tests_require=['pytest==4.4.1'],
         description='Apache Flink ML Python API',
         long_description=long_description,
diff --git a/pom.xml b/pom.xml
index 0ef7f2e4..9bb29be9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,7 +77,7 @@ under the License.
     <junit.version>4.13.2</junit.version>
     <flink.forkCount>1C</flink.forkCount>
     <flink.reuseForks>true</flink.reuseForks>
-    <flink.version>1.15.1</flink.version>
+    <flink.version>1.16.1</flink.version>
     <zookeeper.version>3.6.3</zookeeper.version>
     <hadoop.version>2.10.1</hadoop.version>