You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by li...@apache.org on 2023/04/25 06:18:39 UTC
[flink-ml] branch master updated: [FLINK-31887] Upgrade Flink version of Flink ML to 1.16.1
This is an automated email from the ASF dual-hosted git repository.
lindong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-ml.git
The following commit(s) were added to refs/heads/master by this push:
new 5b193dee [FLINK-31887] Upgrade Flink version of Flink ML to 1.16.1
5b193dee is described below
commit 5b193deedff2aa2be96fbfb5304f812caace9e12
Author: JiangXin <ji...@gmail.com>
AuthorDate: Tue Apr 25 14:18:33 2023 +0800
[FLINK-31887] Upgrade Flink version of Flink ML to 1.16.1
This closes #234.
---
docs/content/docs/operators/feature/minhashlsh.md | 3 +-
.../try-flink-ml/java/build-your-own-project.md | 2 +-
docs/content/docs/try-flink-ml/java/quick-start.md | 2 +-
.../docs/try-flink-ml/python/quick-start.md | 2 +-
flink-ml-benchmark/README.md | 2 +-
.../flink/iteration/operator/OperatorUtils.java | 55 +++++++++++++---------
.../coordinator/HeadOperatorCoordinator.java | 31 ++++++------
.../perround/AbstractPerRoundWrapperOperator.java | 17 +++++--
.../proxy/state/ProxyKeyedStateBackend.java | 4 +-
.../flink/iteration/operator/HeadOperatorTest.java | 6 ++-
.../coordinator/HeadOperatorCoordinatorTest.java | 9 ++--
.../apache/flink/ml/classification/KnnTest.java | 4 +-
.../org/apache/flink/ml/feature/BinarizerTest.java | 7 ++-
.../apache/flink/ml/feature/BucketizerTest.java | 3 +-
.../flink/ml/feature/CountVectorizerTest.java | 3 +-
.../org/apache/flink/ml/feature/ImputerTest.java | 3 +-
.../apache/flink/ml/feature/InteractionTest.java | 3 +-
.../flink/ml/feature/KBinsDiscretizerTest.java | 3 +-
.../apache/flink/ml/feature/MaxAbsScalerTest.java | 4 +-
.../apache/flink/ml/feature/MinHashLSHTest.java | 8 ++--
.../apache/flink/ml/feature/MinMaxScalerTest.java | 3 +-
.../apache/flink/ml/feature/NormalizerTest.java | 3 +-
.../flink/ml/feature/PolynomialExpansionTest.java | 3 +-
.../flink/ml/feature/RandomSplitterTest.java | 3 +-
.../apache/flink/ml/feature/RobustScalerTest.java | 3 +-
.../ml/feature/VarianceThresholdSelectorTest.java | 3 +-
.../apache/flink/ml/feature/VectorIndexerTest.java | 3 +-
.../feature/stringindexer/StringIndexerTest.java | 3 +-
.../org/apache/flink/ml/stats/ChiSqTestTest.java | 3 +-
flink-ml-python/dev/dev-requirements.txt | 10 ++--
flink-ml-python/pom.xml | 2 +-
.../examples/ml/feature/minhashlsh_example.py | 3 +-
.../pyflink/ml/classification/tests/test_knn.py | 4 +-
.../pyflink/ml/feature/tests/test_minhashlsh.py | 13 ++---
flink-ml-python/pyflink/ml/linalg.py | 12 ++---
flink-ml-python/pyflink/ml/tests/test_pipeline.py | 3 +-
flink-ml-python/setup.py | 7 ++-
pom.xml | 2 +-
38 files changed, 153 insertions(+), 101 deletions(-)
diff --git a/docs/content/docs/operators/feature/minhashlsh.md b/docs/content/docs/operators/feature/minhashlsh.md
index 5c3aee92..22ce0b88 100644
--- a/docs/content/docs/operators/feature/minhashlsh.md
+++ b/docs/content/docs/operators/feature/minhashlsh.md
@@ -221,6 +221,7 @@ public class MinHashLSHExample {
from pyflink.common import Types
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.table import StreamTableEnvironment
+from pyflink.table.expressions import col
from pyflink.ml.linalg import Vectors, SparseVectorTypeInfo
from pyflink.ml.feature.lsh import MinHashLSH
@@ -268,7 +269,7 @@ for result in t_env.to_data_stream(output).execute_and_collect():
# Finds approximate nearest neighbors of the key.
key = Vectors.sparse(6, [1, 3], [1., 1.])
-output = model.approx_nearest_neighbors(data_a, key, 2).select("id, distCol")
+output = model.approx_nearest_neighbors(data_a, key, 2).select(col("id"), col("distCol"))
for result in t_env.to_data_stream(output).execute_and_collect():
id_value = result[field_names.index("id")]
dist_value = result[-1]
diff --git a/docs/content/docs/try-flink-ml/java/build-your-own-project.md b/docs/content/docs/try-flink-ml/java/build-your-own-project.md
index 713bb924..adebe30e 100644
--- a/docs/content/docs/try-flink-ml/java/build-your-own-project.md
+++ b/docs/content/docs/try-flink-ml/java/build-your-own-project.md
@@ -80,7 +80,7 @@ the basic skeleton of a project, with some necessary Flink dependencies.
$ mvn archetype:generate \
-DarchetypeGroupId=org.apache.flink \
-DarchetypeArtifactId=flink-quickstart-java \
- -DarchetypeVersion=1.15.1 \
+ -DarchetypeVersion=1.16.1 \
-DgroupId=kmeans-example \
-DartifactId=kmeans-example \
-Dversion=0.1 \
diff --git a/docs/content/docs/try-flink-ml/java/quick-start.md b/docs/content/docs/try-flink-ml/java/quick-start.md
index 725df28d..f92cb71b 100644
--- a/docs/content/docs/try-flink-ml/java/quick-start.md
+++ b/docs/content/docs/try-flink-ml/java/quick-start.md
@@ -50,7 +50,7 @@ $ java -version
## Download Flink
-[Download 1.15 or a higher version of
+[Download 1.16 or a higher version of
Flink](https://flink.apache.org/downloads.html), then extract the archive:
```shell
diff --git a/docs/content/docs/try-flink-ml/python/quick-start.md b/docs/content/docs/try-flink-ml/python/quick-start.md
index d53ebc40..4ee84df0 100644
--- a/docs/content/docs/try-flink-ml/python/quick-start.md
+++ b/docs/content/docs/try-flink-ml/python/quick-start.md
@@ -248,7 +248,7 @@ $ java -version
### Download Flink
-[Download 1.15 or a higher version of
+[Download 1.16 or a higher version of
Flink](https://flink.apache.org/downloads.html), then extract the archive:
```shell
diff --git a/flink-ml-benchmark/README.md b/flink-ml-benchmark/README.md
index 69e49397..b46244ee 100644
--- a/flink-ml-benchmark/README.md
+++ b/flink-ml-benchmark/README.md
@@ -7,7 +7,7 @@ stages in a Linux/MacOS environment.
### Install Flink
-Please make sure Flink 1.15 or higher version has been installed in your local
+Please make sure Flink 1.16 or higher version has been installed in your local
environment. You can refer to the [local
installation](https://nightlies.apache.org/flink/flink-docs-master/docs/try-flink/local_installation/)
instruction on Flink's document website for how to achieve this.
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java
index 3d67b7fa..7b81e524 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/OperatorUtils.java
@@ -47,7 +47,6 @@ import java.util.Arrays;
import java.util.Random;
import java.util.UUID;
import java.util.concurrent.Executor;
-import java.util.stream.Stream;
import static org.apache.flink.util.Preconditions.checkState;
@@ -136,31 +135,20 @@ public class OperatorUtils {
wrappedConfig.setTypeSerializerOut(
((IterationRecordSerializer<?>) typeSerializerOut).getInnerSerializer());
- Stream.concat(
- config.getChainedOutputs(cl).stream(),
- config.getNonChainedOutputs(cl).stream())
+ config.getChainedOutputs(cl)
.forEach(
- edge -> {
- OutputTag<?> outputTag = edge.getOutputTag();
- if (outputTag != null) {
- TypeSerializer<?> typeSerializerSideOut =
- config.getTypeSerializerSideOut(outputTag, cl);
- checkState(
- typeSerializerSideOut instanceof IterationRecordSerializer,
- "The serializer of side output with tag[%s] should be IterationRecordSerializer but it is %s.",
- outputTag,
- typeSerializerSideOut);
- wrappedConfig.setTypeSerializerSideOut(
- new OutputTag<>(
- outputTag.getId(),
- ((IterationRecordTypeInfo<?>)
- outputTag.getTypeInfo())
- .getInnerTypeInfo()),
- ((IterationRecordSerializer) typeSerializerSideOut)
- .getInnerSerializer());
- }
+ chainedOutput -> {
+ OutputTag<?> outputTag = chainedOutput.getOutputTag();
+ setTypeSerializerSideOut(outputTag, config, wrappedConfig, cl);
+ });
+ config.getOperatorNonChainedOutputs(cl)
+ .forEach(
+ nonChainedOutput -> {
+ OutputTag<?> outputTag = nonChainedOutput.getOutputTag();
+ setTypeSerializerSideOut(outputTag, config, wrappedConfig, cl);
});
+ wrappedConfig.serializeAllConfigs();
return wrappedConfig;
}
@@ -185,4 +173,25 @@ public class OperatorUtils {
operatorId,
UUID.randomUUID().toString()));
}
+
+ private static void setTypeSerializerSideOut(
+ OutputTag<?> outputTag,
+ StreamConfig config,
+ StreamConfig wrappedConfig,
+ ClassLoader cl) {
+ if (outputTag == null) {
+ return;
+ }
+ TypeSerializer<?> typeSerializerSideOut = config.getTypeSerializerSideOut(outputTag, cl);
+ checkState(
+ typeSerializerSideOut instanceof IterationRecordSerializer,
+ "The serializer of side output with tag[%s] should be IterationRecordSerializer but it is %s.",
+ outputTag,
+ typeSerializerSideOut);
+ wrappedConfig.setTypeSerializerSideOut(
+ new OutputTag<>(
+ outputTag.getId(),
+ ((IterationRecordTypeInfo<?>) outputTag.getTypeInfo()).getInnerTypeInfo()),
+ ((IterationRecordSerializer) typeSerializerSideOut).getInnerSerializer());
+ }
}
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java
index 1b6b4d9c..2494c5f7 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinator.java
@@ -58,11 +58,6 @@ public class HeadOperatorCoordinator implements OperatorCoordinator, SharedProgr
@Override
public void start() {}
- @Override
- public void subtaskReady(int subtaskIndex, SubtaskGateway subtaskGateway) {
- this.subtaskGateways[subtaskIndex] = subtaskGateway;
- }
-
@Override
public void resetToCheckpoint(long checkpointId, @Nullable byte[] bytes) {
for (int i = 0; i < context.currentParallelism(); ++i) {
@@ -71,19 +66,15 @@ public class HeadOperatorCoordinator implements OperatorCoordinator, SharedProgr
}
@Override
- public void subtaskFailed(int subtaskIndex, @Nullable Throwable throwable) {
- sharedProgressAligner.removeProgressInfo(context.getOperatorId(), subtaskIndex);
- }
-
- @Override
- public void handleEventFromOperator(int subtaskIndex, OperatorEvent operatorEvent) {
- if (operatorEvent instanceof SubtaskAlignedEvent) {
+ public void handleEventFromOperator(int subtask, int attemptNumber, OperatorEvent event)
+ throws Exception {
+ if (event instanceof SubtaskAlignedEvent) {
sharedProgressAligner.reportSubtaskProgress(
- context.getOperatorId(), subtaskIndex, (SubtaskAlignedEvent) operatorEvent);
- } else if (operatorEvent instanceof TerminatingOnInitializeEvent) {
+ context.getOperatorId(), subtask, (SubtaskAlignedEvent) event);
+ } else if (event instanceof TerminatingOnInitializeEvent) {
sharedProgressAligner.notifyGloballyTerminating();
} else {
- throw new UnsupportedOperationException("Not supported event: " + operatorEvent);
+ throw new UnsupportedOperationException("Not supported event: " + event);
}
}
@@ -116,6 +107,16 @@ public class HeadOperatorCoordinator implements OperatorCoordinator, SharedProgr
@Override
public void subtaskReset(int i, long l) {}
+ @Override
+ public void executionAttemptFailed(int subtask, int attemptNumber, @Nullable Throwable reason) {
+ sharedProgressAligner.removeProgressInfo(context.getOperatorId(), subtask);
+ }
+
+ @Override
+ public void executionAttemptReady(int subtask, int attemptNumber, SubtaskGateway gateway) {
+ this.subtaskGateways[subtask] = gateway;
+ }
+
/** The factory of {@link HeadOperatorCoordinator}. */
public static class HeadOperatorCoordinatorProvider implements Provider {
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java
index 4ae88858..d6728764 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/operator/perround/AbstractPerRoundWrapperOperator.java
@@ -610,6 +610,10 @@ public abstract class AbstractPerRoundWrapperOperator<T, S extends StreamOperato
keyedStateBackend, HeapKeyedStateBackend.class, "registeredKVStates")
.entrySet()
.removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+ ReflectionUtils.<Map<String, ?>>getFieldValue(
+ keyedStateBackend, HeapKeyedStateBackend.class, "createdKVStates")
+ .entrySet()
+ .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
ReflectionUtils.<Map<String, ?>>getFieldValue(
keyedStateBackend,
AbstractKeyedStateBackend.class,
@@ -640,12 +644,17 @@ public abstract class AbstractPerRoundWrapperOperator<T, S extends StreamOperato
});
kvStateInformation.entrySet().removeIf(entry -> entry.getKey().startsWith(roundPrefix));
- Map<String, ?> field =
- ReflectionUtils.getFieldValue(
+ ReflectionUtils.<Map<String, ?>>getFieldValue(
+ keyedStateBackend, RocksDBKeyedStateBackend.class, "createdKVStates")
+ .entrySet()
+ .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+ ReflectionUtils.<Map<String, ?>>getFieldValue(
keyedStateBackend,
AbstractKeyedStateBackend.class,
- "keyValueStatesByName");
- field.entrySet().removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+ "keyValueStatesByName")
+ .entrySet()
+ .removeIf(entry -> entry.getKey().startsWith(roundPrefix));
+
} else {
LOG.warn("Unable to cleanup the keyed state {}", keyedStateBackend);
}
diff --git a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java
index bc51f957..0fb13d09 100644
--- a/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java
+++ b/flink-ml-iteration/src/main/java/org/apache/flink/iteration/proxy/state/ProxyKeyedStateBackend.java
@@ -127,7 +127,7 @@ public class ProxyKeyedStateBackend<K> implements CheckpointableKeyedStateBacken
@Nonnull
@Override
- public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
+ public <N, SV, SEV, S extends State, IS extends S> IS createOrUpdateInternalState(
@Nonnull TypeSerializer<N> namespaceSerializer,
@Nonnull StateDescriptor<S, SV> stateDesc,
@Nonnull
@@ -135,7 +135,7 @@ public class ProxyKeyedStateBackend<K> implements CheckpointableKeyedStateBacken
snapshotTransformFactory)
throws Exception {
StateDescriptor<S, ?> newDescriptor = createNewDescriptor(stateDesc);
- return wrappedBackend.createInternalState(
+ return wrappedBackend.createOrUpdateInternalState(
namespaceSerializer, newDescriptor, snapshotTransformFactory);
}
diff --git a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java
index 17778933..fc9bd6b4 100644
--- a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java
+++ b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/HeadOperatorTest.java
@@ -824,7 +824,11 @@ public class HeadOperatorTest extends TestLogger {
RecordingOperatorEventGateway::new),
operatorId)
.build()) {
- return runnable.apply(harness);
+ try {
+ return runnable.apply(harness);
+ } finally {
+ RecordingHeadOperatorFactory.latestHeadOperator.close();
+ }
}
}
diff --git a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java
index b77d2727..63d1e085 100644
--- a/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java
+++ b/flink-ml-iteration/src/test/java/org/apache/flink/iteration/operator/coordinator/HeadOperatorCoordinatorTest.java
@@ -41,7 +41,7 @@ import static org.junit.Assert.assertEquals;
public class HeadOperatorCoordinatorTest extends TestLogger {
@Test(timeout = 60000L)
- public void testForwardEvents() throws InterruptedException {
+ public void testForwardEvents() throws Exception {
IterationID iterationId = new IterationID();
List<OperatorID> operatorIds = Arrays.asList(new OperatorID(), new OperatorID());
List<Integer> parallelisms = Arrays.asList(2, 3);
@@ -88,19 +88,20 @@ public class HeadOperatorCoordinatorTest extends TestLogger {
EventReceivingTasks receivingTasks,
int parallelism) {
for (int i = 0; i < parallelism; i++) {
- coordinator.subtaskReady(i, receivingTasks.createGatewayForSubtask(i));
+ coordinator.executionAttemptReady(i, 0, receivingTasks.createGatewayForSubtask(i, 0));
}
}
private void receiveEvent(
List<HeadOperatorCoordinator> coordinators,
List<Integer> parallelisms,
- BiFunction<Integer, Integer, List<OperatorEvent>> eventFactory) {
+ BiFunction<Integer, Integer, List<OperatorEvent>> eventFactory)
+ throws Exception {
for (int i = 0; i < coordinators.size(); ++i) {
for (int j = 0; j < parallelisms.get(i); ++j) {
List<OperatorEvent> events = eventFactory.apply(i, j);
for (OperatorEvent event : events) {
- coordinators.get(i).handleEventFromOperator(j, event);
+ coordinators.get(i).handleEventFromOperator(j, 0, event);
}
}
}
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java
index 483dc34d..a44ae5d4 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/classification/KnnTest.java
@@ -151,8 +151,8 @@ public class KnnTest extends AbstractTestBase {
.setFeaturesCol("test_features")
.setK(4)
.setPredictionCol("test_prediction");
- KnnModel model = knn.fit(trainData.as("test_features, test_label"));
- Table output = model.transform(predictData.as("test_features, test_label"))[0];
+ KnnModel model = knn.fit(trainData.as("test_features", "test_label"));
+ Table output = model.transform(predictData.as("test_features", "test_label"))[0];
assertEquals(
Arrays.asList("test_features", "test_label", "test_prediction"),
output.getResolvedSchema().getColumnNames());
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java
index eb06b0c6..86341768 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BinarizerTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -99,8 +100,10 @@ public class BinarizerTest extends AbstractTestBase {
}
doubleValues.sort(Double::compare);
assertArrayEquals(EXPECTED_VALUE_OUTPUT, doubleValues.toArray());
- compareResultCollections(EXPECTED_DENSE_OUTPUT, denseVectorValues, TestUtils::compare);
- compareResultCollections(EXPECTED_SPARSE_OUTPUT, sparseVectorValues, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(
+ EXPECTED_DENSE_OUTPUT, denseVectorValues, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(
+ EXPECTED_SPARSE_OUTPUT, sparseVectorValues, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java
index 2c49dfa6..66adde8e 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/BucketizerTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -97,7 +98,7 @@ public class BucketizerTest extends AbstractTestBase {
result.add(outRow);
}
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expectedResult, result, Comparator.comparingInt(r -> ((Integer) r.getField(0))));
}
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java
index 32d58ccc..531a4eb0 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/CountVectorizerTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -110,7 +111,7 @@ public class CountVectorizerTest extends AbstractTestBase {
(MapFunction<Row, SparseVector>)
row -> (SparseVector) row.getField(outputCol));
List<SparseVector> result = IteratorUtils.toList(stream.executeAndCollect());
- compareResultCollections(expected, result, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
index d8805a70..206695cd 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -138,7 +139,7 @@ public class ImputerTest extends AbstractTestBase {
return outputRow;
})
.collect(Collectors.toList());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expected,
result,
(row1, row2) -> {
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java
index 0b0cc995..dc108531 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/InteractionTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -107,7 +108,7 @@ public class InteractionTest extends AbstractTestBase {
resultVec.add(row.getFieldAs(outputCol));
}
}
- compareResultCollections(expectedData, resultVec, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expectedData, resultVec, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java
index e6959baa..4c70baf8 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/KBinsDiscretizerTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -133,7 +134,7 @@ public class KBinsDiscretizerTest extends AbstractTestBase {
List<Row> collectedResult =
IteratorUtils.toList(
tEnv.toDataStream(output.select($(predictionCol))).executeAndCollect());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expectedOutput,
collectedResult,
(o1, o2) ->
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java
index 6b8edf05..f7eeba7c 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MaxAbsScalerTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -46,7 +47,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import static org.apache.flink.test.util.TestBaseUtils.compareResultCollections;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -135,7 +135,7 @@ public class MaxAbsScalerTest {
VectorTypeInfo.INSTANCE);
List<Vector> result = IteratorUtils.toList(stream.executeAndCollect());
- compareResultCollections(expectedData, result, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expectedData, result, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
index d92f062d..862c190d 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
@@ -35,6 +35,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -103,7 +104,7 @@ public class MinHashLSHTest extends AbstractTestBase {
StreamTableEnvironment tEnv =
(StreamTableEnvironment) ((TableImpl) output).getTableEnvironment();
List<Row> results = IteratorUtils.toList(tEnv.toDataStream(output).executeAndCollect());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expected,
results,
(d0, d1) -> {
@@ -379,7 +380,8 @@ public class MinHashLSHTest extends AbstractTestBase {
Table output =
lshModel.approxNearestNeighbors(inputTable, key, 2).select($("id"), $("distCol"));
List<Row> results = IteratorUtils.toList(output.execute().collect());
- compareResultCollections(expected, results, Comparator.comparing(r -> r.getFieldAs(0)));
+ TestBaseUtils.compareResultCollections(
+ expected, results, Comparator.comparing(r -> r.getFieldAs(0)));
}
@Test
@@ -418,7 +420,7 @@ public class MinHashLSHTest extends AbstractTestBase {
Table output = lshModel.approxSimilarityJoin(dataA, dataB, .6, "id");
List<Row> results = IteratorUtils.toList(output.execute().collect());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expected,
results,
Comparator.<Row>comparingInt(r -> r.getFieldAs(0))
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java
index 32451635..8481b4f3 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinMaxScalerTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -95,7 +96,7 @@ public class MinMaxScalerTest extends AbstractTestBase {
(MapFunction<Row, DenseVector>)
row -> (DenseVector) row.getField(outputCol));
List<DenseVector> result = IteratorUtils.toList(stream.executeAndCollect());
- compareResultCollections(expected, result, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java
index 13e28c68..cf8c5c61 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/NormalizerTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -109,7 +110,7 @@ public class NormalizerTest extends AbstractTestBase {
resultVec.add(row.getFieldAs(outputCol));
}
}
- compareResultCollections(expectedData, resultVec, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expectedData, resultVec, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java
index 5454eabf..be6fa7dc 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/PolynomialExpansionTest.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -100,7 +101,7 @@ public class PolynomialExpansionTest extends AbstractTestBase {
resultVec.add(row.getFieldAs(outputCol));
}
}
- compareResultCollections(expectedData, resultVec, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expectedData, resultVec, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java
index 75ad42ab..4c482b7f 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RandomSplitterTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -152,7 +153,7 @@ public class RandomSplitterTest extends AbstractTestBase {
results.addAll(result);
}
assertEquals(expectedData.size(), results.size());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expectedData, results, Comparator.comparingLong(row -> row.getFieldAs(0)));
}
}
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
index e8179024..2e58b4d1 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -104,7 +105,7 @@ public class RobustScalerTest extends AbstractTestBase {
(MapFunction<Row, DenseVector>)
row -> (DenseVector) row.getField(outputCol));
List<DenseVector> result = IteratorUtils.toList(stream.executeAndCollect());
- compareResultCollections(expected, result, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
index 43a893ce..194b247d 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
@@ -32,6 +32,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -107,7 +108,7 @@ public class VarianceThresholdSelectorTest extends AbstractTestBase {
(MapFunction<Row, Vector>) row -> (Vector) row.getField(outputCol),
VectorTypeInfo.INSTANCE);
List<Vector> result = IteratorUtils.toList(stream.executeAndCollect());
- compareResultCollections(expected, result, TestUtils::compare);
+ TestBaseUtils.compareResultCollections(expected, result, TestUtils::compare);
}
@Test
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java
index de69f4d5..5d01de29 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VectorIndexerTest.java
@@ -30,6 +30,7 @@ import org.apache.flink.table.api.Expressions;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -277,7 +278,7 @@ public class VectorIndexerTest extends AbstractTestBase {
IteratorUtils.toList(
tEnv.toDataStream(output.select(Expressions.$(outputCol)))
.executeAndCollect());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expectedOutput,
collectedResult,
Comparator.comparingInt(o -> (o.getField(0)).hashCode()));
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java
index 9f4a5ae8..6105a3b0 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/stringindexer/StringIndexerTest.java
@@ -25,6 +25,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -421,7 +422,7 @@ public class StringIndexerTest extends AbstractTestBase {
}
static void verifyPredictionResult(List<Row> expected, List<Row> result) {
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expected,
result,
(row1, row2) -> {
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java
index 87a3f376..e1dbd0d3 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/stats/ChiSqTestTest.java
@@ -27,6 +27,7 @@ import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.api.internal.TableImpl;
import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
import org.apache.commons.collections.IteratorUtils;
@@ -120,7 +121,7 @@ public class ChiSqTestTest extends AbstractTestBase {
List<Row> result = IteratorUtils.toList(outputDataStream.executeAndCollect());
- compareResultCollections(
+ TestBaseUtils.compareResultCollections(
expected,
result,
(row1, row2) -> {
diff --git a/flink-ml-python/dev/dev-requirements.txt b/flink-ml-python/dev/dev-requirements.txt
index 68315a66..b8ac56c8 100755
--- a/flink-ml-python/dev/dev-requirements.txt
+++ b/flink-ml-python/dev/dev-requirements.txt
@@ -14,11 +14,13 @@
# limitations under the License.
setuptools>=18.0
wheel
-apache-flink==1.15.1
-pandas>=1.0,<1.2.0
+apache-flink==1.16.1
+pandas>=1.3.0,<1.4.0; python_version >= '3.7'
+pandas>=1.0,<1.2.0; python_version < '3.7'
jsonpickle==2.0.0
-cloudpickle==1.2.2
+cloudpickle==2.1.0
pytest==4.4.1
flake8==4.0.1
mypy==0.910
-numpy>=1.14.3,<1.20
\ No newline at end of file
+numpy>=1.21.4,<1.22.0; python_version >= '3.7'
+numpy>=1.14.3,<1.20; python_version < '3.7'
\ No newline at end of file
diff --git a/flink-ml-python/pom.xml b/flink-ml-python/pom.xml
index 5a24c2f2..3e442fec 100644
--- a/flink-ml-python/pom.xml
+++ b/flink-ml-python/pom.xml
@@ -39,7 +39,7 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-python_2.12</artifactId>
+ <artifactId>flink-python</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py b/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py
index 29b1a4d0..a054f232 100644
--- a/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py
+++ b/flink-ml-python/pyflink/examples/ml/feature/minhashlsh_example.py
@@ -22,6 +22,7 @@
from pyflink.common import Types
from pyflink.datastream import StreamExecutionEnvironment
from pyflink.table import StreamTableEnvironment
+from pyflink.table.expressions import col
from pyflink.ml.linalg import Vectors, SparseVectorTypeInfo
from pyflink.ml.feature.lsh import MinHashLSH
@@ -69,7 +70,7 @@ for result in t_env.to_data_stream(output).execute_and_collect():
# Finds approximate nearest neighbors of the key.
key = Vectors.sparse(6, [1, 3], [1., 1.])
-output = model.approx_nearest_neighbors(data_a, key, 2).select("id, distCol")
+output = model.approx_nearest_neighbors(data_a, key, 2).select(col("id"), col("distCol"))
for result in t_env.to_data_stream(output).execute_and_collect():
id_value = result[field_names.index("id")]
dist_value = result[-1]
diff --git a/flink-ml-python/pyflink/ml/classification/tests/test_knn.py b/flink-ml-python/pyflink/ml/classification/tests/test_knn.py
index 32390952..44ce8d87 100644
--- a/flink-ml-python/pyflink/ml/classification/tests/test_knn.py
+++ b/flink-ml-python/pyflink/ml/classification/tests/test_knn.py
@@ -89,8 +89,8 @@ class KNNTest(PyFlinkMLTestCase):
.set_k(4) \
.set_prediction_col('test_prediction')
- model = knn.fit(self.train_data.alias('test_features, test_label'))
- output = model.transform(self.predict_data.alias('test_features, test_label'))[0]
+ model = knn.fit(self.train_data.alias('test_features', 'test_label'))
+ output = model.transform(self.predict_data.alias('test_features', 'test_label'))[0]
self.assertEqual(output.get_schema().get_field_names(),
['test_features',
'test_label',
diff --git a/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py b/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py
index 5463a1f7..9fed97c9 100644
--- a/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py
+++ b/flink-ml-python/pyflink/ml/feature/tests/test_minhashlsh.py
@@ -26,6 +26,7 @@ from pyflink.ml.wrapper import JavaWithParams
from pyflink.ml.feature.lsh import MinHashLSH, MinHashLSHModel
from pyflink.ml.tests.test_utils import PyFlinkMLTestCase
from pyflink.table import Table
+from pyflink.table.expressions import col
class MinHashLSHTest(PyFlinkMLTestCase):
@@ -103,7 +104,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
.set_num_hash_tables(5) \
.set_num_hash_functions_per_table(3)
model = lsh.fit(self.data)
- output = model.transform(self.data)[0].select("hashes")
+ output = model.transform(self.data)[0].select(col("hashes"))
self.verify_output_hashes(output, self.expected)
def test_estimator_save_load_transform(self):
@@ -117,7 +118,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
lsh.save(path)
lsh = MinHashLSH.load(self.t_env, path)
model = lsh.fit(self.data)
- output = model.transform(self.data)[0].select(lsh.output_col)
+ output = model.transform(self.data)[0].select(col(lsh.output_col))
self.verify_output_hashes(output, self.expected)
def test_model_save_load_transform(self):
@@ -132,7 +133,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
model.save(path)
self.env.execute('save_model')
model = MinHashLSHModel.load(self.t_env, path)
- output = model.transform(self.data)[0].select(lsh.output_col)
+ output = model.transform(self.data)[0].select(col(lsh.output_col))
self.verify_output_hashes(output, self.expected)
def test_get_model_data(self):
@@ -167,7 +168,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
model_data_table = model_a.get_model_data()[0]
model_b: MinHashLSHModel = MinHashLSHModel().set_model_data(model_data_table)
self.update_existing_params(model_b, model_a)
- output = model_b.transform(self.data)[0].select(lsh.output_col)
+ output = model_b.transform(self.data)[0].select(col(lsh.output_col))
self.verify_output_hashes(output, self.expected)
def test_approx_nearest_neighbors(self):
@@ -184,7 +185,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
model: MinHashLSHModel = lsh.fit(self.data)
key = Vectors.sparse(6, [1, 3], [1., 1.])
- output = model.approx_nearest_neighbors(self.data, key, 2).select("id, distCol")
+ output = model.approx_nearest_neighbors(self.data, key, 2).select(col("id"), col("distCol"))
actual_result = [r for r in self.t_env.to_data_stream(output).execute_and_collect()]
actual_result.sort(key=lambda r: r[0])
self.assertEqual(expected, actual_result)
@@ -203,7 +204,7 @@ class MinHashLSHTest(PyFlinkMLTestCase):
model: MinHashLSHModel = lsh.fit(self.data)
key = Vectors.dense([0., 1., 0., 1., 0., 0.])
- output = model.approx_nearest_neighbors(self.data, key, 2).select("id, distCol")
+ output = model.approx_nearest_neighbors(self.data, key, 2).select(col("id"), col("distCol"))
actual_result = [r for r in self.t_env.to_data_stream(output).execute_and_collect()]
actual_result.sort(key=lambda r: r[0])
self.assertEqual(expected, actual_result)
diff --git a/flink-ml-python/pyflink/ml/linalg.py b/flink-ml-python/pyflink/ml/linalg.py
index 739d5593..46dd3464 100644
--- a/flink-ml-python/pyflink/ml/linalg.py
+++ b/flink-ml-python/pyflink/ml/linalg.py
@@ -506,12 +506,12 @@ class SparseVector(Vector):
cur_len = len(self._indices)
indices = np.zeros(cur_len + 1, dtype=np.int32)
values = np.zeros(cur_len + 1, dtype=np.float64)
- indices[0:idx] = self._indices[0:idx]
- values[0:idx] = self._values[0:idx]
+ indices[0:idx] = self._indices[0:idx] # type: ignore
+ values[0:idx] = self._values[0:idx] # type: ignore
indices[idx] = i
values[idx] = value
- indices[idx + 1:] = self._indices[idx:]
- values[idx + 1:] = self._values[idx]
+ indices[idx + 1:] = self._indices[idx:] # type: ignore
+ values[idx + 1:] = self._values[idx] # type: ignore
self._indices = indices
self._values = values
@@ -555,7 +555,7 @@ class SparseVector(Vector):
self_cmind = np.in1d(self._indices, other._indices, assume_unique=True)
self_values = self._values[self_cmind]
if self_values.size == 0:
- return np.float_(0.0)
+ return np.float_(0.0) # type: ignore
else:
other_cmind = np.in1d(other._indices, self._indices, assume_unique=True)
return np.dot(self_values, other._values[other_cmind])
@@ -623,7 +623,7 @@ class SparseVector(Vector):
while j < len(other._indices):
result += other._values[j] * other._values[j]
j += 1
- return np.float_(result)
+ return np.float_(result) # type: ignore
else:
if isinstance(other, (array.array, np.ndarray, list, tuple, range)):
return self.squared_distance(DenseVector(other))
diff --git a/flink-ml-python/pyflink/ml/tests/test_pipeline.py b/flink-ml-python/pyflink/ml/tests/test_pipeline.py
index cd5dffda..57b0b983 100644
--- a/flink-ml-python/pyflink/ml/tests/test_pipeline.py
+++ b/flink-ml-python/pyflink/ml/tests/test_pipeline.py
@@ -18,6 +18,7 @@
import os
from typing import Dict, Any, List
+from pyflink.table.expressions import col
from pyflink.table import Table, StreamTableEnvironment
from pyflink.ml.api import Model
@@ -80,7 +81,7 @@ class PipelineTest(PyFlinkMLTestCase):
class Add10Model(Model):
def transform(self, *inputs: Table) -> List[Table]:
assert len(inputs) == 1
- return [inputs[0].select("a + 10 as a")]
+ return [inputs[0].select(col("a") + 10).alias("a")]
def save(self, path: str) -> None:
from pyflink.ml.util import read_write_utils
diff --git a/flink-ml-python/setup.py b/flink-ml-python/setup.py
index 6c67c9fd..b263f5ed 100644
--- a/flink-ml-python/setup.py
+++ b/flink-ml-python/setup.py
@@ -119,8 +119,11 @@ try:
author='Apache Software Foundation',
author_email='dev@flink.apache.org',
python_requires='>=3.6',
- install_requires=['apache-flink==1.15.1', 'pandas>=1.0,<1.2.0', 'jsonpickle==2.0.0',
- 'cloudpickle==1.2.2', 'numpy>=1.14.3,<1.20'],
+ install_requires=['apache-flink==1.16.1', 'jsonpickle==2.0.0', 'cloudpickle==2.1.0',
+ 'pandas>=1.0,<1.2.0; python_full_version < "3.7"',
+ 'pandas>=1.3.0,<1.4.0; python_full_version >= "3.7"',
+ 'numpy>=1.14.3,<1.20; python_full_version < "3.7"',
+ 'numpy>=1.21.4,<1.22.0; python_full_version >= "3.7"'],
tests_require=['pytest==4.4.1'],
description='Apache Flink ML Python API',
long_description=long_description,
diff --git a/pom.xml b/pom.xml
index 0ef7f2e4..9bb29be9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,7 +77,7 @@ under the License.
<junit.version>4.13.2</junit.version>
<flink.forkCount>1C</flink.forkCount>
<flink.reuseForks>true</flink.reuseForks>
- <flink.version>1.15.1</flink.version>
+ <flink.version>1.16.1</flink.version>
<zookeeper.version>3.6.3</zookeeper.version>
<hadoop.version>2.10.1</hadoop.version>