You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by li...@apache.org on 2023/02/09 05:46:10 UTC
[flink-ml] branch master updated: [FLINK-30688][followup] Disable Kryo fallback for tests in flink-ml-lib
This is an automated email from the ASF dual-hosted git repository.
lindong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-ml.git
The following commit(s) were added to refs/heads/master by this push:
new 19f0f31 [FLINK-30688][followup] Disable Kryo fallback for tests in flink-ml-lib
19f0f31 is described below
commit 19f0f31ca93f4967c99968b3609a11763bfc0afb
Author: JiangXin <ji...@alibaba-inc.com>
AuthorDate: Thu Feb 9 13:46:04 2023 +0800
[FLINK-30688][followup] Disable Kryo fallback for tests in flink-ml-lib
This closes #203.
---
.../ml/common/datastream/DataStreamUtils.java | 35 +++-
.../java/org/apache/flink/ml/linalg/Vector.java | 4 +
.../flink/ml/linalg/typeinfo/VectorTypeInfo.java | 2 +-
.../VectorTypeInfoFactory.java} | 37 ++--
.../ml/common/datastream/AllReduceImplTest.java | 29 +--
.../apache/flink/ml/common/window/WindowsTest.java | 23 ++-
.../naivebayes/NaiveBayesModelData.java | 2 +-
.../common/typeinfo/PriorityQueueSerializer.java | 200 +++++++++++++++++++++
.../ml/common/typeinfo/PriorityQueueTypeInfo.java | 116 ++++++++++++
.../typeinfo/QuantileSummaryTypeInfoFactory.java | 57 ++++++
.../flink/ml/common/util/QuantileSummary.java | 48 +++--
.../org/apache/flink/ml/feature/lsh/LSHModel.java | 61 +++++--
.../typeinfo/PriorityQueueSerializerTest.java | 79 ++++++++
.../QuantileSummaryTypeInfoFactoryTest.java | 77 ++++++++
.../org/apache/flink/ml/feature/ImputerTest.java | 11 +-
.../apache/flink/ml/feature/MinHashLSHTest.java | 11 +-
.../apache/flink/ml/feature/RobustScalerTest.java | 11 +-
.../ml/feature/UnivariateFeatureSelectorTest.java | 11 +-
.../ml/feature/VarianceThresholdSelectorTest.java | 11 +-
19 files changed, 696 insertions(+), 129 deletions(-)
diff --git a/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/DataStreamUtils.java b/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/DataStreamUtils.java
index 33cfd0c..691e770 100644
--- a/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/DataStreamUtils.java
+++ b/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/DataStreamUtils.java
@@ -334,18 +334,41 @@ public class DataStreamUtils {
* @param function The user defined process function.
* @return The data stream that is the result of applying the window function to each window.
*/
- @SuppressWarnings({"rawtypes", "unchecked"})
public static <IN, OUT, W extends Window> SingleOutputStreamOperator<OUT> windowAllAndProcess(
DataStream<IN> input, Windows windows, ProcessAllWindowFunction<IN, OUT, W> function) {
- AllWindowedStream<IN, W> allWindowedStream;
+ AllWindowedStream<IN, W> allWindowedStream = getAllWindowedStream(input, windows);
+ return allWindowedStream.process(function);
+ }
+
+ /**
+ * Creates windows from data in the non key grouped input stream and applies the given window
+ * function to each window.
+ *
+ * @param input The input data stream to be windowed and processed.
+ * @param windows The windowing strategy that defines how input data would be sliced into
+ * batches.
+ * @param function The user defined process function.
+ * @param outType The type information of the output.
+ * @return The data stream that is the result of applying the window function to each window.
+ */
+ public static <IN, OUT, W extends Window> SingleOutputStreamOperator<OUT> windowAllAndProcess(
+ DataStream<IN> input,
+ Windows windows,
+ ProcessAllWindowFunction<IN, OUT, W> function,
+ TypeInformation<OUT> outType) {
+ AllWindowedStream<IN, W> allWindowedStream = getAllWindowedStream(input, windows);
+ return allWindowedStream.process(function, outType);
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static <IN, W extends Window> AllWindowedStream<IN, W> getAllWindowedStream(
+ DataStream<IN> input, Windows windows) {
if (windows instanceof CountTumblingWindows) {
long countWindowSize = ((CountTumblingWindows) windows).getSize();
- allWindowedStream = (AllWindowedStream<IN, W>) input.countWindowAll(countWindowSize);
+ return (AllWindowedStream<IN, W>) input.countWindowAll(countWindowSize);
} else {
- allWindowedStream =
- input.windowAll((WindowAssigner) getDataStreamTimeWindowAssigner(windows));
+ return input.windowAll((WindowAssigner) getDataStreamTimeWindowAssigner(windows));
}
- return allWindowedStream.process(function);
}
private static WindowAssigner<Object, TimeWindow> getDataStreamTimeWindowAssigner(
diff --git a/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java b/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java
index 976e9a7..522e0bc 100644
--- a/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java
+++ b/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java
@@ -18,9 +18,13 @@
package org.apache.flink.ml.linalg;
+import org.apache.flink.api.common.typeinfo.TypeInfo;
+import org.apache.flink.ml.linalg.typeinfo.VectorTypeInfoFactory;
+
import java.io.Serializable;
/** A vector of double values. */
+@TypeInfo(VectorTypeInfoFactory.class)
public interface Vector extends Serializable {
/** Gets the size of the vector. */
diff --git a/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfo.java b/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfo.java
index 6825006..672dedf 100644
--- a/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfo.java
+++ b/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfo.java
@@ -29,7 +29,7 @@ public class VectorTypeInfo extends TypeInformation<Vector> {
public static final VectorTypeInfo INSTANCE = new VectorTypeInfo();
- private VectorTypeInfo() {}
+ public VectorTypeInfo() {}
@Override
public boolean isBasicType() {
diff --git a/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java b/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfoFactory.java
similarity index 55%
copy from flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java
copy to flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfoFactory.java
index 976e9a7..1931ea7 100644
--- a/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/Vector.java
+++ b/flink-ml-core/src/main/java/org/apache/flink/ml/linalg/typeinfo/VectorTypeInfoFactory.java
@@ -16,31 +16,22 @@
* limitations under the License.
*/
-package org.apache.flink.ml.linalg;
+package org.apache.flink.ml.linalg.typeinfo;
-import java.io.Serializable;
+import org.apache.flink.api.common.typeinfo.TypeInfoFactory;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.ml.linalg.Vector;
-/** A vector of double values. */
-public interface Vector extends Serializable {
+import java.lang.reflect.Type;
+import java.util.Map;
- /** Gets the size of the vector. */
- int size();
+/** Used by {@link TypeExtractor} to create a {@link TypeInformation} for {@link Vector}. */
+public class VectorTypeInfoFactory extends TypeInfoFactory<Vector> {
- /** Gets the value of the ith element. */
- double get(int i);
-
- /** Sets the value of the ith element. */
- void set(int i, double value);
-
- /** Converts the instance to a double array. */
- double[] toArray();
-
- /** Converts the instance to a dense vector. */
- DenseVector toDense();
-
- /** Converts the instance to a sparse vector. */
- SparseVector toSparse();
-
- /** Makes a deep copy of the vector. */
- Vector clone();
+ @Override
+ public TypeInformation<Vector> createTypeInfo(
+ Type t, Map<String, TypeInformation<?>> genericParameters) {
+ return new VectorTypeInfo();
+ }
}
diff --git a/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/AllReduceImplTest.java b/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/AllReduceImplTest.java
index 044ce8d..c231187 100644
--- a/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/AllReduceImplTest.java
+++ b/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/AllReduceImplTest.java
@@ -21,12 +21,14 @@ package org.apache.flink.ml.common.datastream;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.typeinfo.BasicTypeInfo;
import org.apache.flink.api.common.typeinfo.PrimitiveArrayTypeInfo;
+import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.NumberSequenceIterator;
+import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.runners.Enclosed;
import org.junit.runner.RunWith;
@@ -61,6 +63,12 @@ public class AllReduceImplTest {
public static class ParameterizedTest {
private static int numElements;
+ private StreamExecutionEnvironment env;
+
+ @Before
+ public void before() {
+ env = TestUtils.getExecutionEnvironment();
+ }
@Parameterized.Parameters
public static Collection<Object[]> params() {
@@ -79,9 +87,6 @@ public class AllReduceImplTest {
@Test
public void testAllReduce() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().enableObjectReuse();
- env.setParallelism(parallelism);
DataStream<double[]> elements =
env.fromParallelCollection(
new NumberSequenceIterator(1L, parallelism),
@@ -114,13 +119,16 @@ public class AllReduceImplTest {
/** Non-parameterized test for {@link AllReduceImpl}. */
public static class NonParameterizedTest {
+ private StreamExecutionEnvironment env;
+
+ @Before
+ public void before() {
+ env = TestUtils.getExecutionEnvironment();
+ }
+
@Test
public void testAllReduceWithMoreThanOneArray() {
try {
- StreamExecutionEnvironment env =
- StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().enableObjectReuse();
- env.setParallelism(parallelism);
DataStream<double[]> elements =
env.fromParallelCollection(
new NumberSequenceIterator(1L, parallelism),
@@ -148,10 +156,6 @@ public class AllReduceImplTest {
@Test
public void testAllReduceWithDifferentLength() {
try {
- StreamExecutionEnvironment env =
- StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().enableObjectReuse();
- env.setParallelism(parallelism);
DataStream<double[]> elements =
env.fromParallelCollection(
new NumberSequenceIterator(1L, parallelism),
@@ -170,9 +174,6 @@ public class AllReduceImplTest {
@Test
public void testAllReduceWithEmptyInput() throws Exception {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().enableObjectReuse();
- env.setParallelism(parallelism);
DataStream<double[]> elements =
env.fromParallelCollection(
new NumberSequenceIterator(1L, parallelism),
diff --git a/flink-ml-core/src/test/java/org/apache/flink/ml/common/window/WindowsTest.java b/flink-ml-core/src/test/java/org/apache/flink/ml/common/window/WindowsTest.java
index 9389eb5..507af90 100644
--- a/flink-ml-core/src/test/java/org/apache/flink/ml/common/window/WindowsTest.java
+++ b/flink-ml-core/src/test/java/org/apache/flink/ml/common/window/WindowsTest.java
@@ -22,7 +22,9 @@ import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.time.Time;
+import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.ml.common.datastream.DataStreamUtils;
+import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.windowing.ProcessAllWindowFunction;
@@ -54,8 +56,7 @@ public class WindowsTest extends AbstractTestBase {
@BeforeClass
public static void beforeClass() {
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
- env.getConfig().enableObjectReuse();
+ StreamExecutionEnvironment env = TestUtils.getExecutionEnvironment();
inputData = new ArrayList<>();
for (long i = 0; i < RECORD_NUM; i++) {
inputData.add(i);
@@ -93,7 +94,8 @@ public class WindowsTest extends AbstractTestBase {
DataStreamUtils.windowAllAndProcess(
inputStream,
GlobalWindows.getInstance(),
- new CreateAllWindowBatchFunction<>());
+ new CreateAllWindowBatchFunction<>(),
+ Types.LIST(Types.LONG));
List<List<Long>> actualBatches = IteratorUtils.toList(outputStream.executeAndCollect());
assertEquals(1, actualBatches.size());
assertEquals(new HashSet<>(inputData), new HashSet<>(actualBatches.get(0)));
@@ -105,7 +107,8 @@ public class WindowsTest extends AbstractTestBase {
DataStreamUtils.windowAllAndProcess(
inputStream,
CountTumblingWindows.of(RECORD_NUM / 7),
- new CreateAllWindowBatchFunction<>());
+ new CreateAllWindowBatchFunction<>(),
+ Types.LIST(Types.LONG));
List<List<Long>> actualBatches = IteratorUtils.toList(outputStream.executeAndCollect());
assertEquals(7, actualBatches.size());
int count = 0;
@@ -121,7 +124,8 @@ public class WindowsTest extends AbstractTestBase {
DataStreamUtils.windowAllAndProcess(
inputStreamWithProcessingTimeGap,
ProcessingTimeTumblingWindows.of(Time.milliseconds(100)),
- new CreateAllWindowBatchFunction<>());
+ new CreateAllWindowBatchFunction<>(),
+ Types.LIST(Types.LONG));
List<List<Long>> actualBatches = IteratorUtils.toList(outputStream.executeAndCollect());
assertTrue(actualBatches.size() > 1);
List<Long> mergedBatches = new ArrayList<>();
@@ -137,7 +141,8 @@ public class WindowsTest extends AbstractTestBase {
DataStreamUtils.windowAllAndProcess(
inputStreamWithEventTime,
EventTimeTumblingWindows.of(Time.milliseconds(RECORD_NUM / 7)),
- new CreateAllWindowBatchFunction<>());
+ new CreateAllWindowBatchFunction<>(),
+ Types.LIST(Types.LONG));
List<List<Long>> actualBatches = IteratorUtils.toList(outputStream.executeAndCollect());
assertEquals(8, actualBatches.size());
List<Long> mergedBatches = new ArrayList<>();
@@ -154,7 +159,8 @@ public class WindowsTest extends AbstractTestBase {
DataStreamUtils.windowAllAndProcess(
inputStreamWithProcessingTimeGap,
ProcessingTimeSessionWindows.withGap(Time.milliseconds(100)),
- new CreateAllWindowBatchFunction<>());
+ new CreateAllWindowBatchFunction<>(),
+ Types.LIST(Types.LONG));
List<List<Long>> actualBatches = IteratorUtils.toList(outputStream.executeAndCollect());
assertTrue(actualBatches.size() > 1);
List<Long> mergedBatches = new ArrayList<>();
@@ -170,7 +176,8 @@ public class WindowsTest extends AbstractTestBase {
DataStreamUtils.windowAllAndProcess(
inputStreamWithEventTime,
EventTimeSessionWindows.withGap(Time.milliseconds(RECORD_NUM / 7)),
- new CreateAllWindowBatchFunction<>());
+ new CreateAllWindowBatchFunction<>(),
+ Types.LIST(Types.LONG));
List<List<Long>> actualBatches = IteratorUtils.toList(outputStream.executeAndCollect());
assertEquals(1, actualBatches.size());
assertEquals(new HashSet<>(inputData), new HashSet<>(actualBatches.get(0)));
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/classification/naivebayes/NaiveBayesModelData.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/classification/naivebayes/NaiveBayesModelData.java
index 097d356..fac5dab 100644
--- a/flink-ml-lib/src/main/java/org/apache/flink/ml/classification/naivebayes/NaiveBayesModelData.java
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/classification/naivebayes/NaiveBayesModelData.java
@@ -53,7 +53,7 @@ import java.util.Map;
*/
public class NaiveBayesModelData {
- private static Map<String, TypeInformation<?>> fields;
+ private static final Map<String, TypeInformation<?>> fields;
static {
fields = new HashMap<>();
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/PriorityQueueSerializer.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/PriorityQueueSerializer.java
new file mode 100644
index 0000000..857996e
--- /dev/null
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/PriorityQueueSerializer.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.common.typeinfo;
+
+import org.apache.flink.api.common.typeutils.CompositeTypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.base.ListSerializer;
+import org.apache.flink.api.java.typeutils.runtime.DataInputViewStream;
+import org.apache.flink.api.java.typeutils.runtime.DataOutputViewStream;
+import org.apache.flink.core.memory.DataInputView;
+import org.apache.flink.core.memory.DataOutputView;
+import org.apache.flink.util.InstantiationUtil;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Objects;
+import java.util.PriorityQueue;
+
+/**
+ * TypeSerializer for {@link java.util.PriorityQueue}.
+ *
+ * @param <T> The type of elements in the PriorityQueue.
+ */
+public class PriorityQueueSerializer<T> extends TypeSerializer<PriorityQueue<T>> {
+
+ private final Comparator<? super T> comparator;
+
+ private final TypeSerializer<T> elementSerializer;
+
+ public PriorityQueueSerializer(
+ Comparator<? super T> comparator, TypeSerializer<T> elementSerializer) {
+ this.comparator = comparator;
+ this.elementSerializer = elementSerializer;
+ }
+
+ @Override
+ public boolean isImmutableType() {
+ return false;
+ }
+
+ @Override
+ public TypeSerializer<PriorityQueue<T>> duplicate() {
+ return new PriorityQueueSerializer<>(comparator, elementSerializer.duplicate());
+ }
+
+ @Override
+ public PriorityQueue<T> createInstance() {
+ return new PriorityQueue<>(comparator);
+ }
+
+ @Override
+ public PriorityQueue<T> copy(PriorityQueue<T> from) {
+ return new PriorityQueue<>(from);
+ }
+
+ @Override
+ public PriorityQueue<T> copy(PriorityQueue<T> from, PriorityQueue<T> reuse) {
+ return new PriorityQueue<>(from);
+ }
+
+ @Override
+ public int getLength() {
+ return -1;
+ }
+
+ @Override
+ public void serialize(PriorityQueue<T> queue, DataOutputView target) throws IOException {
+ List<T> tmpList = new ArrayList<>(queue);
+ ListSerializer<T> listSerializer = new ListSerializer<>(elementSerializer);
+ listSerializer.serialize(tmpList, target);
+ }
+
+ @Override
+ public PriorityQueue<T> deserialize(DataInputView source) throws IOException {
+ ListSerializer<T> listSerializer = new ListSerializer<>(elementSerializer);
+ List<T> tmpList = listSerializer.deserialize(source);
+ PriorityQueue<T> queue = new PriorityQueue<>(comparator);
+ queue.addAll(tmpList);
+ return queue;
+ }
+
+ @Override
+ public PriorityQueue<T> deserialize(PriorityQueue<T> reuse, DataInputView source)
+ throws IOException {
+ return deserialize(source);
+ }
+
+ @Override
+ public void copy(DataInputView source, DataOutputView target) throws IOException {
+ ListSerializer<T> listSerializer = new ListSerializer<>(elementSerializer);
+ listSerializer.copy(source, target);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ PriorityQueueSerializer<?> that = (PriorityQueueSerializer<?>) o;
+ return Objects.equals(elementSerializer, that.elementSerializer)
+ && Objects.equals(comparator, that.comparator);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ comparator != null ? comparator.hashCode() : 0,
+ elementSerializer != null ? elementSerializer.hashCode() : 0);
+ }
+
+ @Override
+ public TypeSerializerSnapshot<PriorityQueue<T>> snapshotConfiguration() {
+ return new PriorityQueueTypeSerializerSnapshot<>(this);
+ }
+
+ private static final class PriorityQueueTypeSerializerSnapshot<T>
+ extends CompositeTypeSerializerSnapshot<PriorityQueue<T>, PriorityQueueSerializer<T>> {
+
+ private static final int CURRENT_VERSION = 1;
+
+ private Comparator<? super T> comparator;
+
+ public PriorityQueueTypeSerializerSnapshot() {
+ super(PriorityQueueSerializer.class);
+ }
+
+ public PriorityQueueTypeSerializerSnapshot(PriorityQueueSerializer<T> serializer) {
+ super(serializer);
+ this.comparator = serializer.comparator;
+ }
+
+ @Override
+ protected int getCurrentOuterSnapshotVersion() {
+ return CURRENT_VERSION;
+ }
+
+ @Override
+ protected TypeSerializer<?>[] getNestedSerializers(
+ PriorityQueueSerializer<T> outerSerializer) {
+ return new TypeSerializer[] {outerSerializer.elementSerializer};
+ }
+
+ @Override
+ protected PriorityQueueSerializer<T> createOuterSerializerWithNestedSerializers(
+ TypeSerializer<?>[] nestedSerializers) {
+ TypeSerializer<T> elementSerializer = (TypeSerializer<T>) nestedSerializers[0];
+ return new PriorityQueueSerializer<>(comparator, elementSerializer);
+ }
+
+ @Override
+ protected void writeOuterSnapshot(DataOutputView out) throws IOException {
+ final DataOutputViewStream stream = new DataOutputViewStream(out);
+ InstantiationUtil.serializeObject(stream, comparator);
+ }
+
+ @Override
+ protected void readOuterSnapshot(
+ int readOuterSnapshotVersion, DataInputView in, ClassLoader userCodeClassLoader)
+ throws IOException {
+ final DataInputViewStream stream = new DataInputViewStream(in);
+ try {
+ comparator = InstantiationUtil.deserializeObject(stream, userCodeClassLoader);
+ } catch (ClassNotFoundException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ protected OuterSchemaCompatibility resolveOuterSchemaCompatibility(
+ PriorityQueueSerializer<T> newSerializer) {
+ return (this.comparator.getClass() == newSerializer.comparator.getClass())
+ ? OuterSchemaCompatibility.COMPATIBLE_AS_IS
+ : OuterSchemaCompatibility.INCOMPATIBLE;
+ }
+ }
+}
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/PriorityQueueTypeInfo.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/PriorityQueueTypeInfo.java
new file mode 100644
index 0000000..ca546e8
--- /dev/null
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/PriorityQueueTypeInfo.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.common.typeinfo;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+
+import java.util.Comparator;
+import java.util.Objects;
+import java.util.PriorityQueue;
+
+/**
+ * TypeInformation for {@link java.util.PriorityQueue}.
+ *
+ * @param <T> The type of elements in the PriorityQueue.
+ */
+public class PriorityQueueTypeInfo<T> extends TypeInformation<PriorityQueue<T>> {
+
+ private final Comparator<? super T> comparator;
+
+ private final TypeInformation<T> elementTypeInfo;
+
+ public PriorityQueueTypeInfo(
+ Comparator<? super T> comparator, TypeInformation<T> elementTypeInfo) {
+ this.comparator = comparator;
+ this.elementTypeInfo = elementTypeInfo;
+ }
+
+ public TypeInformation<T> getElementTypeInfo() {
+ return elementTypeInfo;
+ }
+
+ @Override
+ public boolean isBasicType() {
+ return false;
+ }
+
+ @Override
+ public boolean isTupleType() {
+ return false;
+ }
+
+ @Override
+ public int getArity() {
+ return 1;
+ }
+
+ @Override
+ public int getTotalFields() {
+ return 1;
+ }
+
+ @Override
+ public Class<PriorityQueue<T>> getTypeClass() {
+ return (Class) PriorityQueue.class;
+ }
+
+ @Override
+ public boolean isKeyType() {
+ return false;
+ }
+
+ @Override
+ public TypeSerializer<PriorityQueue<T>> createSerializer(ExecutionConfig config) {
+ return new PriorityQueueSerializer<>(comparator, elementTypeInfo.createSerializer(config));
+ }
+
+ @Override
+ public String toString() {
+ return "PriorityQueue Type";
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+
+ PriorityQueueTypeInfo<T> that = (PriorityQueueTypeInfo<T>) obj;
+ return Objects.equals(comparator, that.comparator)
+ && Objects.equals(elementTypeInfo, that.elementTypeInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(
+ comparator != null ? comparator.hashCode() : 0,
+ elementTypeInfo != null ? elementTypeInfo.hashCode() : 0);
+ }
+
+ @Override
+ public boolean canEqual(Object obj) {
+ return obj instanceof PriorityQueueTypeInfo;
+ }
+}
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/QuantileSummaryTypeInfoFactory.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/QuantileSummaryTypeInfoFactory.java
new file mode 100644
index 0000000..ec84251
--- /dev/null
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/typeinfo/QuantileSummaryTypeInfoFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.common.typeinfo;
+
+import org.apache.flink.api.common.typeinfo.TypeInfoFactory;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.ml.common.util.QuantileSummary;
+
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Used by {@link TypeExtractor} to create a {@link TypeInformation} for implementations of {@link
+ * QuantileSummary}.
+ */
+public class QuantileSummaryTypeInfoFactory extends TypeInfoFactory<QuantileSummary> {
+
+ private static final Map<String, TypeInformation<?>> fields;
+
+ static {
+ fields = new HashMap<>();
+ fields.put("relativeError", Types.DOUBLE);
+ fields.put("compressThreshold", Types.INT);
+ fields.put("count", Types.LONG);
+ fields.put("sampled", Types.LIST(TypeInformation.of(QuantileSummary.StatsTuple.class)));
+ fields.put("headBuffer", Types.LIST(Types.DOUBLE));
+ fields.put("compressed", Types.BOOLEAN);
+ }
+
+ private static final TypeInformation<QuantileSummary> TYPE_INFO =
+ Types.POJO(QuantileSummary.class, fields);
+
+ @Override
+ public TypeInformation<QuantileSummary> createTypeInfo(
+ Type t, Map<String, TypeInformation<?>> genericParameters) {
+ return TYPE_INFO;
+ }
+}
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/common/util/QuantileSummary.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/util/QuantileSummary.java
index 49730fe..b57a9b1 100644
--- a/flink-ml-lib/src/main/java/org/apache/flink/ml/common/util/QuantileSummary.java
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/common/util/QuantileSummary.java
@@ -18,6 +18,8 @@
package org.apache.flink.ml.common.util;
+import org.apache.flink.api.common.typeinfo.TypeInfo;
+import org.apache.flink.ml.common.typeinfo.QuantileSummaryTypeInfoFactory;
import org.apache.flink.util.Preconditions;
import java.io.Serializable;
@@ -36,6 +38,7 @@ import java.util.stream.IntStream;
* algorithm proposed in the paper: "Space-efficient Online Computation of Quantile Summaries" by
* Greenwald, Michael and Khanna, Sanjeev. (https://doi.org/10.1145/375663.375670)
*/
+@TypeInfo(QuantileSummaryTypeInfoFactory.class)
public class QuantileSummary implements Serializable {
/** The default size of head buffer. */
@@ -45,26 +48,29 @@ public class QuantileSummary implements Serializable {
private static final int DEFAULT_COMPRESS_THRESHOLD = 10000;
/** The target relative error. */
- private final double relativeError;
+ private double relativeError;
/**
* The compression threshold. After the internal buffer of statistics crosses this size, it
* attempts to compress the statistics together.
*/
- private final int compressThreshold;
+ private int compressThreshold;
/** The count of all the elements inserted to be calculated. */
- private final long count;
+ private long count;
/** A buffer of quantile statistics. */
- private final List<StatsTuple> sampled;
+ private List<StatsTuple> sampled;
/** A buffer of the latest samples seen so far. */
- private final List<Double> headBuffer = new ArrayList<>();
+ private List<Double> headBuffer = new ArrayList<>();
/** Whether the quantile summary has been compressed. */
private boolean compressed;
+ /** Empty QuantileSummary Constructor. */
+ public QuantileSummary() {}
+
/**
* QuantileSummary Constructor.
*
@@ -95,7 +101,7 @@ public class QuantileSummary implements Serializable {
* @param count The count of all the elements inserted in the sampled buffer.
* @param compressed Whether the statistics have been compressed.
*/
- private QuantileSummary(
+ public QuantileSummary(
double relativeError,
int compressThreshold,
List<StatsTuple> sampled,
@@ -367,6 +373,26 @@ public class QuantileSummary implements Serializable {
return relativeError;
}
+ public int getCompressThreshold() {
+ return compressThreshold;
+ }
+
+ public long getCount() {
+ return count;
+ }
+
+ public List<StatsTuple> getSampled() {
+ return sampled;
+ }
+
+ public List<Double> getHeadBuffer() {
+ return headBuffer;
+ }
+
+ public boolean isCompressed() {
+ return compressed;
+ }
+
private QuantileSummary shallowCopy() {
return new QuantileSummary(relativeError, compressThreshold, sampled, count, compressed);
}
@@ -395,11 +421,13 @@ public class QuantileSummary implements Serializable {
* <li>delta: the maximum span of the rank.
* </ul>
*/
- private static class StatsTuple implements Serializable {
+ public static class StatsTuple implements Serializable {
private static final long serialVersionUID = 1L;
- private final double value;
- private long g;
- private long delta;
+ public double value;
+ public long g;
+ public long delta;
+
+ public StatsTuple() {}
public StatsTuple(double value, long g, long delta) {
this.value = value;
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/lsh/LSHModel.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/lsh/LSHModel.java
index cc706bd..66c88ea 100644
--- a/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/lsh/LSHModel.java
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/lsh/LSHModel.java
@@ -32,8 +32,11 @@ import org.apache.flink.ml.common.broadcast.BroadcastUtils;
import org.apache.flink.ml.common.datastream.DataStreamUtils;
import org.apache.flink.ml.common.datastream.EndOfStreamWindows;
import org.apache.flink.ml.common.datastream.TableUtils;
+import org.apache.flink.ml.common.typeinfo.PriorityQueueTypeInfo;
import org.apache.flink.ml.linalg.DenseVector;
import org.apache.flink.ml.linalg.Vector;
+import org.apache.flink.ml.linalg.typeinfo.DenseVectorTypeInfo;
+import org.apache.flink.ml.linalg.typeinfo.VectorTypeInfo;
import org.apache.flink.ml.param.Param;
import org.apache.flink.ml.util.ParamUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
@@ -46,6 +49,7 @@ import org.apache.flink.util.Preconditions;
import org.apache.commons.lang3.ArrayUtils;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -163,8 +167,13 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
new FilterByBucketFunction(getInputCol(), getOutputCol(), key),
outputTypeInfo);
});
+ TopKFunction topKFunction = new TopKFunction(distCol, k);
DataStream<List<Row>> topKList =
- DataStreamUtils.aggregate(filteredData, new TopKFunction(distCol, k));
+ DataStreamUtils.aggregate(
+ filteredData,
+ topKFunction,
+ new PriorityQueueTypeInfo(topKFunction.getComparator(), outputTypeInfo),
+ Types.LIST(outputTypeInfo));
DataStream<Row> topKData =
topKList.flatMap(
(value, out) -> {
@@ -206,6 +215,14 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
DataStream<Row> explodedA = preprocessData(datasetA, idCol);
DataStream<Row> explodedB = preprocessData(datasetB, idCol);
+ RowTypeInfo inputTypeInfo = getOutputType(datasetA, idCol);
+ RowTypeInfo outputTypeInfo =
+ new RowTypeInfo(
+ inputTypeInfo.getTypeAt(0),
+ inputTypeInfo.getTypeAt(0),
+ inputTypeInfo.getTypeAt(1),
+ inputTypeInfo.getTypeAt(1));
+
DataStream<? extends LSHModelData> modelData =
tEnv.toDataStream(modelDataTable, modelDataClass);
DataStream<Row> sameBucketPairs =
@@ -220,7 +237,9 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
r0.getField(0),
r1.getField(0),
r0.getField(1),
- r1.getField(1)));
+ r1.getField(1)),
+ outputTypeInfo);
+
DataStream<Row> distinctSameBucketPairs =
DataStreamUtils.reduce(
sameBucketPairs.keyBy(
@@ -230,10 +249,11 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
return Tuple2.of(r.getFieldAs(0), r.getFieldAs(1));
}
}),
- (r0, r1) -> r0);
+ (r0, r1) -> r0,
+ outputTypeInfo);
- RowTypeInfo inputTypeInfo = TableUtils.getRowTypeInfo(datasetA.getResolvedSchema());
- TypeInformation<?> idColType = inputTypeInfo.getTypeAt(idCol);
+ TypeInformation<?> idColType =
+ TableUtils.getRowTypeInfo(datasetA.getResolvedSchema()).getTypeAt(idCol);
DataStream<Row> pairsWithDists =
BroadcastUtils.withBroadcastStream(
Collections.singletonList(distinctSameBucketPairs),
@@ -268,25 +288,30 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
(dataTable.getResolvedSchema().getColumnNames().contains(getOutputCol()))
? dataTable
: transform(dataTable)[0];
+ RowTypeInfo outputTypeInfo = getOutputType(dataTable, idCol);
- RowTypeInfo inputTypeInfo = TableUtils.getRowTypeInfo(dataTable.getResolvedSchema());
- TypeInformation<?> idColType = inputTypeInfo.getTypeAt(idCol);
+ return tEnv.toDataStream(dataTable)
+ .flatMap(
+ new ExplodeHashValuesFunction(idCol, getInputCol(), getOutputCol()),
+ outputTypeInfo);
+ }
+
+ private RowTypeInfo getOutputType(Table dataTable, String idCol) {
final String indexCol = "index";
final String hashValueCol = "hashValue";
+ RowTypeInfo inputTypeInfo = TableUtils.getRowTypeInfo(dataTable.getResolvedSchema());
+ TypeInformation<?> idColType = inputTypeInfo.getTypeAt(idCol);
+
RowTypeInfo outputTypeInfo =
new RowTypeInfo(
new TypeInformation[] {
idColType,
- TypeInformation.of(Vector.class),
+ VectorTypeInfo.INSTANCE,
Types.INT,
- TypeInformation.of(DenseVector.class)
+ DenseVectorTypeInfo.INSTANCE
},
new String[] {idCol, getInputCol(), indexCol, hashValueCol});
-
- return tEnv.toDataStream(dataTable)
- .flatMap(
- new ExplodeHashValuesFunction(idCol, getInputCol(), getOutputCol()),
- outputTypeInfo);
+ return outputTypeInfo;
}
private static class PredictFunction extends RichMapFunction<Row, Row> {
@@ -353,7 +378,7 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
private final int numNearestNeighbors;
private final String distCol;
- private static class DistColComparator implements Comparator<Row> {
+ private static class DistColComparator implements Comparator<Row>, Serializable {
private final String distCol;
@@ -374,7 +399,7 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
@Override
public PriorityQueue<Row> createAccumulator() {
- return new PriorityQueue<>(numNearestNeighbors, new DistColComparator(distCol));
+ return new PriorityQueue<>(numNearestNeighbors, getComparator());
}
@Override
@@ -402,6 +427,10 @@ abstract class LSHModel<T extends LSHModel<T>> implements Model<T>, LSHModelPara
}
return merged;
}
+
+ private Comparator<Row> getComparator() {
+ return new DistColComparator(distCol);
+ }
}
private static class ExplodeHashValuesFunction implements FlatMapFunction<Row, Row> {
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/common/typeinfo/PriorityQueueSerializerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/common/typeinfo/PriorityQueueSerializerTest.java
new file mode 100644
index 0000000..a42ce48
--- /dev/null
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/common/typeinfo/PriorityQueueSerializerTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.common.typeinfo;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.types.Row;
+import org.apache.flink.util.TestLogger;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.io.Serializable;
+import java.util.Comparator;
+import java.util.PriorityQueue;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+/** Tests the serialization and deserialization for the {@link java.util.PriorityQueue}. */
+public class PriorityQueueSerializerTest extends TestLogger {
+
+ @Test
+ public void testSerializationDeserialization() throws IOException {
+ final MockComparator comparator = new MockComparator();
+ TypeInformation<PriorityQueue<Row>> typeInfo =
+ new PriorityQueueTypeInfo<>(comparator, Types.ROW(Types.DOUBLE, Types.STRING));
+ TypeSerializer<PriorityQueue<Row>> serializer =
+ typeInfo.createSerializer(new ExecutionConfig());
+ assertSame(PriorityQueueSerializer.class, serializer.getClass());
+
+ PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
+ DataInputViewStreamWrapper reader = new DataInputViewStreamWrapper(pipedInput);
+ DataOutputViewStreamWrapper writer =
+ new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
+
+ PriorityQueue<Row> queue = new PriorityQueue<>(comparator);
+ queue.add(Row.of(2.0, "b"));
+ queue.add(Row.of(1.0, "a"));
+ queue.add(Row.of(3.0, "c"));
+
+ serializer.serialize(queue, writer);
+ writer.write(reader, writer.size());
+
+ PriorityQueue<Row> deserialized = serializer.deserialize(reader);
+
+ assertEquals(3, deserialized.size());
+ assertEquals("a", deserialized.peek().getFieldAs(1));
+ }
+
+ private static class MockComparator implements Comparator<Row>, Serializable {
+ @Override
+ public int compare(Row o1, Row o2) {
+ return Double.compare(o1.getFieldAs(0), o2.getFieldAs(0));
+ }
+ }
+}
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/common/typeinfo/QuantileSummaryTypeInfoFactoryTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/common/typeinfo/QuantileSummaryTypeInfoFactoryTest.java
new file mode 100644
index 0000000..ffbf291
--- /dev/null
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/common/typeinfo/QuantileSummaryTypeInfoFactoryTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.common.typeinfo;
+
+import org.apache.flink.api.common.ExecutionConfig;
+import org.apache.flink.api.common.typeutils.TypeSerializer;
+import org.apache.flink.api.java.typeutils.TypeExtractor;
+import org.apache.flink.api.java.typeutils.runtime.PojoSerializer;
+import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
+import org.apache.flink.ml.common.util.QuantileSummary;
+
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
+
+/** Tests {@link QuantileSummaryTypeInfoFactory}. */
+public class QuantileSummaryTypeInfoFactoryTest {
+
+ private static final double EPS = 1.0e-5;
+
+ @Test
+ public void testSerializationDeserialization() throws IOException {
+ TypeSerializer<QuantileSummary> serializer =
+ TypeExtractor.createTypeInfo(QuantileSummary.class)
+ .createSerializer(new ExecutionConfig());
+ assertSame(PojoSerializer.class, serializer.getClass());
+
+ PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
+ DataInputViewStreamWrapper reader = new DataInputViewStreamWrapper(pipedInput);
+ DataOutputViewStreamWrapper writer =
+ new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
+
+ List<QuantileSummary.StatsTuple> sampled =
+ Arrays.asList(
+ new QuantileSummary.StatsTuple(10.0, 1L, 1L),
+ new QuantileSummary.StatsTuple(20.0, 1L, 2L),
+ new QuantileSummary.StatsTuple(30.0, 1L, 3L));
+ QuantileSummary summary = new QuantileSummary(0.1, 100, sampled, 3, true);
+ summary.insert(5.0);
+
+ serializer.serialize(summary, writer);
+ writer.write(reader, writer.size());
+
+ QuantileSummary deserialized = serializer.deserialize(reader);
+ assertEquals(0.1, deserialized.getRelativeError(), EPS);
+ assertEquals(100, deserialized.getCompressThreshold());
+ assertEquals(3, deserialized.getSampled().size());
+ assertEquals(5.0, deserialized.getHeadBuffer().get(0), EPS);
+ assertEquals(3, deserialized.getCount());
+ assertFalse(deserialized.isCompressed());
+ }
+}
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
index 20cc7c8..760a997 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/ImputerTest.java
@@ -18,13 +18,10 @@
package org.apache.flink.ml.feature;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.ml.feature.imputer.Imputer;
import org.apache.flink.ml.feature.imputer.ImputerModel;
import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
@@ -114,13 +111,7 @@ public class ImputerTest extends AbstractTestBase {
@Before
public void before() {
- Configuration config = new Configuration();
- config.set(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true);
- env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- env.getConfig().enableObjectReuse();
- env.setParallelism(4);
- env.enableCheckpointing(100);
- env.setRestartStrategy(RestartStrategies.noRestart());
+ env = TestUtils.getExecutionEnvironment();
tEnv = StreamTableEnvironment.create(env);
trainDataTable =
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
index 1678a57..fdb71f6 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/MinHashLSHTest.java
@@ -18,8 +18,6 @@
package org.apache.flink.ml.feature;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.ml.feature.lsh.MinHashLSH;
import org.apache.flink.ml.feature.lsh.MinHashLSHModel;
import org.apache.flink.ml.feature.lsh.MinHashLSHModelData;
@@ -30,7 +28,6 @@ import org.apache.flink.ml.linalg.Vectors;
import org.apache.flink.ml.util.ReadWriteUtils;
import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.Schema;
@@ -118,13 +115,7 @@ public class MinHashLSHTest extends AbstractTestBase {
@Before
public void before() {
- Configuration config = new Configuration();
- config.set(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true);
- env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- env.getConfig().enableObjectReuse();
- env.setParallelism(4);
- env.enableCheckpointing(100);
- env.setRestartStrategy(RestartStrategies.noRestart());
+ env = TestUtils.getExecutionEnvironment();
tEnv = StreamTableEnvironment.create(env);
List<Row> inputRows =
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
index aca8dbf..940dcbe 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/RobustScalerTest.java
@@ -19,8 +19,6 @@
package org.apache.flink.ml.feature;
import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.ml.feature.robustscaler.RobustScaler;
import org.apache.flink.ml.feature.robustscaler.RobustScalerModel;
import org.apache.flink.ml.linalg.DenseVector;
@@ -28,7 +26,6 @@ import org.apache.flink.ml.linalg.SparseVector;
import org.apache.flink.ml.linalg.Vectors;
import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Expressions;
import org.apache.flink.table.api.Table;
@@ -91,13 +88,7 @@ public class RobustScalerTest extends AbstractTestBase {
@Before
public void before() {
- Configuration config = new Configuration();
- config.set(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true);
- env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- env.getConfig().enableObjectReuse();
- env.setParallelism(4);
- env.enableCheckpointing(100);
- env.setRestartStrategy(RestartStrategies.noRestart());
+ env = TestUtils.getExecutionEnvironment();
tEnv = StreamTableEnvironment.create(env);
trainDataTable = tEnv.fromDataStream(env.fromCollection(TRAIN_DATA)).as("id", "input");
predictDataTable = tEnv.fromDataStream(env.fromCollection(PREDICT_DATA)).as("input");
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/UnivariateFeatureSelectorTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/UnivariateFeatureSelectorTest.java
index 76076d9..9851deb 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/UnivariateFeatureSelectorTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/UnivariateFeatureSelectorTest.java
@@ -18,9 +18,7 @@
package org.apache.flink.ml.feature;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.ml.feature.univariatefeatureselector.UnivariateFeatureSelector;
import org.apache.flink.ml.feature.univariatefeatureselector.UnivariateFeatureSelectorModel;
import org.apache.flink.ml.linalg.Vector;
@@ -28,7 +26,6 @@ import org.apache.flink.ml.linalg.Vectors;
import org.apache.flink.ml.linalg.typeinfo.VectorTypeInfo;
import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
@@ -466,13 +463,7 @@ public class UnivariateFeatureSelectorTest extends AbstractTestBase {
@Before
public void before() {
- Configuration config = new Configuration();
- config.set(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true);
- env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- env.setParallelism(4);
- env.enableCheckpointing(100);
- env.setRestartStrategy(RestartStrategies.noRestart());
- env.getConfig().enableObjectReuse();
+ env = TestUtils.getExecutionEnvironment();
tEnv = StreamTableEnvironment.create(env);
selectorWithChiSqTest =
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
index b7c5604..9d584ad 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/feature/VarianceThresholdSelectorTest.java
@@ -19,9 +19,7 @@
package org.apache.flink.ml.feature;
import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.restartstrategy.RestartStrategies;
import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.ml.feature.variancethresholdselector.VarianceThresholdSelector;
import org.apache.flink.ml.feature.variancethresholdselector.VarianceThresholdSelectorModel;
import org.apache.flink.ml.linalg.Vector;
@@ -29,7 +27,6 @@ import org.apache.flink.ml.linalg.Vectors;
import org.apache.flink.ml.linalg.typeinfo.VectorTypeInfo;
import org.apache.flink.ml.util.TestUtils;
import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
@@ -85,13 +82,7 @@ public class VarianceThresholdSelectorTest extends AbstractTestBase {
@Before
public void before() {
- Configuration config = new Configuration();
- config.set(ExecutionCheckpointingOptions.ENABLE_CHECKPOINTS_AFTER_TASKS_FINISH, true);
- env = StreamExecutionEnvironment.getExecutionEnvironment(config);
- env.getConfig().enableObjectReuse();
- env.setParallelism(4);
- env.enableCheckpointing(100);
- env.setRestartStrategy(RestartStrategies.noRestart());
+ env = TestUtils.getExecutionEnvironment();
tEnv = StreamTableEnvironment.create(env);
trainDataTable =