You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by li...@apache.org on 2023/01/06 06:24:05 UTC

[flink-ml] branch master updated: [FLINK-30249] Fix TableUtils.getRowTypeInfo by using ExternalTypeInfo.of()

This is an automated email from the ASF dual-hosted git repository.

lindong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink-ml.git


The following commit(s) were added to refs/heads/master by this push:
     new 77aad8f  [FLINK-30249] Fix TableUtils.getRowTypeInfo by using ExternalTypeInfo.of()
77aad8f is described below

commit 77aad8f9bbf28eac2363a4fbf98d91cae0d82fa8
Author: Zhipeng Zhang <zh...@gmail.com>
AuthorDate: Fri Jan 6 14:24:01 2023 +0800

    [FLINK-30249] Fix TableUtils.getRowTypeInfo by using ExternalTypeInfo.of()
    
    This closes #188.
---
 .../flink/ml/common/datastream/TableUtils.java     |  45 +++++-
 .../flink/ml/common/datastream/TableUtilsTest.java | 151 +++++++++++++++++++++
 .../flink/ml/feature/binarizer/Binarizer.java      |   8 +-
 .../org/apache/flink/ml/clustering/KMeansTest.java |   5 +-
 .../java/org/apache/flink/ml/util/TestUtils.java   |   6 +-
 5 files changed, 202 insertions(+), 13 deletions(-)

diff --git a/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/TableUtils.java b/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/TableUtils.java
index 9245c77..6c8a1ac 100644
--- a/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/TableUtils.java
+++ b/flink-ml-core/src/main/java/org/apache/flink/ml/common/datastream/TableUtils.java
@@ -26,18 +26,49 @@ import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.table.catalog.Column;
 import org.apache.flink.table.catalog.ResolvedSchema;
+import org.apache.flink.table.runtime.typeutils.ExternalTypeInfo;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
 import org.apache.flink.types.Row;
 
+import java.util.HashSet;
+import java.util.Set;
+
 /** Utility class for operations related to Table API. */
 public class TableUtils {
-    // Constructs a RowTypeInfo from the given schema.
+
+    // Logical type roots that may cause wrong type conversion between Table and DataStream.
+    private static final Set<LogicalTypeRoot> LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO =
+            new HashSet<>();
+
+    static {
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.CHAR);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.VARCHAR);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.BINARY);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.VARBINARY);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.DECIMAL);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.DATE);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.TIME_WITHOUT_TIME_ZONE);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(
+                LogicalTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.INTERVAL_DAY_TIME);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(
+                LogicalTypeRoot.TIMESTAMP_WITHOUT_TIME_ZONE);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.ARRAY);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.MAP);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.MULTISET);
+        LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.add(LogicalTypeRoot.ROW);
+    }
+
+    // Constructs a RowTypeInfo from the given schema. Currently, this function does not support
+    // the case when the input contains DataTypes.TIMESTAMP_WITH_TIME_ZONE().
     public static RowTypeInfo getRowTypeInfo(ResolvedSchema schema) {
         TypeInformation<?>[] types = new TypeInformation<?>[schema.getColumnCount()];
         String[] names = new String[schema.getColumnCount()];
 
         for (int i = 0; i < schema.getColumnCount(); i++) {
             Column column = schema.getColumn(i).get();
-            types[i] = TypeInformation.of(column.getDataType().getConversionClass());
+            types[i] = getTypeInformationFromDataType(column.getDataType());
             names[i] = column.getName();
         }
         return new RowTypeInfo(types, names);
@@ -48,7 +79,7 @@ public class TableUtils {
     public static TypeInformation<?> getTypeInfoByName(ResolvedSchema schema, String name) {
         for (Column column : schema.getColumns()) {
             if (column.getName().equals(name)) {
-                return TypeInformation.of(column.getDataType().getConversionClass());
+                return getTypeInformationFromDataType(column.getDataType());
             }
         }
         return null;
@@ -59,4 +90,12 @@ public class TableUtils {
         DataStream<Row> dataStream = tEnv.toDataStream(table);
         return dataStream.getExecutionEnvironment();
     }
+
+    private static TypeInformation<?> getTypeInformationFromDataType(DataType dataType) {
+        if (LOGICAL_TYPE_ROOTS_USING_EXTERNAL_TYPE_INFO.contains(
+                dataType.getLogicalType().getTypeRoot())) {
+            return ExternalTypeInfo.of(dataType);
+        }
+        return TypeInformation.of(dataType.getConversionClass());
+    }
 }
diff --git a/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/TableUtilsTest.java b/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/TableUtilsTest.java
new file mode 100644
index 0000000..9da5cda
--- /dev/null
+++ b/flink-ml-core/src/test/java/org/apache/flink/ml/common/datastream/TableUtilsTest.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.ml.common.datastream;
+
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.typeinfo.Types;
+import org.apache.flink.api.java.typeutils.RowTypeInfo;
+import org.apache.flink.ml.linalg.DenseMatrix;
+import org.apache.flink.ml.linalg.DenseVector;
+import org.apache.flink.ml.linalg.SparseVector;
+import org.apache.flink.ml.linalg.typeinfo.DenseMatrixTypeInfo;
+import org.apache.flink.ml.linalg.typeinfo.DenseVectorTypeInfo;
+import org.apache.flink.ml.linalg.typeinfo.SparseVectorTypeInfo;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.Table;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.types.AbstractDataType;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.types.Row;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.math.BigDecimal;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/** Tests the {@link TableUtils}. */
+public class TableUtilsTest {
+    private StreamExecutionEnvironment env;
+    private StreamTableEnvironment tEnv;
+
+    @Before
+    public void before() {
+        env = StreamExecutionEnvironment.getExecutionEnvironment();
+        tEnv = StreamTableEnvironment.create(env);
+    }
+
+    @Test
+    public void testGetRowTypeInfo() {
+        List<AbstractDataType<?>> preDefinedDataTypes = new ArrayList<>();
+        List<Object> dataFields = new ArrayList<>();
+
+        preDefinedDataTypes.add(DataTypes.CHAR(10));
+        dataFields.add("char");
+        preDefinedDataTypes.add(DataTypes.VARCHAR(100));
+        dataFields.add("varchar");
+        preDefinedDataTypes.add(DataTypes.STRING());
+        dataFields.add("string");
+        preDefinedDataTypes.add(DataTypes.BOOLEAN());
+        dataFields.add(true);
+        preDefinedDataTypes.add(DataTypes.BINARY(10));
+        dataFields.add(new byte[] {'a', 'b', 'c'});
+        preDefinedDataTypes.add(DataTypes.VARBINARY(100));
+        dataFields.add(new byte[] {'a', 'b', 'c'});
+        preDefinedDataTypes.add(DataTypes.BYTES());
+        dataFields.add(new byte[] {'a', 'b', 'c'});
+        preDefinedDataTypes.add(DataTypes.DECIMAL(11, 3));
+        dataFields.add(new BigDecimal(100));
+        preDefinedDataTypes.add(DataTypes.TINYINT());
+        dataFields.add((byte) 'a');
+        preDefinedDataTypes.add(DataTypes.SMALLINT());
+        dataFields.add((short) 1);
+        preDefinedDataTypes.add(DataTypes.INT());
+        dataFields.add(1);
+        preDefinedDataTypes.add(DataTypes.BIGINT());
+        dataFields.add(1L);
+        preDefinedDataTypes.add(DataTypes.FLOAT());
+        dataFields.add(0.1f);
+        preDefinedDataTypes.add(DataTypes.DOUBLE());
+        dataFields.add(0.1);
+        preDefinedDataTypes.add(DataTypes.DATE());
+        dataFields.add(1);
+        preDefinedDataTypes.add(DataTypes.TIME(4));
+        dataFields.add(1);
+        preDefinedDataTypes.add(DataTypes.TIMESTAMP(9));
+        dataFields.add(new Timestamp(1));
+        preDefinedDataTypes.add(DataTypes.TIMESTAMP_LTZ());
+        dataFields.add(1);
+        preDefinedDataTypes.add(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(9));
+        dataFields.add(1);
+        preDefinedDataTypes.add(DataTypes.INTERVAL(DataTypes.SECOND()));
+        dataFields.add(1);
+        preDefinedDataTypes.add(DataTypes.ARRAY(DataTypes.TIME()));
+        dataFields.add(new int[] {1, 2});
+        preDefinedDataTypes.add(DataTypes.MAP(DataTypes.INT(), DataTypes.DOUBLE()));
+        dataFields.add(Collections.singletonMap(1, 0.1));
+        preDefinedDataTypes.add(DataTypes.MULTISET(DataTypes.DOUBLE()));
+        dataFields.add(Collections.singletonMap(0.1, 1));
+        preDefinedDataTypes.add(DataTypes.ROW(DataTypes.INT(), DataTypes.BIGINT()));
+        dataFields.add(Row.of(1, 2L));
+        preDefinedDataTypes.add(DataTypes.RAW(DenseVectorTypeInfo.INSTANCE));
+        dataFields.add(new DenseVector(new double[] {0.1, 0.2}));
+        preDefinedDataTypes.add(DataTypes.RAW(SparseVectorTypeInfo.INSTANCE));
+        dataFields.add(new SparseVector(2, new int[] {0}, new double[] {0.1}));
+        preDefinedDataTypes.add(DataTypes.RAW(DenseMatrixTypeInfo.INSTANCE));
+        dataFields.add(new DenseMatrix(2, 2));
+
+        Schema.Builder builder = Schema.newBuilder();
+        for (int i = 0; i < preDefinedDataTypes.size(); i++) {
+            builder.column("f" + i, preDefinedDataTypes.get(i));
+        }
+        Schema schema = builder.build();
+
+        Table inputTable =
+                tEnv.fromDataStream(env.fromElements(Row.of(dataFields.toArray())), schema);
+
+        RowTypeInfo inputTypeInfo = TableUtils.getRowTypeInfo(inputTable.getResolvedSchema());
+        RowTypeInfo outputTypeInfo =
+                new RowTypeInfo(
+                        ArrayUtils.addAll(inputTypeInfo.getFieldTypes(), Types.INT),
+                        ArrayUtils.addAll(inputTypeInfo.getFieldNames(), "outputCol"));
+
+        DataStream<Row> mappedOutput =
+                tEnv.toDataStream(inputTable)
+                        .map(
+                                (MapFunction<Row, Row>) row -> Row.of(row.getField(0), 1),
+                                outputTypeInfo);
+
+        List<DataType> inputDataTypes = inputTable.getResolvedSchema().getColumnDataTypes();
+        List<DataType> outputDataTypes =
+                tEnv.fromDataStream(mappedOutput).getResolvedSchema().getColumnDataTypes();
+        Assert.assertEquals(inputDataTypes.size() + 1, outputDataTypes.size());
+        for (int i = 0; i < inputDataTypes.size(); i++) {
+            Assert.assertEquals(inputDataTypes.get(i), outputDataTypes.get(i));
+        }
+    }
+}
diff --git a/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/binarizer/Binarizer.java b/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/binarizer/Binarizer.java
index bdf1637..aafbf6e 100644
--- a/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/binarizer/Binarizer.java
+++ b/flink-ml-lib/src/main/java/org/apache/flink/ml/feature/binarizer/Binarizer.java
@@ -26,6 +26,7 @@ import org.apache.flink.ml.api.Transformer;
 import org.apache.flink.ml.common.datastream.TableUtils;
 import org.apache.flink.ml.linalg.DenseVector;
 import org.apache.flink.ml.linalg.SparseVector;
+import org.apache.flink.ml.linalg.Vector;
 import org.apache.flink.ml.linalg.typeinfo.DenseVectorTypeInfo;
 import org.apache.flink.ml.linalg.typeinfo.SparseVectorTypeInfo;
 import org.apache.flink.ml.linalg.typeinfo.VectorTypeInfo;
@@ -70,11 +71,12 @@ public class Binarizer implements Transformer<Binarizer>, BinarizerParams<Binari
 
         for (int i = 0; i < inputCols.length; ++i) {
             int idx = inputTypeInfo.getFieldIndex(inputCols[i]);
-            if (inputTypeInfo.getFieldTypes()[idx] instanceof SparseVectorTypeInfo) {
+            Class<?> typeClass = inputTypeInfo.getTypeAt(idx).getTypeClass();
+            if (typeClass.equals(SparseVector.class)) {
                 outputTypes[i] = SparseVectorTypeInfo.INSTANCE;
-            } else if (inputTypeInfo.getFieldTypes()[idx] instanceof DenseVectorTypeInfo) {
+            } else if (typeClass.equals(DenseVector.class)) {
                 outputTypes[i] = DenseVectorTypeInfo.INSTANCE;
-            } else if (inputTypeInfo.getFieldTypes()[idx] instanceof VectorTypeInfo) {
+            } else if (typeClass.equals(Vector.class)) {
                 outputTypes[i] = VectorTypeInfo.INSTANCE;
             } else {
                 outputTypes[i] = Types.DOUBLE;
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/clustering/KMeansTest.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/clustering/KMeansTest.java
index 2e73a9d..292db2f 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/clustering/KMeansTest.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/clustering/KMeansTest.java
@@ -33,8 +33,6 @@ import org.apache.flink.ml.util.TestUtils;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.DataTypes;
-import org.apache.flink.table.api.Schema;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
 import org.apache.flink.test.util.AbstractTestBase;
@@ -168,8 +166,7 @@ public class KMeansTest extends AbstractTestBase {
                 Arrays.asList(
                         Vectors.dense(0.0, 0.1), Vectors.dense(0.0, 0.1), Vectors.dense(0.0, 0.1));
 
-        Schema schema = Schema.newBuilder().column("f0", DataTypes.of(DenseVector.class)).build();
-        Table input = tEnv.fromDataStream(env.fromCollection(data), schema).as("features");
+        Table input = tEnv.fromDataStream(env.fromCollection(data)).as("features");
 
         KMeans kmeans = new KMeans().setK(2);
         KMeansModel model = kmeans.fit(input);
diff --git a/flink-ml-lib/src/test/java/org/apache/flink/ml/util/TestUtils.java b/flink-ml-lib/src/test/java/org/apache/flink/ml/util/TestUtils.java
index 94be85f..09135b3 100644
--- a/flink-ml-lib/src/test/java/org/apache/flink/ml/util/TestUtils.java
+++ b/flink-ml-lib/src/test/java/org/apache/flink/ml/util/TestUtils.java
@@ -24,8 +24,8 @@ import org.apache.flink.api.common.typeinfo.Types;
 import org.apache.flink.api.java.typeutils.RowTypeInfo;
 import org.apache.flink.ml.api.Stage;
 import org.apache.flink.ml.common.datastream.TableUtils;
+import org.apache.flink.ml.linalg.DenseVector;
 import org.apache.flink.ml.linalg.Vector;
-import org.apache.flink.ml.linalg.typeinfo.DenseVectorTypeInfo;
 import org.apache.flink.ml.linalg.typeinfo.SparseVectorTypeInfo;
 import org.apache.flink.streaming.api.datastream.DataStream;
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
@@ -75,9 +75,9 @@ public class TestUtils {
         RowTypeInfo inputTypeInfo = TableUtils.getRowTypeInfo(table.getResolvedSchema());
         TypeInformation<?>[] fieldTypes = inputTypeInfo.getFieldTypes();
         for (int i = 0; i < fieldTypes.length; i++) {
-            if (fieldTypes[i].equals(DenseVectorTypeInfo.INSTANCE)) {
+            if (fieldTypes[i].getTypeClass().equals(DenseVector.class)) {
                 fieldTypes[i] = SparseVectorTypeInfo.INSTANCE;
-            } else if (fieldTypes[i].equals(Types.DOUBLE)) {
+            } else if (fieldTypes[i].getTypeClass().equals(Double.class)) {
                 fieldTypes[i] = Types.INT;
             }
         }