You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2019/06/17 17:52:55 UTC

[flink] branch master updated (b88f23c -> 3d8889e)

This is an automated email from the ASF dual-hosted git repository.

bli pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.


    from b88f23c  [FLINK-12711][table] Refactor function lookup
     new 8131c40  [FLINK-12853][table-api][table-planner] TableSourceUtils#validateTableSource method to common module
     new 3d8889e  [FLINK-12657][hive] Integrate Flink with Hive UDF

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../functions/hive/FlinkHiveUDFException.java      |  24 +-
 .../flink/table/functions/hive/HiveFunction.java   |  56 ++++
 .../table/functions/hive/HiveFunctionWrapper.java  |  91 ++++++
 .../table/functions/hive/HiveScalarFunction.java   | 147 ++++++++++
 .../flink/table/functions/hive/HiveSimpleUDF.java  | 135 +++++++++
 .../functions/hive/conversion/HiveInspectors.java  | 250 ++++++++++++++++
 .../hive/conversion/HiveObjectConversion.java      |  14 +-
 .../hive/conversion/IdentityConversion.java        |  16 +-
 .../table/functions/hive/HiveSimpleUDFTest.java    | 100 +++++++
 .../kafka/KafkaTableSourceSinkFactoryTestBase.java |   4 +-
 .../flink/table/sources/TableSourceValidation.java | 325 +++++++++++++++++++++
 .../apache/flink/table/api/BatchTableEnvImpl.scala |   4 +-
 .../flink/table/api/StreamTableEnvImpl.scala       |   4 +-
 .../flink/table/plan/schema/TableSourceTable.scala |   4 +-
 .../flink/table/sources/TableSourceUtil.scala      | 110 -------
 .../api/validation/TableSourceValidationTest.scala |  56 +++-
 16 files changed, 1199 insertions(+), 141 deletions(-)
 copy flink-runtime/src/main/java/org/apache/flink/runtime/rpc/akka/exceptions/AkkaRpcInvalidStateException.java => flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/FlinkHiveUDFException.java (66%)
 create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunction.java
 create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunctionWrapper.java
 create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveScalarFunction.java
 create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveSimpleUDF.java
 create mode 100644 flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveInspectors.java
 copy flink-core/src/main/java/org/apache/flink/api/common/Archiveable.java => flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveObjectConversion.java (80%)
 copy flink-java/src/main/java/org/apache/flink/api/java/functions/IdPartitioner.java => flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/IdentityConversion.java (72%)
 create mode 100644 flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/functions/hive/HiveSimpleUDFTest.java
 create mode 100644 flink-table/flink-table-common/src/main/java/org/apache/flink/table/sources/TableSourceValidation.java


[flink] 02/02: [FLINK-12657][hive] Integrate Flink with Hive UDF

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 3d8889e4c2b0f4b525cbd48ddf097fb5c9ee0b0d
Author: bowen.li <bo...@gmail.com>
AuthorDate: Tue Jun 11 22:13:17 2019 -0700

    [FLINK-12657][hive] Integrate Flink with Hive UDF
    
    This PR integrates Flink with Hive UDF.
    
    This closes #8700.
---
 .../functions/hive/FlinkHiveUDFException.java      |  48 ++++
 .../flink/table/functions/hive/HiveFunction.java   |  56 +++++
 .../table/functions/hive/HiveFunctionWrapper.java  |  91 ++++++++
 .../table/functions/hive/HiveScalarFunction.java   | 147 ++++++++++++
 .../flink/table/functions/hive/HiveSimpleUDF.java  | 135 +++++++++++
 .../functions/hive/conversion/HiveInspectors.java  | 250 +++++++++++++++++++++
 .../hive/conversion/HiveObjectConversion.java      |  31 +++
 .../hive/conversion/IdentityConversion.java        |  37 +++
 .../table/functions/hive/HiveSimpleUDFTest.java    | 100 +++++++++
 9 files changed, 895 insertions(+)

diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/FlinkHiveUDFException.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/FlinkHiveUDFException.java
new file mode 100644
index 0000000..4ed947e
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/FlinkHiveUDFException.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive;
+
+import org.apache.flink.util.FlinkRuntimeException;
+
+/**
+ * Hive UDF related exceptions in Flink.
+ */
+public class FlinkHiveUDFException extends FlinkRuntimeException {
+	/**
+	 * @param message the detail message.
+	 */
+	public FlinkHiveUDFException(String message) {
+		super(message);
+	}
+
+	/**
+	 * @param cause the cause.
+	 */
+	public FlinkHiveUDFException(Throwable cause) {
+		super(cause);
+	}
+
+	/**
+	 * @param message the detail message.
+	 * @param cause the cause.
+	 */
+	public FlinkHiveUDFException(String message, Throwable cause) {
+		super(message, cause);
+	}
+}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunction.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunction.java
new file mode 100644
index 0000000..3ee9b60
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunction.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.types.DataType;
+
+/**
+ * Interface for Hive simple udf, generic udf, and generic udtf.
+ * TODO: Note: this is only a temporary interface for workaround when Flink type system and udf system
+ * 	rework is not finished. Should adapt to Flink type system and Flink UDF framework later on.
+ */
+@Internal
+public interface HiveFunction {
+
+	/**
+	 * Set arguments and argTypes for Function instance.
+	 * In this way, the correct method can be really deduced by the function instance.
+	 *
+	 * @param constantArguments arguments of a function call (only literal arguments
+	 *                  are passed, nulls for non-literal ones)
+	 * @param argTypes types of arguments
+	 */
+	void setArgumentTypesAndConstants(Object[] constantArguments, DataType[] argTypes);
+
+	/**
+	 * Get result type by arguments and argTypes.
+	 *
+	 * <p>We can't use getResultType(Object[], Class[]). The Class[] is the classes of what
+	 * is defined in eval(), for example, if eval() is "public Integer eval(Double)", the
+	 * argTypes would be Class[Double]. However, in our wrapper, the signature of eval() is
+	 * "public Object eval(Object... args)", which means we cannot get any info from the interface.
+	 *
+	 * @param constantArguments arguments of a function call (only literal arguments
+	 *                  are passed, nulls for non-literal ones)
+	 * @param argTypes types of arguments
+	 * @return result type.
+	 */
+	DataType getHiveResultType(Object[] constantArguments, DataType[] argTypes);
+}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunctionWrapper.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunctionWrapper.java
new file mode 100644
index 0000000..5761247
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveFunctionWrapper.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive;
+
+import org.apache.flink.annotation.Internal;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+
+import java.io.Serializable;
+
+/**
+ * A wrapper of Hive functions that instantiate function instances and ser/de functino instance cross process boundary.
+ *
+ * @param <UDFType> The type of UDF.
+ */
+@Internal
+public class HiveFunctionWrapper<UDFType> implements Serializable {
+
+	public static final long serialVersionUID = 393313529306818205L;
+
+	private final String className;
+
+	private transient UDFType instance = null;
+
+	public HiveFunctionWrapper(String className) {
+		this.className = className;
+	}
+
+	/**
+	 * Instantiate a Hive function instance.
+	 *
+	 * @return a Hive function instance
+	 */
+	public UDFType createFunction() {
+		if (instance != null) {
+			return instance;
+		} else {
+			UDFType func = null;
+			try {
+				func = (UDFType) Thread.currentThread().getContextClassLoader().loadClass(className).newInstance();
+			} catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+				throw new FlinkHiveUDFException(
+					String.format("Failed to create function from %s", className), e);
+			}
+
+			if (!(func instanceof UDF)) {
+				// We cache the function if it is not the Simple UDF,
+				// as we always have to create new instance for Simple UDF.
+				instance = func;
+			}
+
+			return func;
+		}
+	}
+
+	/**
+	 * Get class name of the Hive function.
+	 *
+	 * @return class name of the Hive function
+	 */
+	public String getClassName() {
+		return className;
+	}
+
+	/**
+	 * Get class of the Hive function.
+	 *
+	 * @return class of the Hive function
+	 * @throws ClassNotFoundException thrown when the class is not found in classpath
+	 */
+	public Class<UDFType> getUDFClass() throws ClassNotFoundException {
+		return (Class<UDFType>) Class.forName(className);
+	}
+}
+
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveScalarFunction.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveScalarFunction.java
new file mode 100644
index 0000000..3058139
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveScalarFunction.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.table.functions.FunctionContext;
+import org.apache.flink.table.functions.ScalarFunction;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.logical.ArrayType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
+import org.apache.flink.table.types.utils.LegacyTypeInfoDataTypeConverter;
+
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+/**
+ * Abstract class to provide more information for Hive {@link UDF} and {@link GenericUDF} functions.
+ */
+@Internal
+public abstract class HiveScalarFunction<UDFType> extends ScalarFunction implements HiveFunction {
+
+	protected final HiveFunctionWrapper<UDFType> hiveFunctionWrapper;
+
+	protected Object[] constantArguments;
+	protected DataType[] argTypes;
+
+	protected transient UDFType function;
+	protected transient ObjectInspector returnInspector;
+
+	private transient boolean isArgsSingleArray;
+
+	HiveScalarFunction(HiveFunctionWrapper<UDFType> hiveFunctionWrapper) {
+		this.hiveFunctionWrapper = hiveFunctionWrapper;
+	}
+
+	@Override
+	public void setArgumentTypesAndConstants(Object[] constantArguments, DataType[] argTypes) {
+		this.constantArguments = constantArguments;
+		this.argTypes = argTypes;
+	}
+
+	@Override
+	public boolean isDeterministic() {
+		try {
+			org.apache.hadoop.hive.ql.udf.UDFType udfType =
+				hiveFunctionWrapper.getUDFClass()
+					.getAnnotation(org.apache.hadoop.hive.ql.udf.UDFType.class);
+
+			return udfType != null && udfType.deterministic() && !udfType.stateful();
+		} catch (ClassNotFoundException e) {
+			throw new FlinkHiveUDFException(e);
+		}
+	}
+
+	@Override
+	public TypeInformation getResultType(Class[] signature) {
+		return LegacyTypeInfoDataTypeConverter.toLegacyTypeInfo(
+			getHiveResultType(this.constantArguments, this.argTypes));
+	}
+
+	@Override
+	public void open(FunctionContext context) {
+		openInternal();
+
+		for (DataType dataType : argTypes) {
+			if (isPrimitiveArray(dataType)) {
+				throw new FlinkHiveUDFException("Flink doesn't support primitive array for Hive functions yet.");
+			}
+		}
+
+		isArgsSingleArray = argTypes.length == 1 && isArrayType(argTypes[0]);
+	}
+
+	private static boolean isPrimitiveArray(DataType dataType) {
+		if (isArrayType(dataType)) {
+			ArrayType arrayType = (ArrayType) dataType.getLogicalType();
+
+			LogicalType elementType = arrayType.getElementType();
+			return !(elementType.isNullable() || !isPrimitive(elementType));
+		} else {
+			return false;
+		}
+	}
+
+	private static boolean isArrayType(DataType dataType) {
+		return dataType.getLogicalType().getTypeRoot().equals(LogicalTypeRoot.ARRAY);
+	}
+
+	// This is copied from PlannerTypeUtils in flink-table-runtime-blink that we shouldn't depend on
+	// TODO: remove this and use the original code when it's moved to accessible, dependable module
+	private static boolean isPrimitive(LogicalType type) {
+		switch (type.getTypeRoot()) {
+			case BOOLEAN:
+			case TINYINT:
+			case SMALLINT:
+			case INTEGER:
+			case BIGINT:
+			case FLOAT:
+			case DOUBLE:
+				return true;
+			default:
+				return false;
+		}
+	}
+
+	/**
+	 * See {@link ScalarFunction#open(FunctionContext)}.
+	 */
+	protected abstract void openInternal();
+
+	public Object eval(Object... args) {
+
+		// When the parameter is (Integer, Array[Double]), Flink calls udf.eval(Integer, Array[Double]), which is not a problem.
+		// But when the parameter is an single array, Flink calls udf.eval(Array[Double]),
+		// at this point java's var-args will cast Array[Double] to Array[Object] and let it be
+		// Object... args, So we need wrap it.
+		if (isArgsSingleArray) {
+			args = new Object[] {args};
+		}
+
+		return evalInternal(args);
+	}
+
+	/**
+	 * Evaluation logical, args will be wrapped when is a single array.
+	 */
+	protected abstract Object evalInternal(Object[] args);
+}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveSimpleUDF.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveSimpleUDF.java
new file mode 100644
index 0000000..98c6025
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/HiveSimpleUDF.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.catalog.hive.util.HiveTypeUtil;
+import org.apache.flink.table.functions.hive.conversion.HiveInspectors;
+import org.apache.flink.table.functions.hive.conversion.HiveObjectConversion;
+import org.apache.flink.table.functions.hive.conversion.IdentityConversion;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.flink.util.Preconditions.checkArgument;
+
+/**
+ *  A ScalarFunction implementation that calls Hive's {@link UDF}.
+ */
+@Internal
+public class HiveSimpleUDF extends HiveScalarFunction<UDF> {
+
+	private static final Logger LOG = LoggerFactory.getLogger(HiveSimpleUDF.class);
+
+	private transient Method method;
+	private transient GenericUDFUtils.ConversionHelper conversionHelper;
+	private transient HiveObjectConversion[] conversions;
+	private transient boolean allIdentityConverter;
+
+	public HiveSimpleUDF(HiveFunctionWrapper<UDF> hiveFunctionWrapper) {
+		super(hiveFunctionWrapper);
+		LOG.info("Creating HiveSimpleUDF from '{}'", this.hiveFunctionWrapper.getClassName());
+	}
+
+	@Override
+	public void openInternal() {
+		LOG.info("Opening HiveSimpleUDF as '{}'", hiveFunctionWrapper.getClassName());
+
+		function = hiveFunctionWrapper.createFunction();
+
+		List<TypeInfo> typeInfos = new ArrayList<>();
+
+		for (DataType arg : argTypes) {
+			typeInfos.add(HiveTypeUtil.toHiveTypeInfo(arg));
+		}
+
+		try {
+			method = function.getResolver().getEvalMethod(typeInfos);
+			returnInspector = ObjectInspectorFactory.getReflectionObjectInspector(method.getGenericReturnType(),
+				ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
+			ObjectInspector[] argInspectors = new ObjectInspector[typeInfos.size()];
+
+			for (int i = 0; i < argTypes.length; i++) {
+				argInspectors[i] = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfos.get(i));
+			}
+
+			conversionHelper = new GenericUDFUtils.ConversionHelper(method, argInspectors);
+			conversions = new HiveObjectConversion[argInspectors.length];
+			for (int i = 0; i < argInspectors.length; i++) {
+				conversions[i] = HiveInspectors.getConversion(argInspectors[i]);
+			}
+
+			allIdentityConverter = Arrays.stream(conversions)
+				.allMatch(conv -> conv instanceof IdentityConversion);
+		} catch (Exception e) {
+			throw new FlinkHiveUDFException(
+				String.format("Failed to open HiveSimpleUDF from %s", hiveFunctionWrapper.getClassName()), e);
+		}
+	}
+
+	@Override
+	public Object evalInternal(Object[] args) {
+		checkArgument(args.length == conversions.length);
+
+		if (!allIdentityConverter) {
+			for (int i = 0; i < args.length; i++) {
+				args[i] = conversions[i].toHiveObject(args[i]);
+			}
+		}
+
+		try {
+			return HiveInspectors.toFlinkObject(returnInspector,
+				FunctionRegistry.invoke(method, function, conversionHelper.convertIfNecessary(args)));
+		} catch (HiveException e) {
+			throw new FlinkHiveUDFException(e);
+		}
+	}
+
+	@Override
+	public DataType getHiveResultType(Object[] constantArguments, DataType[] argTypes) {
+		try {
+			List<TypeInfo> argTypeInfo = new ArrayList<>();
+			for (DataType argType : argTypes) {
+				argTypeInfo.add(HiveTypeUtil.toHiveTypeInfo(argType));
+			}
+			Class returnType = hiveFunctionWrapper.createFunction()
+				.getResolver().getEvalMethod(argTypeInfo).getReturnType();
+
+			return HiveInspectors.toFlinkType(
+				HiveInspectors.getObjectInspector(returnType));
+		} catch (UDFArgumentException e) {
+			throw new FlinkHiveUDFException(e);
+		}
+	}
+}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveInspectors.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveInspectors.java
new file mode 100644
index 0000000..a5d0c45
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveInspectors.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive.conversion;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.table.catalog.hive.util.HiveTypeUtil;
+import org.apache.flink.table.functions.hive.FlinkHiveUDFException;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+
+/**
+ * Util for any ObjectInspector related inspection and conversion of Hive data to/from Flink data.
+ *
+ * <p>Hive ObjectInspector is a group of flexible APIs to inspect value in different data representation,
+ * and developers can extend those API as needed, so technically, object inspector supports arbitrary data type in java.
+ */
+@Internal
+public class HiveInspectors {
+
+	/**
+	 * Get conversion for converting Flink object to Hive object from an ObjectInspector.
+	 */
+	public static HiveObjectConversion getConversion(ObjectInspector inspector) {
+		if (inspector instanceof PrimitiveObjectInspector) {
+			if (inspector instanceof JavaBooleanObjectInspector) {
+				if (((JavaBooleanObjectInspector) inspector).preferWritable()) {
+					return o -> new BooleanWritable((Boolean) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaStringObjectInspector) {
+				if (((StringObjectInspector) inspector).preferWritable()) {
+					return o -> new Text((String) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaByteObjectInspector) {
+				if (((JavaByteObjectInspector) inspector).preferWritable()) {
+					return o -> new ByteWritable((Byte) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaShortObjectInspector) {
+				if (((JavaShortObjectInspector) inspector).preferWritable()) {
+					return o -> new ShortWritable((Short) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaIntObjectInspector) {
+				if (((JavaIntObjectInspector) inspector).preferWritable()) {
+					return o -> new IntWritable((Integer) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaLongObjectInspector) {
+				if (((JavaLongObjectInspector) inspector).preferWritable()) {
+					return o -> new LongWritable((Long) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaFloatObjectInspector) {
+				if (((JavaFloatObjectInspector) inspector).preferWritable()) {
+					return o -> new FloatWritable((Float) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			} else if (inspector instanceof JavaDoubleObjectInspector) {
+				if (((JavaDoubleObjectInspector) inspector).preferWritable()) {
+					return o -> new DoubleWritable((Double) o);
+				} else {
+					return IdentityConversion.INSTANCE;
+				}
+			}
+		}
+
+		throw new FlinkHiveUDFException(
+			String.format("Flink doesn't support convert object conversion for %s yet", inspector));
+	}
+
+	/**
+	 * Converts a Hive object to Flink object with an ObjectInspector.
+	 */
+	public static Object toFlinkObject(ObjectInspector inspector, Object data) {
+		if (data == null) {
+			return null;
+		}
+
+		if (inspector instanceof VoidObjectInspector) {
+			return null;
+		}
+
+		if (inspector instanceof PrimitiveObjectInspector) {
+			if (inspector instanceof BooleanObjectInspector) {
+				BooleanObjectInspector oi = (BooleanObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveJavaObject(data);
+			} else if (inspector instanceof StringObjectInspector) {
+				StringObjectInspector oi = (StringObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.getPrimitiveWritableObject(data).toString() :
+					oi.getPrimitiveJavaObject(data);
+			} else if (inspector instanceof ByteObjectInspector) {
+				ByteObjectInspector oi = (ByteObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveJavaObject(data);
+			} else if (inspector instanceof ShortObjectInspector) {
+				ShortObjectInspector oi = (ShortObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveWritableObject(data);
+			} else if (inspector instanceof IntObjectInspector) {
+				IntObjectInspector oi = (IntObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveWritableObject(data);
+			} else if (inspector instanceof LongObjectInspector) {
+				LongObjectInspector oi = (LongObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveWritableObject(data);
+			} else if (inspector instanceof FloatObjectInspector) {
+				FloatObjectInspector oi = (FloatObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveWritableObject(data);
+			} else if (inspector instanceof DoubleObjectInspector) {
+				DoubleObjectInspector oi = (DoubleObjectInspector) inspector;
+
+				return oi.preferWritable() ?
+					oi.get(data) :
+					oi.getPrimitiveWritableObject(data);
+			}
+
+			// TODO: handle more primitive types like char, varchar, timestamp, date, decimal
+		}
+
+		// TODO: handle complex types like struct, list, and map
+
+		throw new FlinkHiveUDFException(
+			String.format("Unwrap does not support ObjectInspector '%s' yet", inspector));
+	}
+
+	public static ObjectInspector getObjectInspector(Class clazz) {
+		TypeInfo typeInfo;
+
+		if (clazz.equals(String.class) || clazz.equals(Text.class)) {
+
+			typeInfo = TypeInfoFactory.stringTypeInfo;
+		} else if (clazz.equals(Boolean.class) || clazz.equals(BooleanWritable.class)) {
+
+			typeInfo = TypeInfoFactory.booleanTypeInfo;
+		} else if (clazz.equals(Byte.class) || clazz.equals(ByteWritable.class)) {
+
+			typeInfo = TypeInfoFactory.byteTypeInfo;
+		} else if (clazz.equals(Short.class) || clazz.equals(ShortWritable.class)) {
+
+			typeInfo = TypeInfoFactory.shortTypeInfo;
+		} else if (clazz.equals(Integer.class) || clazz.equals(IntWritable.class)) {
+
+			typeInfo = TypeInfoFactory.intTypeInfo;
+		} else if (clazz.equals(Long.class) || clazz.equals(LongWritable.class)) {
+
+			typeInfo = TypeInfoFactory.longTypeInfo;
+		} else if (clazz.equals(Float.class) || clazz.equals(FloatWritable.class)) {
+
+			typeInfo = TypeInfoFactory.floatTypeInfo;
+		} else if (clazz.equals(Double.class) || clazz.equals(DoubleWritable.class)) {
+
+			typeInfo = TypeInfoFactory.doubleTypeInfo;
+		} else {
+			throw new FlinkHiveUDFException(
+				String.format("Class %s is not supported yet", clazz.getName()));
+		}
+
+		return getObjectInspector(typeInfo);
+	}
+
+	private static ObjectInspector getObjectInspector(TypeInfo type) {
+		switch (type.getCategory()) {
+			case PRIMITIVE:
+				PrimitiveTypeInfo primitiveType = (PrimitiveTypeInfo) type;
+				return PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(primitiveType);
+			default:
+				throw new FlinkHiveUDFException(
+					String.format("TypeInfo %s is not supported yet", type));
+		}
+	}
+
+	public static DataType toFlinkType(ObjectInspector inspector) {
+		return HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(inspector.getTypeName()));
+	}
+}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveObjectConversion.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveObjectConversion.java
new file mode 100644
index 0000000..cbf9ffd
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/HiveObjectConversion.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive.conversion;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Interface to convert Flink object to Hive object.
+ */
+@FunctionalInterface
+@Internal
+public interface HiveObjectConversion {
+
+	Object toHiveObject(Object o);
+}
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/IdentityConversion.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/IdentityConversion.java
new file mode 100644
index 0000000..ba69785
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/table/functions/hive/conversion/IdentityConversion.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive.conversion;
+
+import org.apache.flink.annotation.Internal;
+
+/**
+ * Conversion that return the same object.
+ */
+@Internal
+public class IdentityConversion implements HiveObjectConversion {
+
+	public static final IdentityConversion INSTANCE = new IdentityConversion();
+
+	private IdentityConversion() {}
+
+	@Override
+	public Object toHiveObject(Object o) {
+		return o;
+	}
+}
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/functions/hive/HiveSimpleUDFTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/functions/hive/HiveSimpleUDFTest.java
new file mode 100644
index 0000000..25c331f
--- /dev/null
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/functions/hive/HiveSimpleUDFTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.functions.hive;
+
+import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.types.DataType;
+
+import org.apache.hadoop.hive.ql.udf.UDFBin;
+import org.apache.hadoop.hive.ql.udf.UDFJson;
+import org.apache.hadoop.hive.ql.udf.UDFMinute;
+import org.apache.hadoop.hive.ql.udf.UDFRand;
+import org.apache.hadoop.hive.ql.udf.UDFWeekOfYear;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test for {@link HiveSimpleUDF}.
+ */
+public class HiveSimpleUDFTest {
+
+	@Test
+	public void testUDFRand() {
+		HiveSimpleUDF udf = init(UDFRand.class, new DataType[0]);
+
+		double result = (double) udf.eval();
+
+		assertTrue(result >= 0 && result < 1);
+	}
+
+	@Test
+	public void testUDFBin() {
+		HiveSimpleUDF udf = init(UDFBin.class, new DataType[]{ DataTypes.INT() });
+
+		assertEquals("1100", udf.eval(12));
+	}
+
+	@Test
+	public void testUDFJson() {
+		HiveSimpleUDF udf = init(
+			UDFJson.class,
+			new DataType[]{
+				DataTypes.STRING(),
+				DataTypes.STRING()
+			});
+
+		assertEquals("amy", udf.eval("{\"store\": \"test\", \"owner\": \"amy\"}", "$.owner"));
+	}
+
+	@Test
+	public void testUDFMinute() {
+		HiveSimpleUDF udf = init(
+			UDFMinute.class,
+			new DataType[]{
+				DataTypes.STRING()
+			});
+
+		assertEquals(17, udf.eval("1969-07-20 20:17:40"));
+	}
+
+	@Test
+	public void testUDFWeekOfYear() {
+		HiveSimpleUDF udf = init(
+			UDFWeekOfYear.class,
+			new DataType[]{
+				DataTypes.STRING()
+			});
+
+		assertEquals(29, udf.eval("1969-07-20"));
+	}
+
+	private HiveSimpleUDF init(Class hiveUdfClass, DataType[] argTypes) {
+		HiveSimpleUDF udf = new HiveSimpleUDF(new HiveFunctionWrapper(hiveUdfClass.getName()));
+
+		// Hive UDF won't have literal args
+		udf.setArgumentTypesAndConstants(new Object[0], argTypes);
+		udf.getHiveResultType(new Object[0], argTypes);
+
+		udf.open(null);
+
+		return udf;
+	}
+}


[flink] 01/02: [FLINK-12853][table-api][table-planner] TableSourceUtils#validateTableSource method to common module

Posted by bl...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

bli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git

commit 8131c406616a75512c50aaed14d2133875a61e34
Author: Dawid Wysakowicz <dw...@apache.org>
AuthorDate: Fri Jun 14 13:15:09 2019 +0200

    [FLINK-12853][table-api][table-planner] TableSourceUtils#validateTableSource method to common module
    
    This closes #8739
---
 .../kafka/KafkaTableSourceSinkFactoryTestBase.java |   4 +-
 .../flink/table/sources/TableSourceValidation.java | 325 +++++++++++++++++++++
 .../apache/flink/table/api/BatchTableEnvImpl.scala |   4 +-
 .../flink/table/api/StreamTableEnvImpl.scala       |   4 +-
 .../flink/table/plan/schema/TableSourceTable.scala |   4 +-
 .../flink/table/sources/TableSourceUtil.scala      | 110 -------
 .../api/validation/TableSourceValidationTest.scala |  56 +++-
 7 files changed, 387 insertions(+), 120 deletions(-)

diff --git a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
index f28b9f9..4a036cb 100644
--- a/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
+++ b/flink-connectors/flink-connector-kafka-base/src/test/java/org/apache/flink/streaming/connectors/kafka/KafkaTableSourceSinkFactoryTestBase.java
@@ -49,7 +49,7 @@ import org.apache.flink.table.factories.utils.TestTableFormat;
 import org.apache.flink.table.sinks.TableSink;
 import org.apache.flink.table.sources.RowtimeAttributeDescriptor;
 import org.apache.flink.table.sources.TableSource;
-import org.apache.flink.table.sources.TableSourceUtil;
+import org.apache.flink.table.sources.TableSourceValidation;
 import org.apache.flink.table.sources.tsextractors.ExistingField;
 import org.apache.flink.table.sources.wmstrategies.AscendingTimestamps;
 import org.apache.flink.types.Row;
@@ -143,7 +143,7 @@ public abstract class KafkaTableSourceSinkFactoryTestBase extends TestLogger {
 			StartupMode.SPECIFIC_OFFSETS,
 			specificOffsets);
 
-		TableSourceUtil.validateTableSource(expected);
+		TableSourceValidation.validateTableSource(expected);
 
 		// construct table source using descriptors and table source factory
 
diff --git a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/sources/TableSourceValidation.java b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/sources/TableSourceValidation.java
new file mode 100644
index 0000000..9b94dad
--- /dev/null
+++ b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/sources/TableSourceValidation.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.sources;
+
+import org.apache.flink.annotation.Internal;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.common.typeutils.CompositeType;
+import org.apache.flink.table.api.TableSchema;
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.types.AtomicDataType;
+import org.apache.flink.table.types.DataType;
+import org.apache.flink.table.types.FieldsDataType;
+import org.apache.flink.table.types.logical.LegacyTypeInformationType;
+import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeFamily;
+import org.apache.flink.table.types.utils.DataTypeDefaultVisitor;
+import org.apache.flink.table.types.utils.TypeConversions;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.flink.table.types.logical.utils.LogicalTypeChecks.hasFamily;
+
+/**
+ * Logic to validate {@link TableSource} types.
+ */
+@Internal
+public class TableSourceValidation {
+
+	/**
+	 * Validates a TableSource.
+	 *
+	 * <ul>
+	 * <li>checks that all fields of the schema can be resolved</li>
+	 * <li>checks that resolved fields have the correct type</li>
+	 * <li>checks that the time attributes are correctly configured.</li>
+	 * </ul>
+	 *
+	 * @param tableSource The {@link TableSource} for which the time attributes are checked.
+	 */
+	public static void validateTableSource(TableSource<?> tableSource){
+		TableSchema schema = tableSource.getTableSchema();
+
+		List<RowtimeAttributeDescriptor> rowtimeAttributes = getRowtimeAttributes(tableSource);
+		Optional<String> proctimeAttribute = getProctimeAttribute(tableSource);
+
+		validateSingleRowtimeAttribute(rowtimeAttributes);
+		validateRowtimeAttributesExistInSchema(rowtimeAttributes, schema);
+		validateProctimeAttributesExistInSchema(proctimeAttribute, schema);
+		validateLogicalToPhysicalMapping(tableSource, schema, rowtimeAttributes, proctimeAttribute);
+		validateTimestampExtractorArguments(rowtimeAttributes, tableSource);
+		validateNotOverlapping(rowtimeAttributes, proctimeAttribute);
+	}
+
+	private static void validateSingleRowtimeAttribute(List<RowtimeAttributeDescriptor> rowtimeAttributes) {
+		if (rowtimeAttributes.size() > 1) {
+			throw new ValidationException("Currently, only a single rowtime attribute is supported. " +
+				"Please remove all but one RowtimeAttributeDescriptor.");
+		}
+	}
+
+	private static void validateRowtimeAttributesExistInSchema(
+			List<RowtimeAttributeDescriptor> rowtimeAttributes,
+			TableSchema tableSchema) {
+		rowtimeAttributes.forEach(r -> {
+				if (!tableSchema.getFieldDataType(r.getAttributeName()).isPresent()) {
+					throw new ValidationException(String.format(
+						"Found a rowtime attribute for field '%s' but it does not exist in the Table. TableSchema: %s",
+						r.getAttributeName(),
+						tableSchema));
+				}
+			}
+		);
+	}
+
+	private static void validateProctimeAttributesExistInSchema(
+			Optional<String> proctimeAttribute,
+			TableSchema tableSchema) {
+		proctimeAttribute.ifPresent(r -> {
+				if (!tableSchema.getFieldDataType(r).isPresent()) {
+					throw new ValidationException(String.format(
+						"Found a proctime attribute for field '%s' but it does not exist in the Table. TableSchema: %s",
+						r,
+						tableSchema));
+				}
+			}
+		);
+	}
+
+	private static void validateNotOverlapping(
+			List<RowtimeAttributeDescriptor> rowtimeAttributes,
+			Optional<String> proctimeAttribute) {
+		proctimeAttribute.ifPresent(proctime -> {
+				if (rowtimeAttributes.stream()
+					.anyMatch(rowtimeAttribute -> rowtimeAttribute.getAttributeName().equals(proctime))) {
+					throw new ValidationException(String.format(
+						"Field '%s' must not be processing time and rowtime attribute at the same time.",
+						proctime));
+				}
+			}
+		);
+	}
+
+	private static void validateLogicalToPhysicalMapping(
+			TableSource<?> tableSource,
+			TableSchema schema,
+			List<RowtimeAttributeDescriptor> rowtimeAttributes,
+			Optional<String> proctimeAttribute) {
+		// validate that schema fields can be resolved to a return type field of correct type
+		int mappedFieldCnt = 0;
+		for (int i = 0; i < schema.getFieldCount(); i++) {
+			DataType fieldType = schema.getFieldDataType(i).get();
+			LogicalType logicalFieldType = fieldType.getLogicalType();
+			String fieldName = schema.getFieldName(i).get();
+
+			if (proctimeAttribute.map(p -> p.equals(fieldName)).orElse(false)) {
+				if (!(hasFamily(logicalFieldType, LogicalTypeFamily.TIMESTAMP))) {
+					throw new ValidationException(String.format("Processing time field '%s' has invalid type %s. " +
+						"Processing time attributes must be of type SQL_TIMESTAMP.", fieldName, logicalFieldType));
+				}
+			} else if (rowtimeAttributes.stream().anyMatch(p -> p.getAttributeName().equals(fieldName))) {
+				if (!(hasFamily(logicalFieldType, LogicalTypeFamily.TIMESTAMP))) {
+					throw new ValidationException(String.format("Rowtime time field '%s' has invalid type %s. " +
+						"Rowtime time attributes must be of type SQL_TIMESTAMP.", fieldName, logicalFieldType));
+				}
+			} else {
+				validateLogicalTypeEqualsPhysical(fieldName, fieldType, tableSource);
+				mappedFieldCnt += 1;
+			}
+		}
+
+		// ensure that only one field is mapped to an atomic type
+		DataType producedDataType = tableSource.getProducedDataType();
+		if (!isCompositeType(producedDataType) && mappedFieldCnt > 1) {
+			throw new ValidationException(
+				String.format(
+					"More than one table field matched to atomic input type %s.",
+					producedDataType));
+		}
+	}
+
+	private static boolean isCompositeType(DataType producedDataType) {
+		LogicalType logicalType = producedDataType.getLogicalType();
+		return producedDataType instanceof FieldsDataType ||
+			(logicalType instanceof LegacyTypeInformationType &&
+				((LegacyTypeInformationType) logicalType).getTypeInformation() instanceof CompositeType);
+	}
+
+	private static void validateLogicalTypeEqualsPhysical(
+			String fieldName,
+			DataType logicalType,
+			TableSource<?> tableSource) {
+		ResolvedField resolvedField = resolveField(fieldName, tableSource);
+		if (!resolvedField.getType().equals(logicalType)) {
+			throw new ValidationException(String.format(
+				"Type %s of table field '%s' does not " +
+					"match with type '%s; of the field '%s' of the TableSource return type.",
+				logicalType,
+				resolvedField.getType(),
+				fieldName,
+				resolvedField.getType()));
+		}
+	}
+
+	private static void validateTimestampExtractorArguments(
+			List<RowtimeAttributeDescriptor> descriptors,
+			TableSource<?> tableSource) {
+		if (descriptors.size() == 1) {
+			RowtimeAttributeDescriptor descriptor = descriptors.get(0);
+			// look up extractor input fields in return type
+			String[] extractorInputFields = descriptor.getTimestampExtractor().getArgumentFields();
+			TypeInformation[] physicalTypes = Arrays.stream(extractorInputFields)
+				.map(fieldName -> resolveField(fieldName, tableSource))
+				.map(resolvedField -> TypeConversions.fromDataTypeToLegacyInfo(resolvedField.getType()))
+				.toArray(TypeInformation[]::new);
+			// validate timestamp extractor
+			descriptor.getTimestampExtractor().validateArgumentFields(physicalTypes);
+		}
+	}
+
+	private static class ResolvedField {
+		private final String name;
+		private final DataType type;
+
+		private ResolvedField(String name, DataType type) {
+			this.type = type;
+			this.name = name;
+		}
+
+		public DataType getType() {
+			return type;
+		}
+
+		public String getName() {
+			return name;
+		}
+	}
+
+	/**
+	 * Identifies for a field name of the logical schema, the corresponding physical field in the
+	 * return type of a {@link TableSource}.
+	 *
+	 * @param fieldName The logical field to look up.
+	 * @param tableSource The table source in which to look for the field.
+	 * @return The name, index, and type information of the physical field.
+	 */
+	private static ResolvedField resolveField(String fieldName, TableSource<?> tableSource) {
+
+		DataType producedDataType = tableSource.getProducedDataType();
+
+		if (tableSource instanceof DefinedFieldMapping) {
+			Map<String, String> fieldMapping = ((DefinedFieldMapping) tableSource).getFieldMapping();
+			if (fieldMapping != null) {
+				String resolvedFieldName = fieldMapping.get(fieldName);
+				if (resolvedFieldName == null) {
+					throw new ValidationException(String.format(
+						"Field '%s' could not be resolved by the field mapping.",
+						fieldName));
+				}
+
+				return new ResolvedField(
+					resolvedFieldName,
+					lookupFieldType(
+						producedDataType,
+						resolvedFieldName,
+						String.format(
+							"Table field '%s' was resolved to TableSource return type field " +
+								"'%s', but field '%s' was not found in the return " +
+								"type %s of the TableSource. " +
+								"Please verify the field mapping of the TableSource.",
+							fieldName,
+							resolvedFieldName,
+							resolvedFieldName,
+							producedDataType)));
+			}
+		}
+
+		return new ResolvedField(
+			fieldName,
+			lookupFieldType(
+				producedDataType,
+				fieldName,
+				String.format("Table field '%s' was not found in the return type $returnType of the " +
+					"TableSource.", fieldName)));
+	}
+
+	/** Look up a field by name in a {@link DataType}. */
+	private static DataType lookupFieldType(DataType inputType, String fieldName, String failMsg) {
+		return inputType.accept(new TypeExtractor(fieldName)).orElseThrow(() -> new ValidationException(failMsg));
+	}
+
+	private static class TypeExtractor extends DataTypeDefaultVisitor<Optional<DataType>> {
+		private final String fieldName;
+
+		TypeExtractor(String fieldName) {
+			this.fieldName = fieldName;
+		}
+
+		@Override
+		public Optional<DataType> visit(AtomicDataType atomicDataType) {
+			//  This is check for backwards compatibility. We should also support legacy type with composite type info
+			LogicalType logicalType = atomicDataType.getLogicalType();
+			if (logicalType instanceof LegacyTypeInformationType) {
+				LegacyTypeInformationType<?> legacyTypeInformationType = (LegacyTypeInformationType<?>) logicalType;
+				TypeInformation<?> typeInformation = legacyTypeInformationType.getTypeInformation();
+				if (typeInformation instanceof CompositeType<?>) {
+					CompositeType<?> compositeType = (CompositeType<?>) typeInformation;
+					return Optional.of(TypeConversions.fromLegacyInfoToDataType(compositeType.getTypeAt(fieldName)));
+				}
+			}
+
+			return Optional.of(atomicDataType);
+		}
+
+		@Override
+		public Optional<DataType> visit(FieldsDataType fieldsDataType) {
+			return Optional.ofNullable(fieldsDataType.getFieldDataTypes().get(fieldName));
+		}
+
+		@Override
+		protected Optional<DataType> defaultMethod(DataType dataType) {
+			return Optional.of(dataType);
+		}
+	}
+
+	/** Returns a list with all rowtime attribute descriptors of the {@link TableSource}. */
+	private static List<RowtimeAttributeDescriptor> getRowtimeAttributes(TableSource<?> tableSource) {
+		if (tableSource instanceof DefinedRowtimeAttributes) {
+			return ((DefinedRowtimeAttributes) tableSource).getRowtimeAttributeDescriptors();
+		}
+
+		return Collections.emptyList();
+	}
+
+	/** Returns the proctime attribute of the {@link TableSource} if it is defined. */
+	private static Optional<String> getProctimeAttribute(TableSource<?> tableSource) {
+		if (tableSource instanceof DefinedProctimeAttribute) {
+			return Optional.ofNullable(((DefinedProctimeAttribute) tableSource).getProctimeAttribute());
+		}
+
+		return Optional.empty();
+	}
+
+	private TableSourceValidation() {
+	}
+}
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/BatchTableEnvImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/BatchTableEnvImpl.scala
index 6073598..e9b4d85 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/BatchTableEnvImpl.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/BatchTableEnvImpl.scala
@@ -37,7 +37,7 @@ import org.apache.flink.table.plan.nodes.dataset.DataSetRel
 import org.apache.flink.table.planner.Conversions
 import org.apache.flink.table.runtime.MapRunner
 import org.apache.flink.table.sinks._
-import org.apache.flink.table.sources.{BatchTableSource, InputFormatTableSource, TableSource, TableSourceUtil}
+import org.apache.flink.table.sources.{BatchTableSource, InputFormatTableSource, TableSource, TableSourceValidation}
 import org.apache.flink.table.types.utils.TypeConversions
 import org.apache.flink.table.types.utils.TypeConversions.fromDataTypeToLegacyInfo
 import org.apache.flink.table.typeutils.FieldInfoUtils.{getFieldsInfo, validateInputTypeInfo}
@@ -72,7 +72,7 @@ abstract class BatchTableEnvImpl(
     * @param tableSource The [[TableSource]] to register.
     */
   override protected def validateTableSource(tableSource: TableSource[_]): Unit = {
-    TableSourceUtil.validateTableSource(tableSource)
+    TableSourceValidation.validateTableSource(tableSource)
 
     if (!tableSource.isInstanceOf[BatchTableSource[_]] &&
         !tableSource.isInstanceOf[InputFormatTableSource[_]]) {
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/StreamTableEnvImpl.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/StreamTableEnvImpl.scala
index 1ee3d4a..27e1c06 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/StreamTableEnvImpl.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/api/StreamTableEnvImpl.scala
@@ -40,7 +40,7 @@ import org.apache.flink.table.plan.util.UpdatingPlanChecker
 import org.apache.flink.table.planner.DataStreamConversions
 import org.apache.flink.table.runtime.types.CRow
 import org.apache.flink.table.sinks._
-import org.apache.flink.table.sources.{StreamTableSource, TableSource, TableSourceUtil}
+import org.apache.flink.table.sources.{StreamTableSource, TableSource, TableSourceUtil, TableSourceValidation}
 import org.apache.flink.table.types.utils.TypeConversions
 import org.apache.flink.table.types.utils.TypeConversions.fromDataTypeToLegacyInfo
 import org.apache.flink.table.typeutils.FieldInfoUtils.getFieldsInfo
@@ -76,7 +76,7 @@ abstract class StreamTableEnvImpl(
     */
   override protected def validateTableSource(tableSource: TableSource[_]): Unit = {
 
-    TableSourceUtil.validateTableSource(tableSource)
+    TableSourceValidation.validateTableSource(tableSource)
     tableSource match {
 
       // check for proper stream table source
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/schema/TableSourceTable.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/schema/TableSourceTable.scala
index 9dab249..daeda7b 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/schema/TableSourceTable.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/plan/schema/TableSourceTable.scala
@@ -23,7 +23,7 @@ import org.apache.calcite.schema.Statistic
 import org.apache.calcite.schema.impl.AbstractTable
 import org.apache.flink.table.calcite.FlinkTypeFactory
 import org.apache.flink.table.plan.stats.FlinkStatistic
-import org.apache.flink.table.sources.{TableSource, TableSourceUtil}
+import org.apache.flink.table.sources.{TableSource, TableSourceUtil, TableSourceValidation}
 
 /**
   * Abstract class which define the interfaces required to convert a [[TableSource]] to
@@ -35,7 +35,7 @@ class TableSourceTable[T](
     val statistic: FlinkStatistic)
   extends AbstractTable {
 
-  TableSourceUtil.validateTableSource(tableSource)
+  TableSourceValidation.validateTableSource(tableSource)
 
   /**
     * Returns statistics of current table
diff --git a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
index 4173188..ccc4733 100644
--- a/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
+++ b/flink-table/flink-table-planner/src/main/scala/org/apache/flink/table/sources/TableSourceUtil.scala
@@ -49,116 +49,6 @@ object TableSourceUtil {
     getProctimeAttribute(tableSource).nonEmpty
 
   /**
-    * Validates a TableSource.
-    *
-    * - checks that all fields of the schema can be resolved
-    * - checks that resolved fields have the correct type
-    * - checks that the time attributes are correctly configured.
-    *
-    * @param tableSource The [[TableSource]] for which the time attributes are checked.
-    */
-  def validateTableSource(tableSource: TableSource[_]): Unit = {
-
-    val schema = tableSource.getTableSchema
-    val tableFieldNames = schema.getFieldNames
-    val tableFieldTypes = schema.getFieldTypes
-
-    // get rowtime and proctime attributes
-    val rowtimeAttributes = getRowtimeAttributes(tableSource)
-    val proctimeAttribute = getProctimeAttribute(tableSource)
-
-    // validate that schema fields can be resolved to a return type field of correct type
-    var mappedFieldCnt = 0
-    tableFieldTypes.zip(tableFieldNames).foreach {
-      case (t: SqlTimeTypeInfo[_], name: String)
-        if t.getTypeClass == classOf[Timestamp] && proctimeAttribute.contains(name) =>
-        // OK, field was mapped to proctime attribute
-      case (t: SqlTimeTypeInfo[_], name: String)
-        if t.getTypeClass == classOf[Timestamp] && rowtimeAttributes.contains(name) =>
-        // OK, field was mapped to rowtime attribute
-      case (t: TypeInformation[_], name) =>
-        // check if field is registered as time indicator
-        if (getProctimeAttribute(tableSource).contains(name)) {
-          throw new ValidationException(s"Processing time field '$name' has invalid type $t. " +
-            s"Processing time attributes must be of type ${Types.SQL_TIMESTAMP}.")
-        }
-        if (getRowtimeAttributes(tableSource).contains(name)) {
-          throw new ValidationException(s"Rowtime field '$name' has invalid type $t. " +
-            s"Rowtime attributes must be of type ${Types.SQL_TIMESTAMP}.")
-        }
-        // check that field can be resolved in input type
-        val (physicalName, _, tpe) = resolveInputField(name, tableSource)
-        // validate that mapped fields are are same type
-        if (tpe != t) {
-          throw new ValidationException(s"Type $t of table field '$name' does not " +
-            s"match with type $tpe of the field '$physicalName' of the TableSource return type.")
-        }
-        mappedFieldCnt += 1
-    }
-    // ensure that only one field is mapped to an atomic type
-    val producedType = fromDataTypeToLegacyInfo(tableSource.getProducedDataType)
-    if (!producedType.isInstanceOf[CompositeType[_]] && mappedFieldCnt > 1) {
-      throw new ValidationException(
-        s"More than one table field matched to atomic input type $producedType.")
-    }
-
-    // validate rowtime attributes
-    tableSource match {
-      case r: DefinedRowtimeAttributes =>
-        val descriptors = r.getRowtimeAttributeDescriptors
-        if (descriptors.size() > 1) {
-          throw new ValidationException(
-            "Currently, only a single rowtime attribute is supported. " +
-            s"Please remove all but one RowtimeAttributeDescriptor.")
-        } else if (descriptors.size() == 1) {
-          val descriptor = descriptors.get(0)
-          val rowtimeAttribute = descriptor.getAttributeName
-          val rowtimeIdx = schema.getFieldNames.indexOf(rowtimeAttribute)
-          // ensure that field exists
-          if (rowtimeIdx < 0) {
-            throw new ValidationException(s"Found a rowtime attribute for field " +
-              s"'$rowtimeAttribute' but field '$rowtimeAttribute' does not exist in table.")
-          }
-          // ensure that field is of type TIMESTAMP
-          if (schema.getFieldTypes()(rowtimeIdx) != Types.SQL_TIMESTAMP) {
-            throw new ValidationException(s"Found a rowtime attribute for field " +
-              s"'$rowtimeAttribute' but field '$rowtimeAttribute' is not of type TIMESTAMP.")
-          }
-          // look up extractor input fields in return type
-          val extractorInputFields = descriptor.getTimestampExtractor.getArgumentFields
-          val physicalTypes = resolveInputFields(extractorInputFields, tableSource).map(_._3)
-          // validate timestamp extractor
-          descriptor.getTimestampExtractor.validateArgumentFields(physicalTypes)
-        }
-      case _ => // nothing to validate
-    }
-
-    // validate proctime attribute
-    tableSource match {
-      case p: DefinedProctimeAttribute if p.getProctimeAttribute != null =>
-        val proctimeAttribute = p.getProctimeAttribute
-        val proctimeIdx = schema.getFieldNames.indexOf(proctimeAttribute)
-        // ensure that field exists
-        if (proctimeIdx < 0) {
-          throw new ValidationException(s"Found a processing time attribute for field " +
-            s"'$proctimeAttribute' but field '$proctimeAttribute' does not exist in table.")
-        }
-        // ensure that field is of type TIMESTAMP
-        if (schema.getFieldTypes()(proctimeIdx) != Types.SQL_TIMESTAMP) {
-          throw new ValidationException(s"Found a processing time attribute for field " +
-            s"'$proctimeAttribute' but field '$proctimeAttribute' is not of type TIMESTAMP.")
-        }
-      case _ => // nothing to validate
-    }
-
-    // ensure that proctime and rowtime attribute do not overlap
-    if (proctimeAttribute.isDefined && rowtimeAttributes.contains(proctimeAttribute.get)) {
-      throw new ValidationException(s"Field '${proctimeAttribute.get}' must not be " +
-        s"processing time and rowtime attribute at the same time.")
-    }
-  }
-
-  /**
     * Computes the indices that map the input type of the DataStream to the schema of the table.
     *
     * The mapping is based on the field names and fails if a table field cannot be
diff --git a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala
index e0dc5c3..6711a4e 100644
--- a/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala
+++ b/flink-table/flink-table-planner/src/test/scala/org/apache/flink/table/api/validation/TableSourceValidationTest.scala
@@ -20,6 +20,7 @@ package org.apache.flink.table.api.validation
 
 import java.util
 import java.util.Collections
+
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.RowTypeInfo
 import org.apache.flink.streaming.api.TimeCharacteristic
@@ -29,11 +30,11 @@ import org.apache.flink.table.api.{TableException, TableSchema, Types, Validatio
 import org.apache.flink.table.sources._
 import org.apache.flink.table.sources.tsextractors.ExistingField
 import org.apache.flink.table.sources.wmstrategies.AscendingTimestamps
-import org.apache.flink.table.utils.{TestInputFormatTableSource, TestTableSourceWithTime}
+import org.apache.flink.table.utils.{TableTestBase, TestInputFormatTableSource, TestTableSourceWithTime}
 import org.apache.flink.types.Row
 import org.junit.Test
 
-class TableSourceValidationTest {
+class TableSourceValidationTest extends TableTestBase{
 
   @Test(expected = classOf[ValidationException])
   def testUnresolvedSchemaField(): Unit = {
@@ -128,6 +129,57 @@ class TableSourceValidationTest {
     tEnv.registerTableSource("testTable", ts)
   }
 
+  @Test
+  def testDefinedRowtimeDoesNotExist(): Unit = {
+    expectedException.expect(classOf[ValidationException])
+    expectedException
+      .expectMessage(
+        "Found a rowtime attribute for field 'rowtime' but it does not exist in the Table")
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
+    val tEnv = StreamTableEnvironment.create(env)
+
+    val schema = new TableSchema(
+      Array("id", "name", "amount"),
+      Array(Types.LONG, Types.STRING, Types.INT))
+
+    val rowType = new RowTypeInfo(
+      Array(Types.LONG, Types.STRING, Types.SQL_TIMESTAMP(), Types.INT)
+        .asInstanceOf[Array[TypeInformation[_]]],
+      Array("id", "name", "rowtime", "amount"))
+
+    val ts = new TestTableSourceWithTime(schema, rowType, Seq[Row](), rowtime = "rowtime")
+
+    // should fail because rowtime field does not exist in the TableSchema
+    tEnv.registerTableSource("testTable", ts)
+  }
+
+  @Test
+  def testDefinedProctimeDoesNotExist(): Unit = {
+    expectedException.expect(classOf[ValidationException])
+    expectedException
+      .expectMessage(
+        "Found a proctime attribute for field 'proctime' but it does not exist in the Table")
+
+    val env = StreamExecutionEnvironment.getExecutionEnvironment
+    val tEnv = StreamTableEnvironment.create(env)
+
+    val schema = new TableSchema(
+      Array("id", "name", "amount"),
+      Array(Types.LONG, Types.STRING, Types.INT))
+
+    val rowType = new RowTypeInfo(
+      Array(Types.LONG, Types.STRING, Types.SQL_TIMESTAMP(), Types.INT)
+        .asInstanceOf[Array[TypeInformation[_]]],
+      Array("id", "name", "proctime", "amount"))
+
+    val ts = new TestTableSourceWithTime(schema, rowType, Seq[Row](), proctime = "proctime")
+
+    // should fail because proctime field does not exist in the TableSchema
+    tEnv.registerTableSource("testTable", ts)
+  }
+
   @Test(expected = classOf[ValidationException])
   def testNonTimestampRowtimeField(): Unit = {