You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2020/01/02 04:40:52 UTC

[flink] branch release-1.10 updated: [FLINK-15290][streaming] Configure user defined function with global configuration

This is an automated email from the ASF dual-hosted git repository.

bli pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/release-1.10 by this push:
     new 694ae9c  [FLINK-15290][streaming] Configure user defined function with global configuration
694ae9c is described below

commit 694ae9c9c943a8f8cc0aff5aa54ce7207eb3da21
Author: Rui Li <li...@apache.org>
AuthorDate: Tue Dec 17 21:01:55 2019 +0800

    [FLINK-15290][streaming] Configure user defined function with global configuration
    
    Load configurations in HiveTableInputFormat to decide whether vector orc reader should be used.
    
    this closes #10632.
---
 .../flink/connectors/hive/HiveTableSource.java     |  10 +-
 .../connectors/hive/read/HiveTableInputFormat.java |  18 ++--
 .../flink/connectors/hive/HiveTableSourceTest.java | 103 +++++++++++++++++++++
 .../apache/flink/table/HiveVersionTestUtil.java    |   1 +
 4 files changed, 120 insertions(+), 12 deletions(-)

diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java
index 1c0499c..5d654cf 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java
@@ -18,6 +18,7 @@
 
 package org.apache.flink.connectors.hive;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.typeinfo.TypeInformation;
 import org.apache.flink.configuration.Configuration;
 import org.apache.flink.configuration.GlobalConfiguration;
@@ -133,10 +134,10 @@ public class HiveTableSource implements
 		@SuppressWarnings("unchecked")
 		TypeInformation<BaseRow> typeInfo =
 				(TypeInformation<BaseRow>) TypeInfoDataTypeConverter.fromDataTypeToTypeInfo(getProducedDataType());
-		HiveTableInputFormat inputFormat = getInputFormat(allHivePartitions);
+		Configuration conf = GlobalConfiguration.loadConfiguration();
+		HiveTableInputFormat inputFormat = getInputFormat(allHivePartitions, conf.getBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER));
 		DataStreamSource<BaseRow> source = execEnv.createInput(inputFormat, typeInfo);
 
-		Configuration conf = GlobalConfiguration.loadConfiguration();
 		if (conf.getBoolean(HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM)) {
 			int max = conf.getInteger(HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM_MAX);
 			if (max < 1) {
@@ -162,9 +163,10 @@ public class HiveTableSource implements
 		return source.name(explainSource());
 	}
 
-	private HiveTableInputFormat getInputFormat(List<HiveTablePartition> allHivePartitions) {
+	@VisibleForTesting
+	HiveTableInputFormat getInputFormat(List<HiveTablePartition> allHivePartitions, boolean useMapRedReader) {
 		return new HiveTableInputFormat(
-				jobConf, catalogTable, allHivePartitions, projectedFields, limit, hiveVersion);
+				jobConf, catalogTable, allHivePartitions, projectedFields, limit, hiveVersion, useMapRedReader);
 	}
 
 	@Override
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java
index 3e45fe3..f6a424f 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java
@@ -18,12 +18,11 @@
 
 package org.apache.flink.connectors.hive.read;
 
+import org.apache.flink.annotation.VisibleForTesting;
 import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
 import org.apache.flink.api.common.io.statistics.BaseStatistics;
 import org.apache.flink.api.java.hadoop.common.HadoopInputFormatCommonBase;
-import org.apache.flink.configuration.Configuration;
 import org.apache.flink.connectors.hive.FlinkHiveException;
-import org.apache.flink.connectors.hive.HiveOptions;
 import org.apache.flink.connectors.hive.HiveTablePartition;
 import org.apache.flink.core.io.InputSplitAssigner;
 import org.apache.flink.table.catalog.CatalogTable;
@@ -81,9 +80,10 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
 
 	private transient long currentReadCount = 0L;
 
-	private transient SplitReader reader;
+	@VisibleForTesting
+	protected transient SplitReader reader;
 
-	private transient Configuration parameters;
+	private boolean useMapRedReader;
 
 	public HiveTableInputFormat(
 			JobConf jobConf,
@@ -91,7 +91,8 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
 			List<HiveTablePartition> partitions,
 			int[] projectedFields,
 			long limit,
-			String hiveVersion) {
+			String hiveVersion,
+			boolean useMapRedReader) {
 		super(jobConf.getCredentials());
 		this.partitionKeys = catalogTable.getPartitionKeys();
 		this.fieldTypes = catalogTable.getSchema().getFieldDataTypes();
@@ -103,17 +104,16 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
 		this.jobConf = new JobConf(jobConf);
 		int rowArity = catalogTable.getSchema().getFieldCount();
 		selectedFields = projectedFields != null ? projectedFields : IntStream.range(0, rowArity).toArray();
+		this.useMapRedReader = useMapRedReader;
 	}
 
 	@Override
 	public void configure(org.apache.flink.configuration.Configuration parameters) {
-		this.parameters = parameters;
 	}
 
 	@Override
 	public void open(HiveTableInputSplit split) throws IOException {
-		if (!parameters.getBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER) &&
-				useOrcVectorizedRead(split.getHiveTablePartition())) {
+		if (!useMapRedReader && useOrcVectorizedRead(split.getHiveTablePartition())) {
 			this.reader = new HiveVectorizedOrcSplitReader(
 					hiveVersion, jobConf, fieldNames, fieldTypes, selectedFields, split);
 		} else {
@@ -250,6 +250,7 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
 		out.writeObject(selectedFields);
 		out.writeObject(limit);
 		out.writeObject(hiveVersion);
+		out.writeBoolean(useMapRedReader);
 	}
 
 	@SuppressWarnings("unchecked")
@@ -271,5 +272,6 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
 		selectedFields = (int[]) in.readObject();
 		limit = (long) in.readObject();
 		hiveVersion = (String) in.readObject();
+		useMapRedReader = in.readBoolean();
 	}
 }
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java
index 0104534..6da8e5e 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java
@@ -19,11 +19,21 @@
 package org.apache.flink.connectors.hive;
 
 import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.connectors.hive.read.HiveMapredSplitReader;
+import org.apache.flink.connectors.hive.read.HiveTableInputFormat;
+import org.apache.flink.connectors.hive.read.HiveTableInputSplit;
+import org.apache.flink.connectors.hive.read.HiveVectorizedOrcSplitReader;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.table.HiveVersionTestUtil;
 import org.apache.flink.table.api.Table;
 import org.apache.flink.table.api.TableEnvironment;
 import org.apache.flink.table.api.TableUtils;
 import org.apache.flink.table.api.internal.TableEnvironmentImpl;
 import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
 import org.apache.flink.table.catalog.ObjectPath;
 import org.apache.flink.table.catalog.exceptions.CatalogException;
 import org.apache.flink.table.catalog.exceptions.TableNotExistException;
@@ -33,14 +43,18 @@ import org.apache.flink.table.catalog.hive.HiveTestUtils;
 import org.apache.flink.table.planner.delegation.PlannerBase;
 import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
 import org.apache.flink.table.planner.utils.TableTestUtil;
+import org.apache.flink.test.util.TestBaseUtils;
 import org.apache.flink.types.Row;
+import org.apache.flink.util.FileUtils;
 
 import com.klarna.hiverunner.HiveShell;
 import com.klarna.hiverunner.annotations.HiveSQL;
 import org.apache.calcite.rel.RelNode;
 import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.mapred.JobConf;
 import org.junit.AfterClass;
 import org.junit.Assert;
+import org.junit.Assume;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -48,17 +62,24 @@ import org.junit.runner.RunWith;
 
 import javax.annotation.Nullable;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.file.Files;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 
 import static org.apache.flink.table.planner.utils.JavaScalaConversionUtil.toScala;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
 
 /**
  * Tests {@link HiveTableSource}.
@@ -349,6 +370,88 @@ public class HiveTableSourceTest {
 		Assert.assertEquals(2, transformation.getParallelism());
 	}
 
+	@Test
+	public void testVectorReaderSwitch() throws Exception {
+		// vector reader not available for 1.x and we're not testing orc for 2.0.x
+		Assume.assumeTrue(HiveVersionTestUtil.HIVE_210_OR_LATER);
+		Map<String, String> env = System.getenv();
+		hiveShell.execute("create database db1");
+		try {
+			hiveShell.execute("create table db1.src (x int,y string) stored as orc");
+			hiveShell.execute("insert into db1.src values (1,'a'),(2,'b')");
+			testVectorReader(true);
+			testVectorReader(false);
+		} finally {
+			TestBaseUtils.setEnv(env);
+			hiveShell.execute("drop database db1 cascade");
+		}
+	}
+
+	private void testVectorReader(boolean fallback) throws Exception {
+		File tmpDir = Files.createTempDirectory(null).toFile();
+		Runtime.getRuntime().addShutdownHook(new Thread(() -> FileUtils.deleteDirectoryQuietly(tmpDir)));
+
+		File flinkConf = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
+		Configuration conf = new Configuration();
+		conf.setBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER, fallback);
+		BootstrapTools.writeConfiguration(conf, flinkConf);
+		Map<String, String> map = new HashMap<>(System.getenv());
+		map.put(ConfigConstants.ENV_FLINK_CONF_DIR, tmpDir.getAbsolutePath());
+		TestBaseUtils.setEnv(map);
+
+		ObjectPath tablePath = new ObjectPath("db1", "src");
+		CatalogTable catalogTable = (CatalogTable) hiveCatalog.getTable(tablePath);
+
+		HiveTableFactory tableFactorySpy = spy((HiveTableFactory) hiveCatalog.getTableFactory().get());
+		doReturn(new TestVectorReaderSource(new JobConf(hiveCatalog.getHiveConf()), tablePath, catalogTable))
+				.when(tableFactorySpy).createTableSource(any(ObjectPath.class), any(CatalogTable.class));
+		HiveCatalog catalogSpy = spy(hiveCatalog);
+		doReturn(Optional.of(tableFactorySpy)).when(catalogSpy).getTableFactory();
+
+		TableEnvironment tableEnv = HiveTestUtils.createTableEnvWithBlinkPlannerBatchMode();
+		tableEnv.registerCatalog(catalogSpy.getName(), catalogSpy);
+		tableEnv.useCatalog(catalogSpy.getName());
+
+		List<Row> results = TableUtils.collectToList(tableEnv.sqlQuery("select * from db1.src order by x"));
+		assertEquals("[1,a, 2,b]", results.toString());
+	}
+
+	// A sub-class of HiveTableSource to test vector reader switch.
+	private static class TestVectorReaderSource extends HiveTableSource {
+		private final JobConf jobConf;
+		private final CatalogTable catalogTable;
+
+		TestVectorReaderSource(JobConf jobConf, ObjectPath tablePath, CatalogTable catalogTable) {
+			super(jobConf, tablePath, catalogTable);
+			this.jobConf = jobConf;
+			this.catalogTable = catalogTable;
+		}
+
+		@Override
+		HiveTableInputFormat getInputFormat(List<HiveTablePartition> allHivePartitions, boolean useMapRedReader) {
+			return new TestVectorReaderInputFormat(
+					jobConf, catalogTable, allHivePartitions, null, -1, hiveCatalog.getHiveVersion(), useMapRedReader);
+		}
+	}
+
+	// A sub-class of HiveTableInputFormat to test vector reader switch.
+	private static class TestVectorReaderInputFormat extends HiveTableInputFormat {
+
+		private static final long serialVersionUID = 1L;
+
+		TestVectorReaderInputFormat(JobConf jobConf, CatalogTable catalogTable, List<HiveTablePartition> partitions,
+				int[] projectedFields, long limit, String hiveVersion, boolean useMapRedReader) {
+			super(jobConf, catalogTable, partitions, projectedFields, limit, hiveVersion, useMapRedReader);
+		}
+
+		@Override
+		public void open(HiveTableInputSplit split) throws IOException {
+			super.open(split);
+			boolean fallback = GlobalConfiguration.loadConfiguration().getBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER);
+			assertTrue((fallback && reader instanceof HiveMapredSplitReader) || (!fallback && reader instanceof HiveVectorizedOrcSplitReader));
+		}
+	}
+
 	// A sub-class of HiveCatalog to test list partitions by filter.
 	private static class TestPartitionFilterCatalog extends HiveCatalog {
 
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java
index 25442c1..2a3565a 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java
@@ -26,5 +26,6 @@ public class HiveVersionTestUtil {
 	public static final boolean HIVE_120_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V1_2_0) >= 0;
 	public static final boolean HIVE_110_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V1_1_0) >= 0;
 	public static final boolean HIVE_310_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V3_1_0) >= 0;
+	public static final boolean HIVE_210_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V2_1_0) >= 0;
 
 }