You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by bl...@apache.org on 2020/01/02 04:40:52 UTC
[flink] branch release-1.10 updated: [FLINK-15290][streaming]
Configure user defined function with global configuration
This is an automated email from the ASF dual-hosted git repository.
bli pushed a commit to branch release-1.10
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/release-1.10 by this push:
new 694ae9c [FLINK-15290][streaming] Configure user defined function with global configuration
694ae9c is described below
commit 694ae9c9c943a8f8cc0aff5aa54ce7207eb3da21
Author: Rui Li <li...@apache.org>
AuthorDate: Tue Dec 17 21:01:55 2019 +0800
[FLINK-15290][streaming] Configure user defined function with global configuration
Load configurations in HiveTableInputFormat to decide whether vector orc reader should be used.
this closes #10632.
---
.../flink/connectors/hive/HiveTableSource.java | 10 +-
.../connectors/hive/read/HiveTableInputFormat.java | 18 ++--
.../flink/connectors/hive/HiveTableSourceTest.java | 103 +++++++++++++++++++++
.../apache/flink/table/HiveVersionTestUtil.java | 1 +
4 files changed, 120 insertions(+), 12 deletions(-)
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java
index 1c0499c..5d654cf 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/HiveTableSource.java
@@ -18,6 +18,7 @@
package org.apache.flink.connectors.hive;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.GlobalConfiguration;
@@ -133,10 +134,10 @@ public class HiveTableSource implements
@SuppressWarnings("unchecked")
TypeInformation<BaseRow> typeInfo =
(TypeInformation<BaseRow>) TypeInfoDataTypeConverter.fromDataTypeToTypeInfo(getProducedDataType());
- HiveTableInputFormat inputFormat = getInputFormat(allHivePartitions);
+ Configuration conf = GlobalConfiguration.loadConfiguration();
+ HiveTableInputFormat inputFormat = getInputFormat(allHivePartitions, conf.getBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER));
DataStreamSource<BaseRow> source = execEnv.createInput(inputFormat, typeInfo);
- Configuration conf = GlobalConfiguration.loadConfiguration();
if (conf.getBoolean(HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM)) {
int max = conf.getInteger(HiveOptions.TABLE_EXEC_HIVE_INFER_SOURCE_PARALLELISM_MAX);
if (max < 1) {
@@ -162,9 +163,10 @@ public class HiveTableSource implements
return source.name(explainSource());
}
- private HiveTableInputFormat getInputFormat(List<HiveTablePartition> allHivePartitions) {
+ @VisibleForTesting
+ HiveTableInputFormat getInputFormat(List<HiveTablePartition> allHivePartitions, boolean useMapRedReader) {
return new HiveTableInputFormat(
- jobConf, catalogTable, allHivePartitions, projectedFields, limit, hiveVersion);
+ jobConf, catalogTable, allHivePartitions, projectedFields, limit, hiveVersion, useMapRedReader);
}
@Override
diff --git a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java
index 3e45fe3..f6a424f 100644
--- a/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java
+++ b/flink-connectors/flink-connector-hive/src/main/java/org/apache/flink/connectors/hive/read/HiveTableInputFormat.java
@@ -18,12 +18,11 @@
package org.apache.flink.connectors.hive.read;
+import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.io.LocatableInputSplitAssigner;
import org.apache.flink.api.common.io.statistics.BaseStatistics;
import org.apache.flink.api.java.hadoop.common.HadoopInputFormatCommonBase;
-import org.apache.flink.configuration.Configuration;
import org.apache.flink.connectors.hive.FlinkHiveException;
-import org.apache.flink.connectors.hive.HiveOptions;
import org.apache.flink.connectors.hive.HiveTablePartition;
import org.apache.flink.core.io.InputSplitAssigner;
import org.apache.flink.table.catalog.CatalogTable;
@@ -81,9 +80,10 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
private transient long currentReadCount = 0L;
- private transient SplitReader reader;
+ @VisibleForTesting
+ protected transient SplitReader reader;
- private transient Configuration parameters;
+ private boolean useMapRedReader;
public HiveTableInputFormat(
JobConf jobConf,
@@ -91,7 +91,8 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
List<HiveTablePartition> partitions,
int[] projectedFields,
long limit,
- String hiveVersion) {
+ String hiveVersion,
+ boolean useMapRedReader) {
super(jobConf.getCredentials());
this.partitionKeys = catalogTable.getPartitionKeys();
this.fieldTypes = catalogTable.getSchema().getFieldDataTypes();
@@ -103,17 +104,16 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
this.jobConf = new JobConf(jobConf);
int rowArity = catalogTable.getSchema().getFieldCount();
selectedFields = projectedFields != null ? projectedFields : IntStream.range(0, rowArity).toArray();
+ this.useMapRedReader = useMapRedReader;
}
@Override
public void configure(org.apache.flink.configuration.Configuration parameters) {
- this.parameters = parameters;
}
@Override
public void open(HiveTableInputSplit split) throws IOException {
- if (!parameters.getBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER) &&
- useOrcVectorizedRead(split.getHiveTablePartition())) {
+ if (!useMapRedReader && useOrcVectorizedRead(split.getHiveTablePartition())) {
this.reader = new HiveVectorizedOrcSplitReader(
hiveVersion, jobConf, fieldNames, fieldTypes, selectedFields, split);
} else {
@@ -250,6 +250,7 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
out.writeObject(selectedFields);
out.writeObject(limit);
out.writeObject(hiveVersion);
+ out.writeBoolean(useMapRedReader);
}
@SuppressWarnings("unchecked")
@@ -271,5 +272,6 @@ public class HiveTableInputFormat extends HadoopInputFormatCommonBase<BaseRow, H
selectedFields = (int[]) in.readObject();
limit = (long) in.readObject();
hiveVersion = (String) in.readObject();
+ useMapRedReader = in.readBoolean();
}
}
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java
index 0104534..6da8e5e 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/connectors/hive/HiveTableSourceTest.java
@@ -19,11 +19,21 @@
package org.apache.flink.connectors.hive;
import org.apache.flink.api.dag.Transformation;
+import org.apache.flink.configuration.ConfigConstants;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.configuration.GlobalConfiguration;
+import org.apache.flink.connectors.hive.read.HiveMapredSplitReader;
+import org.apache.flink.connectors.hive.read.HiveTableInputFormat;
+import org.apache.flink.connectors.hive.read.HiveTableInputSplit;
+import org.apache.flink.connectors.hive.read.HiveVectorizedOrcSplitReader;
+import org.apache.flink.runtime.clusterframework.BootstrapTools;
+import org.apache.flink.table.HiveVersionTestUtil;
import org.apache.flink.table.api.Table;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.TableUtils;
import org.apache.flink.table.api.internal.TableEnvironmentImpl;
import org.apache.flink.table.catalog.CatalogPartitionSpec;
+import org.apache.flink.table.catalog.CatalogTable;
import org.apache.flink.table.catalog.ObjectPath;
import org.apache.flink.table.catalog.exceptions.CatalogException;
import org.apache.flink.table.catalog.exceptions.TableNotExistException;
@@ -33,14 +43,18 @@ import org.apache.flink.table.catalog.hive.HiveTestUtils;
import org.apache.flink.table.planner.delegation.PlannerBase;
import org.apache.flink.table.planner.plan.nodes.exec.ExecNode;
import org.apache.flink.table.planner.utils.TableTestUtil;
+import org.apache.flink.test.util.TestBaseUtils;
import org.apache.flink.types.Row;
+import org.apache.flink.util.FileUtils;
import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.annotations.HiveSQL;
import org.apache.calcite.rel.RelNode;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.mapred.JobConf;
import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -48,17 +62,24 @@ import org.junit.runner.RunWith;
import javax.annotation.Nullable;
+import java.io.File;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import static org.apache.flink.table.planner.utils.JavaScalaConversionUtil.toScala;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.spy;
/**
* Tests {@link HiveTableSource}.
@@ -349,6 +370,88 @@ public class HiveTableSourceTest {
Assert.assertEquals(2, transformation.getParallelism());
}
+ @Test
+ public void testVectorReaderSwitch() throws Exception {
+ // vector reader not available for 1.x and we're not testing orc for 2.0.x
+ Assume.assumeTrue(HiveVersionTestUtil.HIVE_210_OR_LATER);
+ Map<String, String> env = System.getenv();
+ hiveShell.execute("create database db1");
+ try {
+ hiveShell.execute("create table db1.src (x int,y string) stored as orc");
+ hiveShell.execute("insert into db1.src values (1,'a'),(2,'b')");
+ testVectorReader(true);
+ testVectorReader(false);
+ } finally {
+ TestBaseUtils.setEnv(env);
+ hiveShell.execute("drop database db1 cascade");
+ }
+ }
+
+ private void testVectorReader(boolean fallback) throws Exception {
+ File tmpDir = Files.createTempDirectory(null).toFile();
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> FileUtils.deleteDirectoryQuietly(tmpDir)));
+
+ File flinkConf = new File(tmpDir, GlobalConfiguration.FLINK_CONF_FILENAME);
+ Configuration conf = new Configuration();
+ conf.setBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER, fallback);
+ BootstrapTools.writeConfiguration(conf, flinkConf);
+ Map<String, String> map = new HashMap<>(System.getenv());
+ map.put(ConfigConstants.ENV_FLINK_CONF_DIR, tmpDir.getAbsolutePath());
+ TestBaseUtils.setEnv(map);
+
+ ObjectPath tablePath = new ObjectPath("db1", "src");
+ CatalogTable catalogTable = (CatalogTable) hiveCatalog.getTable(tablePath);
+
+ HiveTableFactory tableFactorySpy = spy((HiveTableFactory) hiveCatalog.getTableFactory().get());
+ doReturn(new TestVectorReaderSource(new JobConf(hiveCatalog.getHiveConf()), tablePath, catalogTable))
+ .when(tableFactorySpy).createTableSource(any(ObjectPath.class), any(CatalogTable.class));
+ HiveCatalog catalogSpy = spy(hiveCatalog);
+ doReturn(Optional.of(tableFactorySpy)).when(catalogSpy).getTableFactory();
+
+ TableEnvironment tableEnv = HiveTestUtils.createTableEnvWithBlinkPlannerBatchMode();
+ tableEnv.registerCatalog(catalogSpy.getName(), catalogSpy);
+ tableEnv.useCatalog(catalogSpy.getName());
+
+ List<Row> results = TableUtils.collectToList(tableEnv.sqlQuery("select * from db1.src order by x"));
+ assertEquals("[1,a, 2,b]", results.toString());
+ }
+
+ // A sub-class of HiveTableSource to test vector reader switch.
+ private static class TestVectorReaderSource extends HiveTableSource {
+ private final JobConf jobConf;
+ private final CatalogTable catalogTable;
+
+ TestVectorReaderSource(JobConf jobConf, ObjectPath tablePath, CatalogTable catalogTable) {
+ super(jobConf, tablePath, catalogTable);
+ this.jobConf = jobConf;
+ this.catalogTable = catalogTable;
+ }
+
+ @Override
+ HiveTableInputFormat getInputFormat(List<HiveTablePartition> allHivePartitions, boolean useMapRedReader) {
+ return new TestVectorReaderInputFormat(
+ jobConf, catalogTable, allHivePartitions, null, -1, hiveCatalog.getHiveVersion(), useMapRedReader);
+ }
+ }
+
+ // A sub-class of HiveTableInputFormat to test vector reader switch.
+ private static class TestVectorReaderInputFormat extends HiveTableInputFormat {
+
+ private static final long serialVersionUID = 1L;
+
+ TestVectorReaderInputFormat(JobConf jobConf, CatalogTable catalogTable, List<HiveTablePartition> partitions,
+ int[] projectedFields, long limit, String hiveVersion, boolean useMapRedReader) {
+ super(jobConf, catalogTable, partitions, projectedFields, limit, hiveVersion, useMapRedReader);
+ }
+
+ @Override
+ public void open(HiveTableInputSplit split) throws IOException {
+ super.open(split);
+ boolean fallback = GlobalConfiguration.loadConfiguration().getBoolean(HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_READER);
+ assertTrue((fallback && reader instanceof HiveMapredSplitReader) || (!fallback && reader instanceof HiveVectorizedOrcSplitReader));
+ }
+ }
+
// A sub-class of HiveCatalog to test list partitions by filter.
private static class TestPartitionFilterCatalog extends HiveCatalog {
diff --git a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java
index 25442c1..2a3565a 100644
--- a/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java
+++ b/flink-connectors/flink-connector-hive/src/test/java/org/apache/flink/table/HiveVersionTestUtil.java
@@ -26,5 +26,6 @@ public class HiveVersionTestUtil {
public static final boolean HIVE_120_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V1_2_0) >= 0;
public static final boolean HIVE_110_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V1_1_0) >= 0;
public static final boolean HIVE_310_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V3_1_0) >= 0;
+ public static final boolean HIVE_210_OR_LATER = HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V2_1_0) >= 0;
}