You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by yu...@apache.org on 2023/06/08 01:43:49 UTC
[flink] branch master updated (6b04e8c464e -> 98f5500fea9)
This is an automated email from the ASF dual-hosted git repository.
yuxia pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
from 6b04e8c464e [FLINK-32267] Upgrade tesctcontainers from 1.17.6 to 1.18.3
new 3f7e9245de5 [FLINK-30660][e2e] Add e2e test for hive
new 98f5500fea9 [FLINK-30660][sql-client] remove hive dependency in sql-client test This closes #22679
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../pom.xml | 174 +++------
.../org/apache/flink/tests/hive/HiveITCase.java | 294 +++++++++++++++
.../flink/tests/hive/containers/HiveContainer.java | 102 ++++++
.../src/test/resources/hive-site.xml | 104 ++++++
.../resources/hive_catalog_read_write_hive.sql | 29 +-
.../src/test/resources/hive_dialect.sql | 65 ++--
.../src/test/resources/log4j2-test.properties | 0
flink-end-to-end-tests/pom.xml | 1 +
flink-table/flink-sql-client/pom.xml | 407 +--------------------
.../flink/table/client/cli/CliClientITCase.java | 11 -
.../flink/table/client/cli/CliClientTest.java | 20 +-
.../table/client/cli/utils/SqlParserHelper.java | 44 +--
.../table/client/util/TestHiveCatalogFactory.java | 125 -------
.../org.apache.flink.table.factories.Factory | 16 -
.../src/test/resources/sql/catalog_database.q | 102 +-----
.../src/test/resources/sql/function.q | 25 --
.../src/test/resources/sql/module.q | 58 ++-
.../flink-sql-client/src/test/resources/sql/set.q | 89 +----
.../org/apache/flink/util/DockerImageVersions.java | 2 +
19 files changed, 692 insertions(+), 976 deletions(-)
copy flink-end-to-end-tests/{flink-end-to-end-tests-hbase => flink-end-to-end-tests-hive}/pom.xml (53%)
create mode 100644 flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/HiveITCase.java
create mode 100644 flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
create mode 100644 flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive-site.xml
copy flink-libraries/flink-state-processing-api/src/main/java/org/apache/flink/state/api/output/EndOfDataMarker.java => flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_catalog_read_write_hive.sql (57%)
copy flink-runtime/src/test/java/org/apache/flink/runtime/util/HardwareTest.java => flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_dialect.sql (51%)
copy {flink-architecture-tests/flink-architecture-tests-production => flink-end-to-end-tests/flink-end-to-end-tests-hive}/src/test/resources/log4j2-test.properties (100%)
delete mode 100644 flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/util/TestHiveCatalogFactory.java
delete mode 100644 flink-table/flink-sql-client/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
[flink] 01/02: [FLINK-30660][e2e] Add e2e test for hive
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuxia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 3f7e9245de5f6618c535d2b59251bb6e3adb0def
Author: luoyuxia <lu...@alumni.sjtu.edu.cn>
AuthorDate: Tue May 30 18:23:58 2023 +0800
[FLINK-30660][e2e] Add e2e test for hive
---
.../flink-end-to-end-tests-hive/pom.xml | 208 +++++++++++++++
.../org/apache/flink/tests/hive/HiveITCase.java | 294 +++++++++++++++++++++
.../flink/tests/hive/containers/HiveContainer.java | 102 +++++++
.../src/test/resources/hive-site.xml | 104 ++++++++
.../resources/hive_catalog_read_write_hive.sql | 42 +++
.../src/test/resources/hive_dialect.sql | 54 ++++
.../src/test/resources/log4j2-test.properties | 28 ++
flink-end-to-end-tests/pom.xml | 1 +
.../org/apache/flink/util/DockerImageVersions.java | 2 +
9 files changed, 835 insertions(+)
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/pom.xml b/flink-end-to-end-tests/flink-end-to-end-tests-hive/pom.xml
new file mode 100644
index 00000000000..13e7b2d190b
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/pom.xml
@@ -0,0 +1,208 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+ <parent>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-end-to-end-tests</artifactId>
+ <version>1.18-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <properties>
+ <!--
+ Hive requires Hadoop 2 to avoid
+ java.lang.NoClassDefFoundError: org/apache/hadoop/metrics/Updater errors
+ Using this dedicated property avoids CI failures with the Hadoop 3 profile
+ -->
+ <hive.hadoop.version>2.10.2</hive.hadoop.version>
+ </properties>
+
+
+
+ <artifactId>flink-end-to-end-tests-hive</artifactId>
+ <name>Flink : E2E Tests : Hive</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-end-to-end-tests-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>flink-sql-connector-hive-${hive.version}_${scala.binary.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <!-- hadoop dependencies for end-to-end test -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${flink.hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- This dependency is no longer shipped with the JDK since Java 9.-->
+ <groupId>jdk.tools</groupId>
+ <artifactId>jdk.tools</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>${hive.hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-reload4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.pentaho</groupId>
+ <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-enforcer-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>dependency-convergence</id>
+ <goals>
+ <goal>enforce</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>pre-integration-test</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.flink</groupId>
+ <artifactId>
+ flink-sql-connector-hive-${hive.version}_${scala.binary.version}
+ </artifactId>
+ <version>${project.version}</version>
+ <destFileName>
+ sql-hive-${hive.version}_${scala.binary.version}.jar
+ </destFileName>
+ <type>jar</type>
+ <outputDirectory>${project.build.directory}/dependencies
+ </outputDirectory>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ <execution>
+ <id>store-classpath-in-target-for-tests</id>
+ <phase>package</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <outputFile>${project.build.directory}/hadoop.classpath</outputFile>
+ <excludeGroupIds>org.apache.flink,org.codehaus.janino</excludeGroupIds>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <!-- Activate these profiles with -Phive3 to build and test against Hive 3.x versions -->
+ <profile>
+ <id>hive3</id>
+ <properties>
+ <hive.version>3.1.3</hive.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <!-- Bump arrow netty dependency -->
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ <version>4.1.46.Final</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <dependency>
+ <!-- Bump arrow netty dependency -->
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ <version>4.1.46.Final</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+
+</project>
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/HiveITCase.java b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/HiveITCase.java
new file mode 100644
index 00000000000..299d0e30a48
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/HiveITCase.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.hive;
+
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.table.catalog.hive.HiveCatalog;
+import org.apache.flink.test.resources.ResourceTestUtils;
+import org.apache.flink.test.util.SQLJobSubmission;
+import org.apache.flink.tests.hive.containers.HiveContainer;
+import org.apache.flink.tests.util.flink.ClusterController;
+import org.apache.flink.tests.util.flink.FlinkResource;
+import org.apache.flink.tests.util.flink.FlinkResourceSetup;
+import org.apache.flink.tests.util.flink.JarLocation;
+import org.apache.flink.tests.util.flink.LocalStandaloneFlinkResourceFactory;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.TestLogger;
+import org.apache.flink.util.UserClassLoaderJarTestUtils;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/** IT case for Hive including reading & writing hive and Hive dialect. */
+public class HiveITCase extends TestLogger {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveITCase.class);
+
+ @ClassRule public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+
+ @ClassRule
+ public static final HiveContainer HIVE_CONTAINER =
+ new HiveContainer(
+ Arrays.asList("hive_sink1", "hive_sink2", "h_table_sink1", "h_table_sink2"));
+
+ private static final String HIVE_ADD_ONE_UDF_CLASS = "HiveAddOneFunc";
+ private static final String HIVE_ADD_ONE_UDF_CODE =
+ "public class "
+ + HIVE_ADD_ONE_UDF_CLASS
+ + " extends org.apache.hadoop.hive.ql.exec.UDF {\n"
+ + " public int evaluate(int content) {\n"
+ + " return content + 1;\n"
+ + " }"
+ + "}\n";
+
+ private static final Path HADOOP_CLASS_PATH =
+ ResourceTestUtils.getResource(".*hadoop.classpath");
+ private static final Path sqlHiveJar = ResourceTestUtils.getResource(".*sql-hive-.*.jar");
+ private static String hiveConfDirPath;
+ private static String udfDependsPath;
+
+ @Rule public final FlinkResource flink = buildFlinkResource();
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initUDFJar();
+ initHiveConfFile();
+ }
+
+ private static void initUDFJar() throws Exception {
+ Path tmpPath = TMP_FOLDER.getRoot().toPath();
+ LOG.info("The current temporary path: {}", tmpPath);
+ Map<String, String> classNameCodes = new HashMap<>();
+ classNameCodes.put(HIVE_ADD_ONE_UDF_CLASS, HIVE_ADD_ONE_UDF_CODE);
+ File udfJar =
+ UserClassLoaderJarTestUtils.createJarFile(
+ TMP_FOLDER.newFolder("test-jar"),
+ "test-classloader-udf.jar",
+ classNameCodes);
+ udfDependsPath = udfJar.toURI().getPath();
+ }
+
+ private static void initHiveConfFile() {
+ // create a hive conf file
+ File hiveSiteFile = createHiveConfFile();
+ hiveConfDirPath = hiveSiteFile.getParent();
+ }
+
+ @Test
+ public void testReadWriteHive() throws Exception {
+ runAndCheckSQL(
+ "hive_catalog_read_write_hive.sql",
+ generateReplaceVars(),
+ Paths.get(HIVE_CONTAINER.getWarehousePath()).resolve("hive_sink2"),
+ 2,
+ Arrays.asList("1\u0001v1", "2\u0001v2"));
+ }
+
+ @Test
+ public void testHiveDialect() throws Exception {
+ runAndCheckSQL(
+ "hive_dialect.sql",
+ generateReplaceVars(),
+ Paths.get(HIVE_CONTAINER.getWarehousePath()).resolve("h_table_sink2"),
+ 3,
+ Arrays.asList("2\u0001v1", "3\u0001v2", "4\u0001v3"));
+ }
+
+ private Map<String, String> generateReplaceVars() {
+ Map<String, String> varsMap = new HashMap<>();
+ varsMap.put("$VAR_UDF_JAR_PATH", udfDependsPath);
+ varsMap.put("$VAR_HIVE_CONF_DIR", hiveConfDirPath);
+ varsMap.put("$VAR_HIVE_WAREHOUSE", "file://" + HIVE_CONTAINER.getWarehousePath());
+ return varsMap;
+ }
+
+ private void runAndCheckSQL(
+ String sqlPath,
+ Map<String, String> varsMap,
+ Path resultPath,
+ int resultSize,
+ List<String> resultItems)
+ throws Exception {
+ try (ClusterController clusterController = flink.startCluster(1)) {
+ List<String> sqlLines = initializeSqlLines(sqlPath, varsMap);
+
+ executeSqlStatements(clusterController, sqlLines);
+
+ // Wait until all the results flushed to the json file.
+ LOG.info("Verify the result file.");
+ checkResultFile(resultPath, resultSize, resultItems);
+ LOG.info("The SQL client test run successfully.");
+ }
+ }
+
+ private List<String> readResultFiles(Path path) throws IOException {
+ File filePath = path.toFile();
+ // list all the non-hidden files
+ File[] txtFiles = filePath.listFiles((dir, name) -> !name.startsWith("."));
+ List<String> result = new ArrayList<>();
+ if (txtFiles != null) {
+ for (File file : txtFiles) {
+ result.addAll(Files.readAllLines(file.toPath()));
+ }
+ }
+ return result;
+ }
+
+ private void checkResultFile(Path resultPath, int resultSize, List<String> items)
+ throws Exception {
+ boolean success = false;
+ final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(20));
+ List<String> lines = null;
+ while (deadline.hasTimeLeft()) {
+ if (Files.exists(resultPath)) {
+ lines = readResultFiles(resultPath);
+ if (lines.size() == resultSize) {
+ success = true;
+ assertThat(lines).hasSameElementsAs(items);
+ break;
+ } else {
+ LOG.info(
+ "The target result file {} does not contain enough records, current {} "
+ + "records, left time: {}s",
+ resultPath,
+ lines.size(),
+ deadline.timeLeft().getSeconds());
+ }
+ } else {
+ LOG.info("The target Json {} does not exist now", resultPath);
+ }
+ Thread.sleep(500);
+ }
+ assertTrue(
+ success,
+ String.format(
+ "Did not get expected results before timeout, actual result: %s.", lines));
+ }
+
+ private List<String> initializeSqlLines(String sqlPath, Map<String, String> vars)
+ throws IOException {
+ URL url = HiveITCase.class.getClassLoader().getResource(sqlPath);
+ if (url == null) {
+ throw new FileNotFoundException(sqlPath);
+ }
+
+ List<String> lines = Files.readAllLines(new File(url.getFile()).toPath());
+ List<String> result = new ArrayList<>();
+ for (String line : lines) {
+ for (Map.Entry<String, String> var : vars.entrySet()) {
+ line = line.replace(var.getKey(), var.getValue());
+ }
+ result.add(line);
+ }
+
+ return result;
+ }
+
+ private void executeSqlStatements(ClusterController clusterController, List<String> sqlLines)
+ throws Exception {
+ clusterController.submitSQLJob(
+ new SQLJobSubmission.SQLJobSubmissionBuilder(sqlLines).addJar(sqlHiveJar).build(),
+ Duration.ofMinutes(2L));
+ }
+
+ private static FlinkResource buildFlinkResource() {
+ // swap planner
+ // todo: should remove planner swap logic after FLINK-31575
+ FlinkResourceSetup.FlinkResourceSetupBuilder builder =
+ FlinkResourceSetup.builder()
+ .addJar(sqlHiveJar, JarLocation.LIB)
+ .moveJar("flink-table-planner", JarLocation.OPT, JarLocation.LIB)
+ .moveJar("flink-table-planner-loader", JarLocation.LIB, JarLocation.OPT);
+
+ // add hadoop jars
+ File hadoopClasspathFile = new File(HADOOP_CLASS_PATH.toAbsolutePath().toString());
+ if (!hadoopClasspathFile.exists()) {
+ throw new RuntimeException(
+ "File that contains hadoop classpath "
+ + HADOOP_CLASS_PATH
+ + " does not exist.");
+ }
+ try {
+ String classPathContent = FileUtils.readFileUtf8(hadoopClasspathFile);
+ Arrays.stream(classPathContent.split(":"))
+ .map(Paths::get)
+ .forEach(jar -> builder.addJar(jar, JarLocation.LIB));
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to build the FlinkResource.", e);
+ }
+
+ return new LocalStandaloneFlinkResourceFactory().create(builder.build());
+ }
+
+ private static File createHiveConfFile() {
+ HiveConf hiveConf = new HiveConf();
+ try (InputStream inputStream =
+ Files.newInputStream(
+ new File(
+ Objects.requireNonNull(
+ HiveITCase.class
+ .getClassLoader()
+ .getResource(
+ HiveCatalog.HIVE_SITE_FILE))
+ .toURI())
+ .toPath())) {
+ hiveConf.addResource(inputStream, HiveCatalog.HIVE_SITE_FILE);
+ // trigger a read from the conf so that the input stream is read
+ hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to load hive-site.xml from specified path", e);
+ }
+ hiveConf.set(HiveConf.ConfVars.METASTOREURIS.varname, HIVE_CONTAINER.getHiveMetastoreURL());
+ try {
+ File site = TMP_FOLDER.newFile(HiveCatalog.HIVE_SITE_FILE);
+ try (OutputStream out = Files.newOutputStream(site.toPath())) {
+ hiveConf.writeXml(out);
+ }
+ return site;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to create hive conf.", e);
+ }
+ }
+}
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
new file mode 100644
index 00000000000..24a5e532cb7
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/java/org/apache/flink/tests/hive/containers/HiveContainer.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.hive.containers;
+
+import org.apache.flink.table.catalog.hive.client.HiveShimLoader;
+import org.apache.flink.util.DockerImageVersions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+/** Test container for Hive. */
+public class HiveContainer extends GenericContainer<HiveContainer> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HiveContainer.class);
+ public static final String HOST_NAME = "hadoop-master";
+
+ private static final boolean HIVE_310_OR_LATER =
+ HiveShimLoader.getHiveVersion().compareTo(HiveShimLoader.HIVE_VERSION_V3_1_0) >= 0;
+ private String hiveWarehouseDir;
+
+ public static final int HIVE_METASTORE_PORT = 9083;
+
+ public HiveContainer(List<String> initTableNames) {
+ super(
+ DockerImageName.parse(
+ HIVE_310_OR_LATER ? DockerImageVersions.HIVE3 : DockerImageVersions.HIVE2));
+ withExtraHost(HOST_NAME, "127.0.0.1");
+ addExposedPort(HIVE_METASTORE_PORT);
+ mountHiveWarehouseDirToContainer(initTableNames);
+ }
+
+ @Override
+ protected void doStart() {
+ super.doStart();
+ if (LOG.isInfoEnabled()) {
+ followOutput(new Slf4jLogConsumer(LOG));
+ }
+ }
+
+ public String getHiveMetastoreURL() {
+ return String.format("thrift://%s:%s", getHost(), getMappedPort(HIVE_METASTORE_PORT));
+ }
+
+ public String getWarehousePath() {
+ return hiveWarehouseDir;
+ }
+
+ private void mountHiveWarehouseDirToContainer(List<String> initTableNames) {
+ try {
+ Path warehousePath = Files.createTempDirectory("hive_warehouse");
+ File file = warehousePath.toFile();
+ hiveWarehouseDir = file.getAbsolutePath();
+ LOG.info("mountHiveWarehouseDirToContainer: " + hiveWarehouseDir);
+
+ if (HIVE_310_OR_LATER) {
+ // if it's hive 3.1+, we should first create the dir for the table,
+ // and set it readable & writable by all, otherwise, it'll throw
+ // permission denied exception when try to write the tables
+ for (String tableName : initTableNames) {
+ file = Files.createDirectory(warehousePath.resolve(tableName)).toFile();
+ setFilePermission(file);
+ }
+ }
+
+ withFileSystemBind(
+ warehousePath.toAbsolutePath().toString(),
+ warehousePath.toAbsolutePath().toString());
+ } catch (IOException e) {
+ throw new IllegalStateException("Failed to create warehouse directory", e);
+ }
+ }
+
+ private void setFilePermission(File file) {
+ file.setReadable(true, false);
+ file.setWritable(true, false);
+ }
+}
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive-site.xml b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive-site.xml
new file mode 100644
index 00000000000..d80f03f6f34
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive-site.xml
@@ -0,0 +1,104 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<configuration>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ <property>
+ <name>hive.metastore.uris</name>
+ <value>thrift://localhost:9083</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:mysql://localhost:3306/metastore?useSSL=false</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>com.mysql.cj.jdbc.Driver</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionUserName</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>javax.jdo.option.ConnectionPassword</name>
+ <value>root</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.connect.retries</name>
+ <value>15</value>
+ </property>
+
+ <property>
+ <name>hive.metastore.disallow.incompatible.col.type.changes</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <!-- https://community.hortonworks.com/content/supportkb/247055/errorjavalangunsupportedoperationexception-storage.html -->
+ <name>metastore.storage.schema.reader.impl</name>
+ <value>org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader</value>
+ </property>
+
+ <property>
+ <name>hive.support.concurrency</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.txn.manager</name>
+ <value>org.apache.hadoop.hive.ql.lockmgr.DbTxnManager</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.initiator.on</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hive.compactor.worker.threads</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>hive.users.in.admin.role</name>
+ <value>hdfs,hive</value>
+ </property>
+
+</configuration>
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_catalog_read_write_hive.sql b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_catalog_read_write_hive.sql
new file mode 100644
index 00000000000..16b1f1e560c
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_catalog_read_write_hive.sql
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+SET 'execution.runtime-mode' = 'batch';
+SET 'table.dml-sync' = 'true';
+SET 'sql-client.verbose' = 'true';
+
+create catalog hivecatalog with (
+ 'type' = 'hive',
+ 'hive-conf-dir' = '$VAR_HIVE_CONF_DIR'
+);
+
+use catalog hivecatalog;
+
+create table hive_sink1 (
+ a int,
+ b varchar(10)
+) with ('connector' = 'hive', 'hive.location-uri' = '$VAR_HIVE_WAREHOUSE/hive_sink1');
+
+insert into hive_sink1 values (1, 'v1'), (2, 'v2');
+
+create table hive_sink2(
+ a int,
+ b varchar(10)
+) with ('connector' = 'hive', 'hive.location-uri' = '$VAR_HIVE_WAREHOUSE/hive_sink2');
+
+insert into hive_sink2 select * from hive_sink1;
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_dialect.sql b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_dialect.sql
new file mode 100644
index 00000000000..b0e1724f080
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/hive_dialect.sql
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+SET 'execution.runtime-mode' = 'batch';
+SET 'table.dml-sync' = 'true';
+set 'sql-client.verbose' = 'true';
+
+create catalog hivecatalog with (
+ 'type' = 'hive',
+ 'hive-conf-dir' = '$VAR_HIVE_CONF_DIR'
+);
+
+use catalog hivecatalog;
+
+-- switch to hive dialect
+set 'table.sql-dialect' = 'hive';
+
+ADD JAR $VAR_UDF_JAR_PATH;
+
+-- create a table sink
+create table h_table_sink1(
+ a int,
+ b string
+) LOCATION '$VAR_HIVE_WAREHOUSE/h_table_sink1';
+
+-- prepare data
+insert into h_table_sink1 values (1, 'v1'), (2, 'v2'), (3, 'v3');
+
+-- create another table sink
+create table h_table_sink2(
+ a int,
+ b string
+) LOCATION '$VAR_HIVE_WAREHOUSE/h_table_sink2';
+
+-- create function using jar
+CREATE FUNCTION hive_add_one as 'HiveAddOneFunc';
+
+-- insert the table with the adding jar
+insert into h_table_sink2 select hive_add_one(a), b from h_table_sink1;
diff --git a/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/log4j2-test.properties b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/log4j2-test.properties
new file mode 100644
index 00000000000..b1c818d7380
--- /dev/null
+++ b/flink-end-to-end-tests/flink-end-to-end-tests-hive/src/test/resources/log4j2-test.properties
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set root logger level to OFF to not flood build logs
+# set manually to INFO for debugging purposes
+rootLogger.level = OFF
+rootLogger.appenderRef.test.ref = TestLogger
+
+appender.testlogger.name = TestLogger
+appender.testlogger.type = CONSOLE
+appender.testlogger.target = SYSTEM_ERR
+appender.testlogger.layout.type = PatternLayout
+appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n
diff --git a/flink-end-to-end-tests/pom.xml b/flink-end-to-end-tests/pom.xml
index f199aaa3832..3a5275f54a1 100644
--- a/flink-end-to-end-tests/pom.xml
+++ b/flink-end-to-end-tests/pom.xml
@@ -76,6 +76,7 @@ under the License.
<module>flink-end-to-end-tests-hbase</module>
<module>flink-end-to-end-tests-scala</module>
<module>flink-end-to-end-tests-sql</module>
+ <module>flink-end-to-end-tests-hive</module>
</modules>
<dependencies>
diff --git a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/DockerImageVersions.java b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/DockerImageVersions.java
index 2de96ebf749..12380b543ac 100644
--- a/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/DockerImageVersions.java
+++ b/flink-test-utils-parent/flink-test-utils-junit/src/main/java/org/apache/flink/util/DockerImageVersions.java
@@ -41,4 +41,6 @@ public class DockerImageVersions {
public static final String ZOOKEEPER = "zookeeper:3.4.14";
public static final String HIVE2 = "prestodb/hdp2.6-hive:10";
+
+ public static final String HIVE3 = "prestodb/hive3.1-hive:10";
}
[flink] 02/02: [FLINK-30660][sql-client] remove hive dependency in sql-client test This closes #22679
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuxia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 98f5500fea96213741aa300b4b9e76be588d867a
Author: luoyuxia <lu...@alumni.sjtu.edu.cn>
AuthorDate: Wed May 31 17:20:14 2023 +0800
[FLINK-30660][sql-client] remove hive dependency in sql-client test
This closes #22679
---
flink-table/flink-sql-client/pom.xml | 407 +--------------------
.../flink/table/client/cli/CliClientITCase.java | 11 -
.../flink/table/client/cli/CliClientTest.java | 20 +-
.../table/client/cli/utils/SqlParserHelper.java | 44 +--
.../table/client/util/TestHiveCatalogFactory.java | 125 -------
.../org.apache.flink.table.factories.Factory | 16 -
.../src/test/resources/sql/catalog_database.q | 102 +-----
.../src/test/resources/sql/function.q | 25 --
.../src/test/resources/sql/module.q | 58 ++-
.../flink-sql-client/src/test/resources/sql/set.q | 89 +----
10 files changed, 74 insertions(+), 823 deletions(-)
diff --git a/flink-table/flink-sql-client/pom.xml b/flink-table/flink-sql-client/pom.xml
index aa919531ee3..3422a2cf9ba 100644
--- a/flink-table/flink-sql-client/pom.xml
+++ b/flink-table/flink-sql-client/pom.xml
@@ -120,34 +120,28 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+ <artifactId>flink-python</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-python</artifactId>
+ <artifactId>flink-table-runtime</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-table-runtime</artifactId>
+ <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
+ <artifactId>flink-connector-files</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
@@ -170,423 +164,46 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-sql-gateway-api</artifactId>
+ <artifactId>flink-table-common</artifactId>
<version>${project.version}</version>
- <type>test-jar</type>
<scope>test</scope>
+ <type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-sql-gateway</artifactId>
+ <artifactId>flink-sql-gateway-api</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <!-- Client parser -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-sql-parser</artifactId>
- <version>${project.version}</version>
- <optional>${flink.markBundledAsOptional}</optional>
- <exclusions>
- <exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-testkit</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <!-- test dependencies for HiveCatalog. The exclusions should be in sync with those flink-connector-hive -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-connector-hive_${scala.binary.version}</artifactId>
+ <artifactId>flink-sql-gateway</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- Client parser -->
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
+ <artifactId>flink-sql-parser</artifactId>
<version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-metastore</artifactId>
- <version>${hive.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javolution</groupId>
- <artifactId>javolution</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.zaxxer</groupId>
- <artifactId>HikariCP</artifactId>
- </exclusion>
- <exclusion>
- <groupId>co.cask.tephra</groupId>
- <artifactId>tephra-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>co.cask.tephra</groupId>
- <artifactId>tephra-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>co.cask.tephra</groupId>
- <artifactId>tephra-hbase-compat-1.0</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.thrift</groupId>
- <artifactId>libfb303</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.transaction</groupId>
- <artifactId>transaction-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ant</groupId>
- <artifactId>ant</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.tdunning</groupId>
- <artifactId>json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.aggregate</groupId>
- <artifactId>jetty-all</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty.orbit</groupId>
- <artifactId>javax.servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-web</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-jvm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.github.joshelser</groupId>
- <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
- </exclusion>
-
- <!-- org.apache.hive:hive-service-rpc -->
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
-
- <!-- org.apache.hive:hive-serde -->
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>net.sf.opencsv</groupId>
- <artifactId>opencsv</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.parquet</groupId>
- <artifactId>parquet-hadoop-bundle</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${hive.version}</version>
- <scope>test</scope>
+ <optional>${flink.markBundledAsOptional}</optional>
<exclusions>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-vector-code-gen</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-llap-tez</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-shims</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j-impl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.antlr</groupId>
- <artifactId>antlr-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.antlr</groupId>
- <artifactId>ST4</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ant</groupId>
- <artifactId>ant</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ivy</groupId>
- <artifactId>ivy</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>apache-curator</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.groovy</groupId>
- <artifactId>groovy-all</artifactId>
- </exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
- <artifactId>calcite-druid</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.calcite.avatica</groupId>
- <artifactId>avatica</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.calcite</groupId>
- <artifactId>calcite-avatica</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- </exclusion>
- <exclusion>
- <groupId>stax</groupId>
- <artifactId>stax-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.pentaho</groupId>
- <artifactId>pentaho-aggdesigner-algorithm</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-digester</groupId>
- <artifactId>commons-digester</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
+ <artifactId>calcite-testkit</artifactId>
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <!-- Replaces hadoop-mapreduce-client-core log4j dependency -->
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- <scope>provided</scope>
- </dependency>
</dependencies>
<dependencyManagement>
diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientITCase.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientITCase.java
index da19e6d8578..bdb308292e6 100644
--- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientITCase.java
+++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientITCase.java
@@ -89,16 +89,6 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Test that runs every {@code xx.q} file in "resources/sql/" path as a test. */
class CliClientITCase {
- private static final String HIVE_ADD_ONE_UDF_CLASS = "HiveAddOneFunc";
- private static final String HIVE_ADD_ONE_UDF_CODE =
- "public class "
- + HIVE_ADD_ONE_UDF_CLASS
- + " extends org.apache.hadoop.hive.ql.exec.UDF {\n"
- + " public int evaluate(int content) {\n"
- + " return content + 1;\n"
- + " }"
- + "}\n";
-
private static Path historyPath;
private static Map<String, String> replaceVars;
@@ -148,7 +138,6 @@ class CliClientITCase {
classNameCodes.put(
GENERATED_UPPER_UDF_CLASS,
String.format(GENERATED_UPPER_UDF_CODE, GENERATED_UPPER_UDF_CLASS));
- classNameCodes.put(HIVE_ADD_ONE_UDF_CLASS, HIVE_ADD_ONE_UDF_CODE);
File udfJar =
UserClassLoaderJarTestUtils.createJarFile(
diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java
index bbfdc64b094..5cf6b527a6c 100644
--- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java
+++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/CliClientTest.java
@@ -24,7 +24,6 @@ import org.apache.flink.configuration.ReadableConfig;
import org.apache.flink.runtime.testutils.CommonTestUtils;
import org.apache.flink.table.api.DataTypes;
import org.apache.flink.table.api.ResultKind;
-import org.apache.flink.table.api.SqlDialect;
import org.apache.flink.table.api.internal.TableResultImpl;
import org.apache.flink.table.catalog.Column;
import org.apache.flink.table.catalog.ResolvedSchema;
@@ -76,9 +75,9 @@ class CliClientTest {
"INSERT INTO MyTable SELECT * FROM MyOtherTable";
private static final String INSERT_OVERWRITE_STATEMENT =
"INSERT OVERWRITE MyTable SELECT * FROM MyOtherTable";
- private static final String ORIGIN_HIVE_SQL = "SELECT pos\t FROM source_table;\n";
- private static final String HIVE_SQL_WITHOUT_COMPLETER = "SELECT pos FROM source_table;";
- private static final String HIVE_SQL_WITH_COMPLETER = "SELECT POSITION FROM source_table;";
+ private static final String ORIGIN_SQL = "SELECT pos\t FROM source_table;\n";
+ private static final String SQL_WITHOUT_COMPLETER = "SELECT pos FROM source_table;";
+ private static final String SQL_WITH_COMPLETER = "SELECT POSITION FROM source_table;";
@Test
void testUpdateSubmission() throws Exception {
@@ -107,23 +106,22 @@ class CliClientTest {
@Test
void testExecuteSqlFileWithoutSqlCompleter() throws Exception {
- MockExecutor executor = new MockExecutor(new SqlParserHelper(SqlDialect.HIVE), false);
- executeSqlFromContent(executor, ORIGIN_HIVE_SQL);
- assertThat(executor.receivedStatement).contains(HIVE_SQL_WITHOUT_COMPLETER);
+ MockExecutor executor = new MockExecutor(new SqlParserHelper(), false);
+ executeSqlFromContent(executor, ORIGIN_SQL);
+ assertThat(executor.receivedStatement).contains(SQL_WITHOUT_COMPLETER);
}
@Test
void testExecuteSqlInteractiveWithSqlCompleter() throws Exception {
- final MockExecutor mockExecutor =
- new MockExecutor(new SqlParserHelper(SqlDialect.HIVE), false);
+ final MockExecutor mockExecutor = new MockExecutor(new SqlParserHelper(), false);
- InputStream inputStream = new ByteArrayInputStream(ORIGIN_HIVE_SQL.getBytes());
+ InputStream inputStream = new ByteArrayInputStream(ORIGIN_SQL.getBytes());
OutputStream outputStream = new ByteArrayOutputStream(256);
try (Terminal terminal = new DumbTerminal(inputStream, outputStream);
CliClient client =
new CliClient(() -> terminal, mockExecutor, historyTempFile(), null)) {
client.executeInInteractiveMode();
- assertThat(mockExecutor.receivedStatement).contains(HIVE_SQL_WITH_COMPLETER);
+ assertThat(mockExecutor.receivedStatement).contains(SQL_WITH_COMPLETER);
}
}
diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/SqlParserHelper.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/SqlParserHelper.java
index 8de112e9f77..bae434a5dbb 100644
--- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/SqlParserHelper.java
+++ b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/cli/utils/SqlParserHelper.java
@@ -19,50 +19,30 @@
package org.apache.flink.table.client.cli.utils;
import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.SqlDialect;
import org.apache.flink.table.api.TableEnvironment;
import org.apache.flink.table.api.internal.TableEnvironmentInternal;
-import org.apache.flink.table.catalog.hive.HiveCatalog;
-import org.apache.flink.table.catalog.hive.HiveTestUtils;
import org.apache.flink.table.delegation.Parser;
/** A utility class that provides pre-prepared tables and sql parser. */
public class SqlParserHelper {
// return the sql parser instance hold by this table evn.
- private TableEnvironment tableEnv;
- private boolean useHiveCatalog;
+ private final TableEnvironment tableEnv;
public SqlParserHelper() {
tableEnv = TableEnvironment.create(EnvironmentSettings.newInstance().build());
}
- public SqlParserHelper(SqlDialect sqlDialect) {
- if (sqlDialect == null || SqlDialect.DEFAULT == sqlDialect) {
- useHiveCatalog = false;
- tableEnv = TableEnvironment.create(EnvironmentSettings.newInstance().build());
- } else if (SqlDialect.HIVE == sqlDialect) {
- useHiveCatalog = true;
- HiveCatalog hiveCatalog = HiveTestUtils.createHiveCatalog();
- tableEnv = TableEnvironment.create(EnvironmentSettings.newInstance().build());
- tableEnv.getConfig().setSqlDialect(sqlDialect);
- tableEnv.registerCatalog(hiveCatalog.getName(), hiveCatalog);
- tableEnv.useCatalog(hiveCatalog.getName());
- }
- }
-
/** prepare some tables for testing. */
public void registerTables() {
- if (!useHiveCatalog) {
- registerTable(
- "create table MyTable (a int, b bigint, c varchar(32)) "
- + "with ('connector' = 'filesystem', 'path' = '/non', 'format' = 'csv')");
- registerTable(
- "create table MyOtherTable (a int, b bigint) "
- + "with ('connector' = 'filesystem', 'path' = '/non', 'format' = 'csv')");
- registerTable(
- "create table MySink (a int, c varchar(32)) with ('connector' = 'COLLECTION' )");
- registerTable("create view MyView as select * from MyTable");
- }
+ registerTable(
+ "create table MyTable (a int, b bigint, c varchar(32)) "
+ + "with ('connector' = 'filesystem', 'path' = '/non', 'format' = 'csv')");
+ registerTable(
+ "create table MyOtherTable (a int, b bigint) "
+ + "with ('connector' = 'filesystem', 'path' = '/non', 'format' = 'csv')");
+ registerTable(
+ "create table MySink (a int, c varchar(32)) with ('connector' = 'COLLECTION' )");
+ registerTable("create view MyView as select * from MyTable");
}
public void registerTable(String createTableStmt) {
@@ -72,8 +52,4 @@ public class SqlParserHelper {
public Parser getSqlParser() {
return ((TableEnvironmentInternal) tableEnv).getParser();
}
-
- public TableEnvironment getTableEnv() {
- return tableEnv;
- }
}
diff --git a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/util/TestHiveCatalogFactory.java b/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/util/TestHiveCatalogFactory.java
deleted file mode 100644
index 23f97ecb45e..00000000000
--- a/flink-table/flink-sql-client/src/test/java/org/apache/flink/table/client/util/TestHiveCatalogFactory.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.table.client.util;
-
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.table.api.DataTypes;
-import org.apache.flink.table.api.Schema;
-import org.apache.flink.table.catalog.Catalog;
-import org.apache.flink.table.catalog.CatalogDatabaseImpl;
-import org.apache.flink.table.catalog.CatalogTable;
-import org.apache.flink.table.catalog.Column;
-import org.apache.flink.table.catalog.ObjectPath;
-import org.apache.flink.table.catalog.ResolvedCatalogTable;
-import org.apache.flink.table.catalog.ResolvedSchema;
-import org.apache.flink.table.catalog.exceptions.CatalogException;
-import org.apache.flink.table.catalog.exceptions.DatabaseAlreadyExistException;
-import org.apache.flink.table.catalog.exceptions.DatabaseNotExistException;
-import org.apache.flink.table.catalog.exceptions.TableAlreadyExistException;
-import org.apache.flink.table.catalog.hive.HiveCatalog;
-import org.apache.flink.table.catalog.hive.HiveTestUtils;
-import org.apache.flink.table.catalog.hive.factories.HiveCatalogFactory;
-import org.apache.flink.table.catalog.hive.factories.HiveCatalogFactoryOptions;
-import org.apache.flink.table.factories.FactoryUtil;
-import org.apache.flink.table.types.DataType;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import static org.apache.flink.table.catalog.hive.util.Constants.IDENTIFIER;
-
-/**
- * A test factory that is the same as {@link HiveCatalogFactory} except returning a {@link
- * HiveCatalog} always with an embedded Hive metastore to test logic of {@link HiveCatalogFactory}.
- */
-public class TestHiveCatalogFactory extends HiveCatalogFactory {
- public static final String ADDITIONAL_TEST_DATABASE = "additional_test_database";
- public static final String TEST_TABLE = "test_table";
- static final String TABLE_WITH_PARAMETERIZED_TYPES = "param_types_table";
-
- @Override
- public String factoryIdentifier() {
- return "hive-test";
- }
-
- @Override
- public Catalog createCatalog(Context context) {
- final Configuration configuration = Configuration.fromMap(context.getOptions());
-
- // Developers may already have their own production/testing hive-site.xml set in their
- // environment,
- // and Flink tests should avoid using those hive-site.xml.
- // Thus, explicitly create a testing HiveConf for unit tests here
- Catalog hiveCatalog =
- HiveTestUtils.createHiveCatalog(
- context.getName(),
- configuration.getString(HiveCatalogFactoryOptions.HIVE_VERSION));
-
- // Creates an additional database to test tableEnv.useDatabase() will switch current
- // database of the catalog
- hiveCatalog.open();
- try {
- hiveCatalog.createDatabase(
- ADDITIONAL_TEST_DATABASE,
- new CatalogDatabaseImpl(Collections.emptyMap(), null),
- false);
- hiveCatalog.createTable(
- new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
- createResolvedTable(new String[] {"testcol"}, new DataType[] {DataTypes.INT()}),
- false);
- // create a table to test parameterized types
- hiveCatalog.createTable(
- new ObjectPath("default", TABLE_WITH_PARAMETERIZED_TYPES),
- createResolvedTable(
- new String[] {"dec", "ch", "vch"},
- new DataType[] {
- DataTypes.DECIMAL(10, 10), DataTypes.CHAR(5), DataTypes.VARCHAR(15)
- }),
- false);
- } catch (DatabaseAlreadyExistException
- | TableAlreadyExistException
- | DatabaseNotExistException e) {
- throw new CatalogException(e);
- }
-
- return hiveCatalog;
- }
-
- private ResolvedCatalogTable createResolvedTable(
- String[] fieldNames, DataType[] fieldDataTypes) {
- final Map<String, String> options = new HashMap<>();
- options.put(FactoryUtil.CONNECTOR.key(), IDENTIFIER);
- final CatalogTable origin =
- CatalogTable.of(
- Schema.newBuilder().fromFields(fieldNames, fieldDataTypes).build(),
- null,
- Collections.emptyList(),
- options);
- final List<Column> resolvedColumns =
- IntStream.range(0, fieldNames.length)
- .mapToObj(i -> Column.physical(fieldNames[i], fieldDataTypes[i]))
- .collect(Collectors.toList());
- return new ResolvedCatalogTable(
- origin, new ResolvedSchema(resolvedColumns, Collections.emptyList(), null));
- }
-}
diff --git a/flink-table/flink-sql-client/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-table/flink-sql-client/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
deleted file mode 100644
index 02388ea20a2..00000000000
--- a/flink-table/flink-sql-client/src/test/resources/META-INF/services/org.apache.flink.table.factories.Factory
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.flink.table.client.util.TestHiveCatalogFactory
diff --git a/flink-table/flink-sql-client/src/test/resources/sql/catalog_database.q b/flink-table/flink-sql-client/src/test/resources/sql/catalog_database.q
index 4755fed07a9..60671ca1db7 100644
--- a/flink-table/flink-sql-client/src/test/resources/sql/catalog_database.q
+++ b/flink-table/flink-sql-client/src/test/resources/sql/catalog_database.q
@@ -200,86 +200,6 @@ drop catalog `mod`;
[INFO] Execute statement succeed.
!info
-# ==========================================================================
-# test hive catalog
-# ==========================================================================
-
-create catalog hivecatalog with (
- 'type' = 'hive-test',
- 'hive-version' = '2.3.4'
-);
-[INFO] Execute statement succeed.
-!info
-
-use catalog hivecatalog;
-[INFO] Execute statement succeed.
-!info
-
-show current catalog;
-+----------------------+
-| current catalog name |
-+----------------------+
-| hivecatalog |
-+----------------------+
-1 row in set
-!ok
-
-show databases;
-+--------------------------+
-| database name |
-+--------------------------+
-| additional_test_database |
-| default |
-+--------------------------+
-2 rows in set
-!ok
-
-show tables;
-+-------------------+
-| table name |
-+-------------------+
-| param_types_table |
-+-------------------+
-1 row in set
-!ok
-
-use additional_test_database;
-[INFO] Execute statement succeed.
-!info
-
-show tables;
-+------------+
-| table name |
-+------------+
-| test_table |
-+------------+
-1 row in set
-!ok
-
-show current database;
-+--------------------------+
-| current database name |
-+--------------------------+
-| additional_test_database |
-+--------------------------+
-1 row in set
-!ok
-
-# ==========================================================================
-# test hive table with parameterized types
-# ==========================================================================
-
-describe hivecatalog.`default`.param_types_table;
-+------+-----------------+------+-----+--------+-----------+
-| name | type | null | key | extras | watermark |
-+------+-----------------+------+-----+--------+-----------+
-| dec | DECIMAL(10, 10) | TRUE | | | |
-| ch | CHAR(5) | TRUE | | | |
-| vch | VARCHAR(15) | TRUE | | | |
-+------+-----------------+------+-----+--------+-----------+
-3 rows in set
-!ok
-
SET 'execution.runtime-mode' = 'batch';
[INFO] Execute statement succeed.
!info
@@ -288,16 +208,15 @@ SET 'sql-client.execution.result-mode' = 'tableau';
[INFO] Execute statement succeed.
!info
-# test the SELECT query can run successfully, even result is empty
-select * from hivecatalog.`default`.param_types_table;
-Empty set
-!ok
-
# ==========================================================================
# test create/drop table with catalog
# ==========================================================================
-use catalog hivecatalog;
+create catalog c2 with ('type'='generic_in_memory');
+[INFO] Execute statement succeed.
+!info
+
+use catalog `c2`;
[INFO] Execute statement succeed.
!info
@@ -314,11 +233,10 @@ show tables;
+------------+
| table name |
+------------+
-| mytable1 |
-| mytable2 |
-| test_table |
+| MyTable1 |
+| MyTable2 |
+------------+
-3 rows in set
+2 rows in set
!ok
show views;
@@ -337,8 +255,8 @@ show views;
+-----------+
| view name |
+-----------+
-| myview1 |
-| myview2 |
+| MyView1 |
+| MyView2 |
+-----------+
2 rows in set
!ok
diff --git a/flink-table/flink-sql-client/src/test/resources/sql/function.q b/flink-table/flink-sql-client/src/test/resources/sql/function.q
index d32ad6d64ae..f07abe6d384 100644
--- a/flink-table/flink-sql-client/src/test/resources/sql/function.q
+++ b/flink-table/flink-sql-client/src/test/resources/sql/function.q
@@ -250,31 +250,6 @@ SHOW JARS;
1 row in set
!ok
-# ==========================================================================
-# test function with hive catalog
-# ==========================================================================
-
-create catalog hivecatalog with ('type'='hive-test', 'hive-version'='2.3.4');
-[INFO] Execute statement succeed.
-!info
-
-use catalog hivecatalog;
-[INFO] Execute statement succeed.
-!info
-
-create function lowerudf AS 'LowerUDF';
-[INFO] Execute statement succeed.
-!info
-
-show user functions;
-+---------------+
-| function name |
-+---------------+
-| lowerudf |
-+---------------+
-1 row in set
-!ok
-
REMOVE JAR '$VAR_UDF_JAR_PATH';
[INFO] Execute statement succeed.
!info
diff --git a/flink-table/flink-sql-client/src/test/resources/sql/module.q b/flink-table/flink-sql-client/src/test/resources/sql/module.q
index c23cf67914d..ba2ecdc2989 100644
--- a/flink-table/flink-sql-client/src/test/resources/sql/module.q
+++ b/flink-table/flink-sql-client/src/test/resources/sql/module.q
@@ -55,10 +55,10 @@ SELECT SUBSTRING_INDEX('www.apache.org', '.', 2) FROM (VALUES (1, 'Hello World')
org.apache.calcite.sql.validate.SqlValidatorException: No match found for function signature SUBSTRING_INDEX(<CHARACTER>, <CHARACTER>, <NUMERIC>)
!error
-# load hive module with module name as string literal
-LOAD MODULE 'hive';
+# load dummy module with module name as string literal
+LOAD MODULE 'dummy';
[ERROR] Could not execute SQL statement. Reason:
-org.apache.flink.sql.parser.impl.ParseException: Encountered "\'hive\'" at line 1, column 13.
+org.apache.flink.sql.parser.impl.ParseException: Encountered "\'dummy\'" at line 1, column 13.
Was expecting one of:
<BRACKET_QUOTED_IDENTIFIER> ...
<QUOTED_IDENTIFIER> ...
@@ -69,24 +69,24 @@ Was expecting one of:
<UNICODE_QUOTED_IDENTIFIER> ...
!error
-# load hive module with module name capitalized
-LOAD MODULE Hive;
+# load dummy module with module name capitalized
+LOAD MODULE Dummy;
[ERROR] Could not execute SQL statement. Reason:
-org.apache.flink.table.api.ValidationException: Could not find any factory for identifier 'Hive' that implements 'org.apache.flink.table.factories.ModuleFactory' in the classpath.
+org.apache.flink.table.api.ValidationException: Could not find any factory for identifier 'Dummy' that implements 'org.apache.flink.table.factories.ModuleFactory' in the classpath.
Available factory identifiers are:
core
-hive
+dummy
!error
-# load hive module with specifying type
-LOAD MODULE myhive WITH ('type' = 'hive');
+# load dummy module with specifying type
+LOAD MODULE mydummy WITH ('type' = 'dummy');
[ERROR] Could not execute SQL statement. Reason:
-org.apache.flink.table.api.ValidationException: Option 'type' = 'hive' is not supported since module name is used to find module
+org.apache.flink.table.api.ValidationException: Option 'type' = 'dummy' is not supported since module name is used to find module
!error
-LOAD MODULE hive;
+LOAD MODULE dummy;
[INFO] Execute statement succeed.
!info
@@ -96,7 +96,7 @@ SHOW MODULES;
| module name |
+-------------+
| core |
-| hive |
+| dummy |
+-------------+
2 rows in set
!ok
@@ -107,33 +107,23 @@ SHOW FULL MODULES;
| module name | used |
+-------------+------+
| core | TRUE |
-| hive | TRUE |
+| dummy | TRUE |
+-------------+------+
2 rows in set
!ok
-# use hive built-in function after loading hive module
-SELECT SUBSTRING_INDEX('www.apache.org', '.', 2) FROM (VALUES (1, 'Hello World')) AS T(id, str);
-+----+--------------------------------+
-| op | EXPR$0 |
-+----+--------------------------------+
-| +I | www.apache |
-+----+--------------------------------+
-Received a total of 1 row
-!ok
-
# ==========================================================================
# test use modules
# ==========================================================================
# use duplicate modules
-USE MODULES hive, core, hive;
+USE MODULES dummy, core, dummy;
[ERROR] Could not execute SQL statement. Reason:
-org.apache.flink.table.api.ValidationException: Module 'hive' appears more than once
+org.apache.flink.table.api.ValidationException: Module 'dummy' appears more than once
!error
# change module resolution order
-USE MODULES hive, core;
+USE MODULES dummy, core;
[INFO] Execute statement succeed.
!info
@@ -141,7 +131,7 @@ SHOW MODULES;
+-------------+
| module name |
+-------------+
-| hive |
+| dummy |
| core |
+-------------+
2 rows in set
@@ -151,13 +141,13 @@ SHOW FULL MODULES;
+-------------+------+
| module name | used |
+-------------+------+
-| hive | TRUE |
+| dummy | TRUE |
| core | TRUE |
+-------------+------+
2 rows in set
!ok
-# disable hive module
+# disable dummy module
USE MODULES core;
[INFO] Execute statement succeed.
!info
@@ -176,17 +166,11 @@ SHOW FULL MODULES;
| module name | used |
+-------------+-------+
| core | TRUE |
-| hive | FALSE |
+| dummy | FALSE |
+-------------+-------+
2 rows in set
!ok
-# use hive built-in function without using hive module
-SELECT SUBSTRING_INDEX('www.apache.org', '.', 2) FROM (VALUES (1, 'Hello World')) AS T(id, str);
-[ERROR] Could not execute SQL statement. Reason:
-org.apache.calcite.sql.validate.SqlValidatorException: No match found for function signature SUBSTRING_INDEX(<CHARACTER>, <CHARACTER>, <NUMERIC>)
-!error
-
# ==========================================================================
# test unload module
# ==========================================================================
@@ -203,7 +187,7 @@ SHOW FULL MODULES;
+-------------+-------+
| module name | used |
+-------------+-------+
-| hive | FALSE |
+| dummy | FALSE |
+-------------+-------+
1 row in set
!ok
diff --git a/flink-table/flink-sql-client/src/test/resources/sql/set.q b/flink-table/flink-sql-client/src/test/resources/sql/set.q
index 80e7ead6115..b7cc6855a1d 100644
--- a/flink-table/flink-sql-client/src/test/resources/sql/set.q
+++ b/flink-table/flink-sql-client/src/test/resources/sql/set.q
@@ -20,76 +20,26 @@ SET 'sql-client.execution.result-mode' = 'tableau';
[INFO] Execute statement succeed.
!info
-SET 'table.sql-dialect' = 'hive';
+SET 'table.dml-sync' = 'true';
[INFO] Execute statement succeed.
!info
-create catalog hivecatalog with (
- 'type' = 'hive-test',
- 'hive-version' = '2.3.4'
-);
-[INFO] Execute statement succeed.
-!info
-
-use catalog hivecatalog;
-[INFO] Execute statement succeed.
-!info
-
-# test SET command
-set table.sql-dialect;
-+------------------------+
-| variables |
-+------------------------+
-| table.sql-dialect=hive |
-+------------------------+
-1 row in set
-!ok
-
-set k1=v1;
-[INFO] Execute statement succeed.
-!info
-
-set k1;
-+-----------+
-| variables |
-+-----------+
-| k1=v1 |
-+-----------+
-1 row in set
-!ok
-
-# test create a hive table to verify the configuration works
-CREATE TABLE hive_table (
- product_id STRING,
- product_name STRING,
- unit_price DECIMAL(10, 4),
- pv_count BIGINT,
- like_count BIGINT,
- comment_count BIGINT,
- update_time TIMESTAMP,
- update_user STRING
-) PARTITIONED BY (pt_year STRING, pt_month STRING, pt_day STRING) TBLPROPERTIES (
- 'streaming-source.enable' = 'true'
-);
-[INFO] Execute statement succeed.
-!info
-
-SET table.dml-sync = true;
-[INFO] Execute statement succeed.
-!info
-
-# test "ctas" in Hive Dialect
-CREATE TABLE foo as select 1;
+# test "ctas"
+CREATE TABLE foo with(
+ 'connector' = 'filesystem',
+ 'path' = '$VAR_STREAMING_PATH',
+ 'format' = 'csv'
+) as select id FROM (VALUES (1)) T(id);
[INFO] Complete execution of the SQL update statement.
!info
-RESET table.dml-sync;
+RESET 'table.dml-sync';
[INFO] Execute statement succeed.
!info
SELECT * from foo;
+----+-------------+
-| op | _o__c0 |
+| op | id |
+----+-------------+
| +I | 1 |
+----+-------------+
@@ -97,7 +47,7 @@ Received a total of 1 row
!ok
# test add jar
-ADD JAR $VAR_UDF_JAR_PATH;
+ADD JAR '$VAR_UDF_JAR_PATH';
[INFO] Execute statement succeed.
!info
@@ -110,19 +60,6 @@ SHOW JARS;
1 row in set
!ok
-CREATE FUNCTION hive_add_one as 'HiveAddOneFunc';
-[INFO] Execute statement succeed.
-!info
-
-SELECT hive_add_one(1);
-+----+-------------+
-| op | _o__c0 |
-+----+-------------+
-| +I | 2 |
-+----+-------------+
-Received a total of 1 row
-!ok
-
REMOVE JAR '$VAR_UDF_JAR_PATH';
[INFO] Execute statement succeed.
!info
@@ -131,7 +68,7 @@ SHOW JARS;
Empty set
!ok
-reset table.resources.download-dir;
+reset 'table.resources.download-dir';
[INFO] Execute statement succeed.
!info
@@ -146,15 +83,13 @@ set;
| execution.shutdown-on-attached-exit | false |
| execution.target | remote |
| jobmanager.rpc.address | $VAR_JOBMANAGER_RPC_ADDRESS |
-| k1 | v1 |
| pipeline.classpaths | |
| pipeline.jars | |
| rest.port | $VAR_REST_PORT |
| sql-client.execution.result-mode | tableau |
| table.exec.legacy-cast-behaviour | DISABLED |
-| table.sql-dialect | hive |
+--------------------------------------------+-----------+
-13 rows in set
+11 rows in set
!ok
# reset the configuration