You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/11/16 07:06:39 UTC
(incubator-paimon) branch master updated: [Hive] Using hive table location when catalog is HiveCatalog (#2315)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1bd1c1151 [Hive] Using hive table location when catalog is HiveCatalog (#2315)
1bd1c1151 is described below
commit 1bd1c1151c5abdb057880f3402b268286df67108
Author: humengyu <he...@gmail.com>
AuthorDate: Thu Nov 16 15:06:35 2023 +0800
[Hive] Using hive table location when catalog is HiveCatalog (#2315)
---
.../java/org/apache/paimon/hive/HiveCatalog.java | 27 +++++++++++++++
.../org/apache/paimon/hive/LocationHelper.java | 5 +++
.../apache/paimon/hive/StorageLocationHelper.java | 16 ++++++---
.../paimon/hive/TBPropertiesLocationHelper.java | 21 ++++++++++++
.../apache/paimon/hive/Hive23CatalogITCase.java | 11 +++---
.../apache/paimon/hive/Hive31CatalogITCase.java | 10 +++---
.../org/apache/paimon/hive/HiveWriteITCase.java | 40 +---------------------
.../org/apache/paimon/hive/FileStoreTestUtils.java | 2 +-
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 11 +++---
.../org/apache/paimon/hive/HiveWriteITCase.java | 10 +++---
.../paimon/hive/PaimonStorageHandlerITCase.java | 2 +-
11 files changed, 91 insertions(+), 64 deletions(-)
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 2a857447e..ca87d2088 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -162,6 +162,33 @@ public class HiveCatalog extends AbstractCatalog {
}
}
+ @Override
+ public Path getDataTableLocation(Identifier identifier) {
+ try {
+ String databaseName = identifier.getDatabaseName();
+ String tableName = identifier.getObjectName();
+ if (client.tableExists(databaseName, tableName)) {
+ String location =
+ locationHelper.getTableLocation(client.getTable(databaseName, tableName));
+ if (location != null) {
+ return new Path(location);
+ }
+ } else {
+ // If the table does not exist,
+ // we should use the database path to generate the table path.
+ String dbLocation =
+ locationHelper.getDatabaseLocation(client.getDatabase(databaseName));
+ if (dbLocation != null) {
+ return new Path(dbLocation, tableName);
+ }
+ }
+
+ return super.getDataTableLocation(identifier);
+ } catch (TException e) {
+ throw new RuntimeException("Can not get table " + identifier + " from metastore.", e);
+ }
+ }
+
@Override
public List<String> listDatabases() {
try {
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java
index 409b79d58..42a9d5f38 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/LocationHelper.java
@@ -28,11 +28,16 @@ import java.io.IOException;
/** helper for specifying the storage location of hive table. */
public interface LocationHelper {
+
void createPathIfRequired(Path dbPath, FileIO fileIO) throws IOException;
void dropPathIfRequired(Path path, FileIO fileIO) throws IOException;
void specifyTableLocation(Table table, String location);
+ String getTableLocation(Table table);
+
void specifyDatabaseLocation(Path path, Database database);
+
+ String getDatabaseLocation(Database database);
}
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java
index 6d62d68f3..e8687fa88 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/StorageLocationHelper.java
@@ -24,20 +24,18 @@ import org.apache.paimon.fs.Path;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.Table;
-import java.io.IOException;
-
/** Helper for Setting Location in Hive Table Storage. */
public final class StorageLocationHelper implements LocationHelper {
public StorageLocationHelper() {}
@Override
- public void createPathIfRequired(Path dbPath, FileIO fileIO) throws IOException {
+ public void createPathIfRequired(Path dbPath, FileIO fileIO) {
// do nothing
}
@Override
- public void dropPathIfRequired(Path path, FileIO fileIO) throws IOException {
+ public void dropPathIfRequired(Path path, FileIO fileIO) {
// do nothing
}
@@ -46,8 +44,18 @@ public final class StorageLocationHelper implements LocationHelper {
table.getSd().setLocation(location);
}
+ @Override
+ public String getTableLocation(Table table) {
+ return table.getSd().getLocation();
+ }
+
@Override
public void specifyDatabaseLocation(Path path, Database database) {
database.setLocationUri(path.toString());
}
+
+ @Override
+ public String getDatabaseLocation(Database database) {
+ return database.getLocationUri();
+ }
}
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java
index a374907ed..28079473b 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/TBPropertiesLocationHelper.java
@@ -51,6 +51,16 @@ public final class TBPropertiesLocationHelper implements LocationHelper {
table.getParameters().put(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY, location);
}
+ @Override
+ public String getTableLocation(Table table) {
+ String location = table.getParameters().get(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY);
+ if (location != null) {
+ return location;
+ }
+
+ return table.getSd().getLocation();
+ }
+
@Override
public void specifyDatabaseLocation(Path path, Database database) {
HashMap<String, String> properties = new HashMap<>();
@@ -60,4 +70,15 @@ public final class TBPropertiesLocationHelper implements LocationHelper {
properties.put(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY, path.toString());
database.setParameters(properties);
}
+
+ @Override
+ public String getDatabaseLocation(Database database) {
+ String location =
+ database.getParameters().get(LocationKeyExtractor.TBPROPERTIES_LOCATION_KEY);
+ if (location != null) {
+ return location;
+ }
+
+ return database.getLocationUri();
+ }
}
diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
index af1c8f405..7c2f96664 100644
--- a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
+++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java
@@ -71,6 +71,7 @@ public class Hive23CatalogITCase extends HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
+ " 'default-database' = 'test_db',",
" 'warehouse' = '" + path + "',",
" 'metastore.client.class' = '"
+ TestHiveMetaStoreClient.class.getName()
@@ -95,6 +96,7 @@ public class Hive23CatalogITCase extends HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
+ " 'default-database' = 'test_db',",
" 'warehouse' = '" + path + "',",
" 'metastore.client.class' = '"
+ CreateFailHiveMetaStoreClient.class.getName()
@@ -108,12 +110,12 @@ public class Hive23CatalogITCase extends HiveCatalogITCaseBase {
.await())
.isInstanceOf(TableException.class)
.hasMessage(
- "Could not execute CreateTable in path `my_hive_custom_client`.`default`.`hive_table`");
+ "Could not execute CreateTable in path `my_hive_custom_client`.`test_db`.`hive_table`");
assertThat(
new SchemaManager(
LocalFileIO.create(),
new org.apache.paimon.fs.Path(
- path, "default.db/hive_table"))
+ path, "test_db.db/hive_table"))
.listAllIds())
.isEmpty();
}
@@ -127,6 +129,7 @@ public class Hive23CatalogITCase extends HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
+ " 'default-database' = 'test_db',",
" 'warehouse' = '" + path + "',",
" 'metastore.client.class' = '"
+ AlterFailHiveMetaStoreClient.class.getName()
@@ -140,13 +143,13 @@ public class Hive23CatalogITCase extends HiveCatalogITCaseBase {
.satisfies(
AssertionUtils.anyCauseMatches(
TableException.class,
- "Could not execute AlterTable in path `my_alter_hive`.`default`.`alter_failed_table`"));
+ "Could not execute AlterTable in path `my_alter_hive`.`test_db`.`alter_failed_table`"));
assertThat(
new SchemaManager(
LocalFileIO.create(),
new org.apache.paimon.fs.Path(
- path, "default.db/alter_failed_table"))
+ path, "test_db.db/alter_failed_table"))
.latest()
.get()
.options())
diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
index 400036484..b4b421fc7 100644
--- a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
+++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java
@@ -95,6 +95,7 @@ public class Hive31CatalogITCase extends HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
+ " 'default-database' = 'test_db',",
" 'warehouse' = '" + path + "',",
" 'metastore.client.class' = '"
+ CreateFailHiveMetaStoreClient.class.getName()
@@ -108,12 +109,12 @@ public class Hive31CatalogITCase extends HiveCatalogITCaseBase {
.await())
.isInstanceOf(TableException.class)
.hasMessage(
- "Could not execute CreateTable in path `my_hive_custom_client`.`default`.`hive_table`");
+ "Could not execute CreateTable in path `my_hive_custom_client`.`test_db`.`hive_table`");
assertThat(
new SchemaManager(
LocalFileIO.create(),
new org.apache.paimon.fs.Path(
- path, "default.db/hive_table"))
+ path, "test_db.db/hive_table"))
.listAllIds())
.isEmpty();
}
@@ -127,6 +128,7 @@ public class Hive31CatalogITCase extends HiveCatalogITCaseBase {
" 'type' = 'paimon',",
" 'metastore' = 'hive',",
" 'uri' = '',",
+ " 'default-database' = 'test_db',",
" 'warehouse' = '" + path + "',",
" 'metastore.client.class' = '"
+ AlterFailHiveMetaStoreClient.class.getName()
@@ -140,13 +142,13 @@ public class Hive31CatalogITCase extends HiveCatalogITCaseBase {
.satisfies(
AssertionUtils.anyCauseMatches(
TableException.class,
- "Could not execute AlterTable in path `my_alter_hive`.`default`.`alter_failed_table`"));
+ "Could not execute AlterTable in path `my_alter_hive`.`test_db`.`alter_failed_table`"));
assertThat(
new SchemaManager(
LocalFileIO.create(),
new org.apache.paimon.fs.Path(
- path, "default.db/alter_failed_table"))
+ path, "test_db.db/alter_failed_table"))
.latest()
.get()
.options())
diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
index 5f4bbd5c1..babbbe30f 100644
--- a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
+++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
@@ -90,51 +90,13 @@ public class HiveWriteITCase {
hiveShell.execute("DROP DATABASE IF EXISTS test_db CASCADE");
}
- private String createChangelogExternalTable(
- RowType rowType,
- List<String> partitionKeys,
- List<String> primaryKeys,
- List<InternalRow> data)
- throws Exception {
-
- return createChangelogExternalTable(rowType, partitionKeys, primaryKeys, data, "");
- }
-
- private String createChangelogExternalTable(
- RowType rowType,
- List<String> partitionKeys,
- List<String> primaryKeys,
- List<InternalRow> data,
- String tableName)
- throws Exception {
- String path = folder.newFolder().toURI().toString();
- String tableNameNotNull =
- StringUtils.isNullOrWhitespaceOnly(tableName) ? TABLE_NAME : tableName;
- String tablePath = String.format("%s/default.db/%s", path, tableNameNotNull);
- Options conf = new Options();
- conf.set(CatalogOptions.WAREHOUSE, path);
- conf.set(CoreOptions.BUCKET, 1);
- conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
- Identifier identifier = Identifier.create(DATABASE_NAME, tableNameNotNull);
- Table table =
- FileStoreTestUtils.createFileStoreTable(
- conf, rowType, partitionKeys, primaryKeys, identifier);
-
- return writeData(table, tablePath, data);
- }
-
- private String createAppendOnlyExternalTable(
- RowType rowType, List<String> partitionKeys, List<InternalRow> data) throws Exception {
- return createAppendOnlyExternalTable(rowType, partitionKeys, data, "");
- }
-
private String createAppendOnlyExternalTable(
RowType rowType, List<String> partitionKeys, List<InternalRow> data, String tableName)
throws Exception {
String path = folder.newFolder().toURI().toString();
String tableNameNotNull =
StringUtils.isNullOrWhitespaceOnly(tableName) ? TABLE_NAME : tableName;
- String tablePath = String.format("%s/default.db/%s", path, tableNameNotNull);
+ String tablePath = String.format("%s/test_db.db/%s", path, tableNameNotNull);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 2);
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FileStoreTestUtils.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FileStoreTestUtils.java
index c93f55e86..f5d9541ea 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FileStoreTestUtils.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/FileStoreTestUtils.java
@@ -35,7 +35,7 @@ public class FileStoreTestUtils {
public static final String TABLE_NAME = "hive_test_table";
- public static final String DATABASE_NAME = "default";
+ public static final String DATABASE_NAME = "test_db";
public static Table createFileStoreTable(
Options conf, RowType rowType, List<String> partitionKeys, List<String> primaryKeys)
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index 46646ad14..022a455a1 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -86,12 +86,6 @@ public abstract class HiveCatalogITCaseBase {
@Minio private static MinioTestContainer minioTestContainer;
private void before(boolean locationInProperties) throws Exception {
- hiveShell.execute("CREATE DATABASE IF NOT EXISTS test_db");
- hiveShell.execute("USE test_db");
- hiveShell.execute("CREATE TABLE hive_table ( a INT, b STRING )");
- hiveShell.execute("INSERT INTO hive_table VALUES (100, 'Hive'), (200, 'Table')");
- hiveShell.executeQuery("SHOW TABLES");
-
Map<String, String> catalogProperties = new HashMap<>();
catalogProperties.put("type", "paimon");
catalogProperties.put("metastore", "hive");
@@ -122,7 +116,12 @@ public abstract class HiveCatalogITCaseBase {
")"))
.await();
tEnv.executeSql("USE CATALOG my_hive").await();
+ tEnv.executeSql("DROP DATABASE IF EXISTS test_db CASCADE");
+ tEnv.executeSql("CREATE DATABASE test_db").await();
tEnv.executeSql("USE test_db").await();
+ hiveShell.execute("USE test_db");
+ hiveShell.execute("CREATE TABLE hive_table ( a INT, b STRING )");
+ hiveShell.execute("INSERT INTO hive_table VALUES (100, 'Hive'), (200, 'Table')");
}
private void after() {
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
index f81476bf9..f50e977d5 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveWriteITCase.java
@@ -129,7 +129,7 @@ public class HiveWriteITCase {
String path = folder.newFolder().toURI().toString();
String tableNameNotNull =
StringUtils.isNullOrWhitespaceOnly(tableName) ? TABLE_NAME : tableName;
- String tablePath = String.format("%s/default.db/%s", path, tableNameNotNull);
+ String tablePath = String.format("%s/test_db.db/%s", path, tableNameNotNull);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 2);
@@ -153,7 +153,7 @@ public class HiveWriteITCase {
String path = folder.newFolder().toURI().toString();
String tableNameNotNull =
StringUtils.isNullOrWhitespaceOnly(tableName) ? TABLE_NAME : tableName;
- String tablePath = String.format("%s/default.db/%s", path, tableNameNotNull);
+ String tablePath = String.format("%s/test_db.db/%s", path, tableNameNotNull);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 2);
@@ -223,7 +223,7 @@ public class HiveWriteITCase {
String innerName = "hive_test_table_output";
String path = folder.newFolder().toURI().toString();
- String tablePath = String.format("%s/default.db/%s", path, innerName);
+ String tablePath = String.format("%s/test_db.db/%s", path, innerName);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 1);
@@ -267,7 +267,7 @@ public class HiveWriteITCase {
String innerName = "hive_test_table_output";
int maxCompact = 3;
String path = folder.newFolder().toURI().toString();
- String tablePath = String.format("%s/default.db/%s", path, innerName);
+ String tablePath = String.format("%s/test_db.db/%s", path, innerName);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);
conf.set(CoreOptions.BUCKET, 1);
@@ -611,7 +611,7 @@ public class HiveWriteITCase {
public void testInsertAllSupportedTypes() throws Exception {
String root = folder.newFolder().toString();
- String tablePath = String.format("%s/default.db/hive_test_table", root);
+ String tablePath = String.format("%s/test_db.db/hive_test_table", root);
Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, root);
conf.set(CoreOptions.FILE_FORMAT, CoreOptions.FileFormatType.AVRO);
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
index 6d568c6fb..16f6e191a 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
@@ -118,7 +118,7 @@ public class PaimonStorageHandlerITCase {
hiveShell.execute("USE test_db");
warehouse = folder.newFolder().toURI().toString();
- tablePath = String.format("%s/default.db/%s", warehouse, TABLE_NAME);
+ tablePath = String.format("%s/test_db.db/%s", warehouse, TABLE_NAME);
identifier = Identifier.create(DATABASE_NAME, TABLE_NAME);
externalTable = "test_table_" + UUID.randomUUID().toString().substring(0, 4);
commitIdentifier = 0;