You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2025/01/22 07:29:13 UTC
(paimon) branch release-1.0 updated: [hive] Fix listTablesImpl possible timeout issue (#4800)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-1.0
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/release-1.0 by this push:
new 49e093396d [hive] Fix listTablesImpl possible timeout issue (#4800)
49e093396d is described below
commit 49e093396dfca4ff645b789a8678832a4227cafa
Author: XiaoHongbo <13...@qq.com>
AuthorDate: Mon Dec 30 13:35:35 2024 +0800
[hive] Fix listTablesImpl possible timeout issue (#4800)
---
.../java/org/apache/paimon/hive/HiveCatalog.java | 55 +++++++++++++++++++++-
.../org/apache/paimon/hive/HiveCatalogTest.java | 53 +++++++++++++++++++++
2 files changed, 106 insertions(+), 2 deletions(-)
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
index 043cabbe35..e2274d7147 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java
@@ -54,6 +54,8 @@ import org.apache.paimon.utils.Preconditions;
import org.apache.paimon.view.View;
import org.apache.paimon.view.ViewImpl;
+import org.apache.paimon.shade.guava30.com.google.common.collect.Lists;
+
import org.apache.flink.table.hive.LegacyHiveClasses;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -137,7 +139,7 @@ public class HiveCatalog extends AbstractCatalog {
private static final String HIVE_PREFIX = "hive.";
public static final String HIVE_SITE_FILE = "hive-site.xml";
private static final String HIVE_EXTERNAL_TABLE_PROP = "EXTERNAL";
-
+ private static final int DEFAULT_TABLE_BATCH_SIZE = 300;
private final HiveConf hiveConf;
private final String clientClassName;
private final Options options;
@@ -434,8 +436,34 @@ public class HiveCatalog extends AbstractCatalog {
protected List<String> listTablesImpl(String databaseName) {
try {
List<String> tableNames = clients.run(client -> client.getAllTables(databaseName));
+ int batchSize = getBatchGetTableSize();
List<Table> hmsTables =
- clients.run(client -> client.getTableObjectsByName(databaseName, tableNames));
+ Lists.partition(tableNames, batchSize).stream()
+ .flatMap(
+ batchTableNames -> {
+ try {
+ return clients
+ .run(
+ client ->
+ client.getTableObjectsByName(
+ databaseName,
+ batchTableNames))
+ .stream();
+ } catch (TException e) {
+ throw new RuntimeException(
+ "Failed to getTableObjectsByName in database "
+ + databaseName,
+ e);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(
+ "Interrupted in call to getTableObjectsByName "
+ + databaseName,
+ e);
+ }
+ })
+ .collect(Collectors.toList());
+
List<String> result = new ArrayList<>(hmsTables.size());
for (Table table : hmsTables) {
if (isPaimonTable(table) || (!formatTableDisabled() && isFormatTable(table))) {
@@ -1406,4 +1434,27 @@ public class HiveCatalog extends AbstractCatalog {
public static String possibleHiveConfPath() {
return System.getenv("HIVE_CONF_DIR");
}
+
+ public int getBatchGetTableSize() {
+ try {
+ int size =
+ Integer.parseInt(
+ this.hiveConf.get(
+ HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname,
+ String.valueOf(
+ HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX
+ .getDefaultValue())));
+ if (size < 1) {
+ return DEFAULT_TABLE_BATCH_SIZE;
+ } else {
+ return size;
+ }
+ } catch (Exception e) {
+ LOG.warn(
+ "parse batch size failed {}, use default batch size",
+ this.hiveConf.get(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname),
+ e);
+ return DEFAULT_TABLE_BATCH_SIZE;
+ }
+ }
}
diff --git a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
index bf6eb02f3e..e733ec16c8 100644
--- a/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
+++ b/paimon-hive/paimon-hive-catalog/src/test/java/org/apache/paimon/hive/HiveCatalogTest.java
@@ -59,6 +59,7 @@ import static org.apache.paimon.hive.HiveCatalog.PAIMON_TABLE_IDENTIFIER;
import static org.apache.paimon.hive.HiveCatalog.TABLE_TYPE_PROP;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
/** Tests for {@link HiveCatalog}. */
@@ -354,6 +355,58 @@ public class HiveCatalogTest extends CatalogTestBase {
}
}
+ @Test
+ public void testListTables() throws Exception {
+ String databaseName = "testListTables";
+ catalog.dropDatabase(databaseName, true, true);
+ catalog.createDatabase(databaseName, true);
+ for (int i = 0; i < 500; i++) {
+ catalog.createTable(
+ Identifier.create(databaseName, "table" + i),
+ Schema.newBuilder().column("col", DataTypes.INT()).build(),
+ true);
+ }
+
+ // use default 300
+ List<String> defaultBatchTables = catalog.listTables(databaseName);
+
+ // use custom 400
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.set(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname, "400");
+ String metastoreClientClass = "org.apache.hadoop.hive.metastore.HiveMetaStoreClient";
+ List<String> customBatchTables;
+ try (HiveCatalog customCatalog =
+ new HiveCatalog(fileIO, hiveConf, metastoreClientClass, warehouse)) {
+ customBatchTables = customCatalog.listTables(databaseName);
+ } catch (Exception e) {
+ throw e;
+ }
+ assertEquals(defaultBatchTables.size(), customBatchTables.size());
+ defaultBatchTables.sort(String::compareTo);
+ customBatchTables.sort(String::compareTo);
+ for (int i = 0; i < defaultBatchTables.size(); i++) {
+ assertEquals(defaultBatchTables.get(i), customBatchTables.get(i));
+ }
+
+ // use invalid batch size
+ HiveConf invalidHiveConf = new HiveConf();
+ invalidHiveConf.set(HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX.varname, "dummy");
+ List<String> invalidBatchSizeTables;
+ try (HiveCatalog invalidBatchSizeCatalog =
+ new HiveCatalog(fileIO, invalidHiveConf, metastoreClientClass, warehouse)) {
+ invalidBatchSizeTables = invalidBatchSizeCatalog.listTables(databaseName);
+ } catch (Exception e) {
+ throw e;
+ }
+ assertEquals(defaultBatchTables.size(), invalidBatchSizeTables.size());
+ invalidBatchSizeTables.sort(String::compareTo);
+ for (int i = 0; i < defaultBatchTables.size(); i++) {
+ assertEquals(defaultBatchTables.get(i), invalidBatchSizeTables.get(i));
+ }
+
+ catalog.dropDatabase(databaseName, true, true);
+ }
+
@Override
protected boolean supportsView() {
return true;