You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2021/12/06 09:54:10 UTC
[iceberg] branch master updated: Hive: HiveCatalog should remove HMS stats for certain engines based on config (#3652)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new bfccea6 Hive: HiveCatalog should remove HMS stats for certain engines based on config (#3652)
bfccea6 is described below
commit bfccea6d7769afb3069d97437b8e2894c75d0c4f
Author: Marton Bod <mb...@cloudera.com>
AuthorDate: Mon Dec 6 10:53:55 2021 +0100
Hive: HiveCatalog should remove HMS stats for certain engines based on config (#3652)
---
.../apache/iceberg/hadoop/ConfigProperties.java | 1 +
.../apache/iceberg/hive/HiveTableOperations.java | 5 +++
.../TestHiveIcebergStorageHandlerWithEngine.java | 38 ++++++++++++++++++++++
.../org/apache/iceberg/mr/hive/TestTables.java | 15 +++++++++
4 files changed, 59 insertions(+)
diff --git a/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java b/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java
index bd0a414..58e5d63 100644
--- a/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java
+++ b/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java
@@ -25,4 +25,5 @@ public class ConfigProperties {
}
public static final String ENGINE_HIVE_ENABLED = "iceberg.engine.hive.enabled";
+ public static final String KEEP_HIVE_STATS = "iceberg.hive.keep.stats";
}
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index ee62814..e7048a9 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -212,6 +212,7 @@ public class HiveTableOperations extends BaseMetastoreTableOperations {
protected void doCommit(TableMetadata base, TableMetadata metadata) {
String newMetadataLocation = writeNewMetadata(metadata, currentVersion() + 1);
boolean hiveEngineEnabled = hiveEngineEnabled(metadata, conf);
+ boolean keepHiveStats = conf.getBoolean(ConfigProperties.KEEP_HIVE_STATS, false);
CommitStatus commitStatus = CommitStatus.FAILURE;
boolean updateHiveTable = false;
@@ -262,6 +263,10 @@ public class HiveTableOperations extends BaseMetastoreTableOperations {
.orElseGet(ImmutableMap::of);
setHmsTableParameters(newMetadataLocation, tbl, metadata.properties(), removedProps, hiveEngineEnabled, summary);
+ if (!keepHiveStats) {
+ tbl.getParameters().remove(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ }
+
try {
persistTable(tbl, updateHiveTable);
commitStatus = CommitStatus.SUCCESS;
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
index 72dc588..577cef7 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
@@ -26,6 +26,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
import org.apache.iceberg.FileFormat;
@@ -35,6 +36,7 @@ import org.apache.iceberg.Table;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
+import org.apache.iceberg.hadoop.ConfigProperties;
import org.apache.iceberg.hive.HiveSchemaUtil;
import org.apache.iceberg.hive.MetastoreUtil;
import org.apache.iceberg.mr.TestHelper;
@@ -739,6 +741,42 @@ public class TestHiveIcebergStorageHandlerWithEngine {
HiveIcebergTestUtils.validateData(target, ImmutableList.of(), 0);
}
+ @Test
+ public void testStatsPopulation() throws Exception {
+ Assume.assumeTrue("Tez write is not implemented yet", executionEngine.equals("mr"));
+ Assume.assumeTrue("Only HiveCatalog can remove stats which become obsolete",
+ testTableType == TestTables.TestTableType.HIVE_CATALOG);
+ shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true);
+
+ // create the table using a catalog which supports updating Hive stats (KEEP_HIVE_STATS is true)
+ shell.setHiveSessionValue(ConfigProperties.KEEP_HIVE_STATS, true);
+ TableIdentifier identifier = TableIdentifier.of("default", "customers");
+ testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of());
+
+ // insert some data and check the stats are up-to-date
+ String insert = testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false);
+ shell.executeStatement(insert);
+ String stats = shell.metastore().getTable(identifier).getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ Assert.assertTrue(stats.startsWith("{\"BASIC_STATS\":\"true\"")); // it's followed by column stats in Hive3
+
+ // Create a Catalog where the KEEP_HIVE_STATS is false
+ shell.metastore().hiveConf().set(ConfigProperties.KEEP_HIVE_STATS, StatsSetupConst.FALSE);
+ TestTables nonHiveTestTables = HiveIcebergStorageHandlerTestUtils.testTables(shell, testTableType, temp);
+ Table nonHiveTable = nonHiveTestTables.loadTable(identifier);
+
+ // Append data to the table through a non-Hive engine (in this case, via the java API) -> should remove stats
+ nonHiveTestTables.appendIcebergTable(shell.getHiveConf(), nonHiveTable, fileFormat, null,
+ HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
+ stats = shell.metastore().getTable(identifier).getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ Assert.assertNull(stats);
+
+ // insert some data again using Hive catalog, and check the stats are back
+ shell.executeStatement(insert);
+ stats = shell.metastore().getTable(identifier).getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+ Assert.assertTrue(stats.startsWith("{\"BASIC_STATS\":\"true\"")); // it's followed by column stats in Hive3
+ }
+
private void testComplexTypeWrite(Schema schema, List<Record> records) throws IOException {
String tableName = "complex_table";
Table table = testTables.createTable(shell, "complex_table", schema, fileFormat, ImmutableList.of());
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java
index 7543f6e..547df22 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java
@@ -218,6 +218,21 @@ abstract class TestTables {
return loadTable(identifier);
}
+ public String getInsertQuery(List<Record> records, TableIdentifier identifier, boolean isOverwrite) {
+ StringBuilder query = new StringBuilder(String.format("INSERT %s %s VALUES ",
+ isOverwrite ? "OVERWRITE TABLE" : "INTO", identifier));
+
+ records.forEach(record -> {
+ query.append("(");
+ query.append(record.struct().fields().stream()
+ .map(field -> getStringValueForInsert(record.getField(field.name()), field.type()))
+ .collect(Collectors.joining(",")));
+ query.append("),");
+ });
+ query.setLength(query.length() - 1);
+ return query.toString();
+ }
+
/**
* Creates a Hive test table. Creates the Iceberg table/data and creates the corresponding Hive table as well when
* needed. The table will be in the 'default' database. The table will be populated with the provided with randomly