You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2021/12/06 09:54:10 UTC

[iceberg] branch master updated: Hive: HiveCatalog should remove HMS stats for certain engines based on config (#3652)

This is an automated email from the ASF dual-hosted git repository.

pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new bfccea6  Hive: HiveCatalog should remove HMS stats for certain engines based on config (#3652)
bfccea6 is described below

commit bfccea6d7769afb3069d97437b8e2894c75d0c4f
Author: Marton Bod <mb...@cloudera.com>
AuthorDate: Mon Dec 6 10:53:55 2021 +0100

    Hive: HiveCatalog should remove HMS stats for certain engines based on config (#3652)
---
 .../apache/iceberg/hadoop/ConfigProperties.java    |  1 +
 .../apache/iceberg/hive/HiveTableOperations.java   |  5 +++
 .../TestHiveIcebergStorageHandlerWithEngine.java   | 38 ++++++++++++++++++++++
 .../org/apache/iceberg/mr/hive/TestTables.java     | 15 +++++++++
 4 files changed, 59 insertions(+)

diff --git a/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java b/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java
index bd0a414..58e5d63 100644
--- a/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java
+++ b/core/src/main/java/org/apache/iceberg/hadoop/ConfigProperties.java
@@ -25,4 +25,5 @@ public class ConfigProperties {
   }
 
   public static final String ENGINE_HIVE_ENABLED = "iceberg.engine.hive.enabled";
+  public static final String KEEP_HIVE_STATS = "iceberg.hive.keep.stats";
 }
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index ee62814..e7048a9 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -212,6 +212,7 @@ public class HiveTableOperations extends BaseMetastoreTableOperations {
   protected void doCommit(TableMetadata base, TableMetadata metadata) {
     String newMetadataLocation = writeNewMetadata(metadata, currentVersion() + 1);
     boolean hiveEngineEnabled = hiveEngineEnabled(metadata, conf);
+    boolean keepHiveStats = conf.getBoolean(ConfigProperties.KEEP_HIVE_STATS, false);
 
     CommitStatus commitStatus = CommitStatus.FAILURE;
     boolean updateHiveTable = false;
@@ -262,6 +263,10 @@ public class HiveTableOperations extends BaseMetastoreTableOperations {
           .orElseGet(ImmutableMap::of);
       setHmsTableParameters(newMetadataLocation, tbl, metadata.properties(), removedProps, hiveEngineEnabled, summary);
 
+      if (!keepHiveStats) {
+        tbl.getParameters().remove(StatsSetupConst.COLUMN_STATS_ACCURATE);
+      }
+
       try {
         persistTable(tbl, updateHiveTable);
         commitStatus = CommitStatus.SUCCESS;
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
index 72dc588..577cef7 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
@@ -26,6 +26,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import org.apache.hadoop.hive.common.StatsSetupConst;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.mr.ExecMapper;
 import org.apache.iceberg.FileFormat;
@@ -35,6 +36,7 @@ import org.apache.iceberg.Table;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.data.GenericRecord;
 import org.apache.iceberg.data.Record;
+import org.apache.iceberg.hadoop.ConfigProperties;
 import org.apache.iceberg.hive.HiveSchemaUtil;
 import org.apache.iceberg.hive.MetastoreUtil;
 import org.apache.iceberg.mr.TestHelper;
@@ -739,6 +741,42 @@ public class TestHiveIcebergStorageHandlerWithEngine {
     HiveIcebergTestUtils.validateData(target, ImmutableList.of(), 0);
   }
 
+  @Test
+  public void testStatsPopulation() throws Exception {
+    Assume.assumeTrue("Tez write is not implemented yet", executionEngine.equals("mr"));
+    Assume.assumeTrue("Only HiveCatalog can remove stats which become obsolete",
+        testTableType == TestTables.TestTableType.HIVE_CATALOG);
+    shell.setHiveSessionValue(HiveConf.ConfVars.HIVESTATSAUTOGATHER.varname, true);
+
+    // create the table using a catalog which supports updating Hive stats (KEEP_HIVE_STATS is true)
+    shell.setHiveSessionValue(ConfigProperties.KEEP_HIVE_STATS, true);
+    TableIdentifier identifier = TableIdentifier.of("default", "customers");
+    testTables.createTable(shell, identifier.name(), HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+        PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of());
+
+    // insert some data and check the stats are up-to-date
+    String insert = testTables.getInsertQuery(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS, identifier, false);
+    shell.executeStatement(insert);
+    String stats = shell.metastore().getTable(identifier).getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+    Assert.assertTrue(stats.startsWith("{\"BASIC_STATS\":\"true\"")); // it's followed by column stats in Hive3
+
+    // Create a Catalog where the KEEP_HIVE_STATS is false
+    shell.metastore().hiveConf().set(ConfigProperties.KEEP_HIVE_STATS, StatsSetupConst.FALSE);
+    TestTables nonHiveTestTables = HiveIcebergStorageHandlerTestUtils.testTables(shell, testTableType, temp);
+    Table nonHiveTable = nonHiveTestTables.loadTable(identifier);
+
+    // Append data to the table through a non-Hive engine (in this case, via the java API) -> should remove stats
+    nonHiveTestTables.appendIcebergTable(shell.getHiveConf(), nonHiveTable, fileFormat, null,
+        HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS);
+    stats = shell.metastore().getTable(identifier).getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+    Assert.assertNull(stats);
+
+    // insert some data again using Hive catalog, and check the stats are back
+    shell.executeStatement(insert);
+    stats = shell.metastore().getTable(identifier).getParameters().get(StatsSetupConst.COLUMN_STATS_ACCURATE);
+    Assert.assertTrue(stats.startsWith("{\"BASIC_STATS\":\"true\"")); // it's followed by column stats in Hive3
+  }
+
   private void testComplexTypeWrite(Schema schema, List<Record> records) throws IOException {
     String tableName = "complex_table";
     Table table = testTables.createTable(shell, "complex_table", schema, fileFormat, ImmutableList.of());
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java
index 7543f6e..547df22 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestTables.java
@@ -218,6 +218,21 @@ abstract class TestTables {
     return loadTable(identifier);
   }
 
+  public String getInsertQuery(List<Record> records, TableIdentifier identifier, boolean isOverwrite) {
+    StringBuilder query = new StringBuilder(String.format("INSERT %s %s VALUES ",
+            isOverwrite ? "OVERWRITE TABLE" : "INTO", identifier));
+
+    records.forEach(record -> {
+      query.append("(");
+      query.append(record.struct().fields().stream()
+              .map(field -> getStringValueForInsert(record.getField(field.name()), field.type()))
+              .collect(Collectors.joining(",")));
+      query.append("),");
+    });
+    query.setLength(query.length() - 1);
+    return query.toString();
+  }
+
   /**
    * Creates a Hive test table. Creates the Iceberg table/data and creates the corresponding Hive table as well when
    * needed. The table will be in the 'default' database. The table will be populated with the provided with randomly