[hive] branch master updated: HIVE-26756: Iceberg: Fetch format version from metadata file to avoid conflicts with spark. (#3778). (Ayush Saxena, reviewed by Adam Szita)

[ay...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new a880e8897bd HIVE-26756: Iceberg: Fetch format version from metadata file to avoid conflicts with spark. (#3778). (Ayush Saxena, reviewed by Adam Szita)
a880e8897bd is described below

commit a880e8897bdcf9404f5def03735796dbd6fc133f
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Thu Nov 24 18:04:04 2022 +0530

    HIVE-26756: Iceberg: Fetch format version from metadata file to avoid conflicts with spark. (#3778). (Ayush Saxena, reviewed by Adam Szita)
---
 .../org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java | 17 +++++++++++++++++
 .../mr/hive/TestHiveIcebergStorageHandlerNoScan.java    | 17 +++++++++++++++++
 .../queries/positive/ctas_iceberg_partitioned_orc.q     |  1 +
 .../org/apache/hadoop/hive/metastore/HiveMetaHook.java  |  8 ++++++++
 .../hadoop/hive/metastore/HiveMetaStoreClient.java      | 11 +++++++++++
 5 files changed, 54 insertions(+)

diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 6aaa8b4ed1b..9ba31fe0ac8 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -79,6 +79,7 @@ import org.apache.iceberg.UpdateProperties;
 import org.apache.iceberg.UpdateSchema;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.exceptions.NotFoundException;
 import org.apache.iceberg.expressions.Expressions;
 import org.apache.iceberg.hive.CachedClientPool;
 import org.apache.iceberg.hive.HiveCommitLock;
@@ -911,6 +912,22 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
     }
   }
 
+  @Override
+  public void postGetTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) {
+    if (hmsTable != null) {
+      try {
+        Table tbl = IcebergTableUtil.getTable(conf, hmsTable);
+        String formatVersion = String.valueOf(((BaseTable) tbl).operations().current().formatVersion());
+        // If it is not the default format version, then set it in the table properties.
+        if (!"1".equals(formatVersion)) {
+          hmsTable.getParameters().put(TableProperties.FORMAT_VERSION, formatVersion);
+        }
+      } catch (NoSuchTableException | NotFoundException ex) {
+        // If the table doesn't exist, ignore throwing exception from here
+      }
+    }
+  }
+
   private class PreAlterTableProperties {
     private String tableLocation;
     private String format;
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index 53ac887ecc6..b00017726b1 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -697,6 +697,23 @@ public class TestHiveIcebergStorageHandlerNoScan {
     }
   }
 
+  @Test
+  public void testFormatVersion() throws IOException {
+    Assume.assumeTrue(testTableType != TestTables.TestTableType.HIVE_CATALOG);
+    TableIdentifier tbl = TableIdentifier.of("default", "customers");
+    // Create the Iceberg table
+    testTables.createIcebergTable(shell.getHiveConf(), "customers", COMPLEX_SCHEMA, FileFormat.PARQUET,
+        Collections.singletonMap("format-version", "2"), Collections.emptyList());
+
+    shell.executeStatement("CREATE EXTERNAL TABLE customers " + "STORED BY ICEBERG " +
+        testTables.locationForCreateTableSQL(TableIdentifier.of("default", "customers")) +
+        testTables.propertiesForCreateTableSQL(ImmutableMap.of()));
+
+    String fmt = shell.executeAndStringify("show create table " + tbl);
+
+    Assert.assertTrue(fmt, fmt.contains("'format-version'='2'"));
+  }
+
   @Test
   public void testCreatePartitionedTableWithPropertiesAndWithColumnSpecification() {
     PartitionSpec spec =
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q b/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
index 3897c0e4ca0..5f558417092 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/ctas_iceberg_partitioned_orc.q
@@ -1,3 +1,4 @@
+--! qt:disabled:disabled the ctas doesn't set table as v2 in HMS & sets copy-on-write which is unsupported at Hive
 set hive.query.lifetime.hooks=org.apache.iceberg.mr.hive.HiveIcebergQueryLifeTimeHook;
 --! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
 set hive.explain.user=false;
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
index 3accd3cd9ea..ac624131430 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
@@ -177,4 +177,12 @@ public interface HiveMetaHook {
   default boolean createHMSTableInHook() {
     return false;
   }
+
+  /**
+   *  Set storage handler specific table properties
+   * @param table
+   */
+  default void postGetTable(Table table) {
+    // Do nothing
+  }
 }
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 91dbbe7fa4d..43ca7619fa1 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -2688,6 +2688,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
         getTableRequest.setProcessorIdentifier(processorIdentifier);
 
       Table t = getTableInternal(getTableRequest).getTable();
+      executePostGetTableHook(t);
       return deepCopy(FilterUtils.filterTableIfEnabled(isClientFilterEnabled, filterHook, t));
     } finally {
       long diff = System.currentTimeMillis() - t1;
@@ -2698,6 +2699,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
     }
   }
 
+  private void executePostGetTableHook(Table t) throws MetaException {
+    HiveMetaHook hook = getHook(t);
+    if (hook != null) {
+      hook.postGetTable(t);
+    }
+  }
+
   @Override
   public List<Table> getTableObjectsByName(String dbName, List<String> tableNames)
       throws TException {
@@ -2721,6 +2729,9 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
       req.setProcessorCapabilities(new ArrayList<String>(Arrays.asList(processorCapabilities)));
     req.setProjectionSpec(projectionsSpec);
     List<Table> tabs = client.get_table_objects_by_name_req(req).getTables();
+    for (Table tbl : tabs) {
+      executePostGetTableHook(tbl);
+    }
     return deepCopyTables(FilterUtils.filterTablesIfEnabled(isClientFilterEnabled, filterHook, tabs));
   }