You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2022/03/28 07:52:41 UTC
[hive] branch master updated: HIVE-26061: Do not add 'from deserializer' comment upon alter commands for Iceberg tables (Peter Vary reviewed by Marton Bod) (#3129)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new b05bc6e HIVE-26061: Do not add 'from deserializer' comment upon alter commands for Iceberg tables (Peter Vary reviewed by Marton Bod) (#3129)
b05bc6e is described below
commit b05bc6e33d4dd253cf00d8f1c28bb0b2a8f50750
Author: pvary <pv...@cloudera.com>
AuthorDate: Mon Mar 28 09:52:15 2022 +0200
HIVE-26061: Do not add 'from deserializer' comment upon alter commands for Iceberg tables (Peter Vary reviewed by Marton Bod) (#3129)
---
.../org/apache/iceberg/hive/TestHiveMetastore.java | 3 +-
.../mr/hive/TestHiveIcebergSchemaEvolution.java | 16 +-
.../hive/TestHiveIcebergStorageHandlerNoScan.java | 33 ++-
.../test/queries/positive/llap_iceberg_read_orc.q | 4 +-
.../alter_multi_part_table_to_iceberg.q.out | 18 +-
.../positive/alter_part_table_to_iceberg.q.out | 12 +-
.../results/positive/alter_table_to_iceberg.q.out | 12 +-
.../results/positive/create_iceberg_table.q.out | 8 +-
...create_iceberg_table_stored_as_fileformat.q.out | 40 ++--
.../create_iceberg_table_stored_by_iceberg.q.out | 8 +-
...le_stored_by_iceberg_with_serdeproperties.q.out | 8 +-
.../describe_iceberg_metadata_tables.q.out | 228 ++++++++++-----------
.../results/positive/describe_iceberg_table.q.out | 50 ++---
.../positive/llap/llap_iceberg_read_orc.q.out | 8 +-
.../llap/vectorized_iceberg_read_mixed.q.out | 8 +-
.../llap/vectorized_iceberg_read_orc.q.out | 8 +-
.../llap/vectorized_iceberg_read_parquet.q.out | 8 +-
.../positive/show_create_iceberg_table.q.out | 42 ++--
.../positive/truncate_force_iceberg_table.q.out | 8 +-
.../results/positive/truncate_iceberg_table.q.out | 16 +-
.../truncate_partitioned_iceberg_table.q.out | 8 +-
.../positive/vectorized_iceberg_read_mixed.q.out | 8 +-
.../positive/vectorized_iceberg_read_orc.q.out | 8 +-
.../positive/vectorized_iceberg_read_parquet.q.out | 8 +-
.../hive/serde2/TestSerdeWithFieldComments.java | 3 +-
.../hadoop/hive/metastore/HiveMetaStoreUtils.java | 17 +-
.../hive/metastore/SerDeStorageSchemaReader.java | 2 +-
.../java/org/apache/hadoop/hive/ql/Compiler.java | 3 +-
.../update/AlterTableUpdateColumnsOperation.java | 3 +-
.../ql/ddl/table/info/desc/DescTableOperation.java | 8 +-
.../storage/serde/AlterTableSetSerdeOperation.java | 2 +-
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 20 +-
.../apache/hadoop/hive/ql/metadata/Partition.java | 5 +-
.../org/apache/hadoop/hive/ql/metadata/Table.java | 5 +-
.../hadoop/hive/metastore/conf/MetastoreConf.java | 4 +
35 files changed, 325 insertions(+), 317 deletions(-)
diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
index 0d86d6f..a956d36 100644
--- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
+++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveMetastore.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.hive.metastore.IHMSHandler;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.RetryingHMSHandler;
import org.apache.hadoop.hive.metastore.TSetIpAddressProcessor;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil;
@@ -196,7 +197,7 @@ public class TestHiveMetastore {
}
public Table getTable(String dbName, String tableName) throws TException, InterruptedException {
- return clientPool.run(client -> client.getTable(dbName, tableName));
+ return clientPool.run(client -> client.getTable(new GetTableRequest(dbName, tableName)));
}
public Table getTable(TableIdentifier identifier) throws TException, InterruptedException {
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSchemaEvolution.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSchemaEvolution.java
index 076d6af..fdf9392 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSchemaEvolution.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergSchemaEvolution.java
@@ -58,7 +58,7 @@ public class TestHiveIcebergSchemaEvolution extends HiveIcebergStorageHandlerWit
Assert.assertEquals(HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA.columns().size(), rows.size());
for (int i = 0; i < HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA.columns().size(); i++) {
Types.NestedField field = HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA.columns().get(i);
- String comment = field.doc() == null ? "from deserializer" : field.doc();
+ String comment = field.doc() == null ? "" : field.doc();
Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
comment}, rows.get(i));
}
@@ -222,17 +222,15 @@ public class TestHiveIcebergSchemaEvolution extends HiveIcebergStorageHandlerWit
// Drop columns via REPLACE COLUMNS
shell.executeStatement("ALTER TABLE orders REPLACE COLUMNS (" +
- "customer_last_name string COMMENT 'from deserializer', order_id int COMMENT 'from deserializer'," +
- " quantity int, nick string COMMENT 'from deserializer'," +
- " fruit string COMMENT 'from deserializer')");
+ "customer_last_name string, order_id int, quantity int, nick string, fruit string)");
result = shell.executeStatement("DESCRIBE orders");
assertQueryResult(result, 5,
- "customer_last_name", "string", "from deserializer",
- "order_id", "int", "from deserializer",
- "quantity", "int", "from deserializer",
- "nick", "string", "from deserializer",
- "fruit", "string", "from deserializer");
+ "customer_last_name", "string", "",
+ "order_id", "int", "",
+ "quantity", "int", "",
+ "nick", "string", "",
+ "fruit", "string", "");
result = shell.executeStatement("SELECT * FROM orders ORDER BY order_id");
assertQueryResult(result, 7,
"Strange", 1, 100, null, "apple",
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index a4b7554..8c45921 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -793,7 +793,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
}
@Test
- public void testCreateTableWithColumnComments() {
+ public void testCreateTableWithColumnComments() throws InterruptedException, TException {
TableIdentifier identifier = TableIdentifier.of("default", "comment_table");
shell.executeStatement("CREATE EXTERNAL TABLE comment_table (" +
"t_int INT COMMENT 'int column', " +
@@ -809,8 +809,12 @@ public class TestHiveIcebergStorageHandlerNoScan {
for (int i = 0; i < icebergTable.schema().columns().size(); i++) {
Types.NestedField field = icebergTable.schema().columns().get(i);
Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
- field.doc() != null ? field.doc() : "from deserializer"}, rows.get(i));
+ field.doc() != null ? field.doc() : ""}, rows.get(i));
}
+
+ // Check the columns directly
+ List<FieldSchema> cols = shell.metastore().getTable(identifier).getSd().getCols();
+ Assert.assertEquals(icebergTable.schema().asStruct(), HiveSchemaUtil.convert(cols).asStruct());
}
@Test
@@ -829,8 +833,8 @@ public class TestHiveIcebergStorageHandlerNoScan {
for (int i = 0; i < icebergTable.schema().columns().size(); i++) {
Types.NestedField field = icebergTable.schema().columns().get(i);
Assert.assertNull(field.doc());
- Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
- "from deserializer"}, rows.get(i));
+ Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(), ""},
+ rows.get(i));
}
}
@@ -855,7 +859,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
for (int i = 0; i < columns.size(); i++) {
Types.NestedField field = columns.get(i);
Assert.assertArrayEquals(new Object[] {field.name(), HiveSchemaUtil.convert(field.type()).getTypeName(),
- field.doc() != null ? field.doc() : "from deserializer"}, rows.get(i));
+ field.doc() != null ? field.doc() : ""}, rows.get(i));
Assert.assertEquals(expectedDoc[i], field.doc());
}
}
@@ -1172,8 +1176,12 @@ public class TestHiveIcebergStorageHandlerNoScan {
);
testTables.createTable(shell, identifier.name(), schema, SPEC, FileFormat.PARQUET, ImmutableList.of());
+ // Run some alter commands. Before the fix the alter changed the column comment, and we would like to check
+ // that this is fixed now.
+ shell.executeStatement("ALTER TABLE default.customers CHANGE COLUMN customer_id customer_id bigint");
+
shell.executeStatement("ALTER TABLE default.customers REPLACE COLUMNS " +
- "(customer_id int, last_name string COMMENT 'This is last name', " +
+ "(customer_id bigint, last_name string COMMENT 'This is last name', " +
"address struct<city:string,street:string>)");
org.apache.iceberg.Table icebergTable = testTables.loadTable(identifier);
@@ -1183,7 +1191,7 @@ public class TestHiveIcebergStorageHandlerNoScan {
List<FieldSchema> hmsSchema = hmsTable.getSd().getCols();
List<FieldSchema> expectedSchema = Lists.newArrayList(
- new FieldSchema("customer_id", "int", null),
+ new FieldSchema("customer_id", "bigint", null),
// first_name column is dropped
new FieldSchema("last_name", "string", "This is last name"),
new FieldSchema("address", "struct<city:string,street:string>", null));
@@ -1265,9 +1273,6 @@ public class TestHiveIcebergStorageHandlerNoScan {
new FieldSchema("family_name", "string", "This is family name"));
Assert.assertEquals(expectedSchema, icebergSchema);
- if (testTableType != TestTables.TestTableType.HIVE_CATALOG) {
- expectedSchema.stream().filter(fs -> fs.getComment() == null).forEach(fs -> fs.setComment("from deserializer"));
- }
Assert.assertEquals(expectedSchema, hmsSchema);
}
@@ -1295,9 +1300,6 @@ public class TestHiveIcebergStorageHandlerNoScan {
new FieldSchema("last_name", "string", "This is last name"));
Assert.assertEquals(expectedSchema, icebergSchema);
- if (testTableType != TestTables.TestTableType.HIVE_CATALOG) {
- expectedSchema.stream().filter(fs -> fs.getComment() == null).forEach(fs -> fs.setComment("from deserializer"));
- }
Assert.assertEquals(expectedSchema, hmsSchema);
}
@@ -1378,11 +1380,6 @@ public class TestHiveIcebergStorageHandlerNoScan {
new FieldSchema("newstringcol", "string", "Column with description"));
Assert.assertEquals(expectedSchema, icebergSchema);
-
- if (testTableType != TestTables.TestTableType.HIVE_CATALOG) {
- expectedSchema.get(0).setComment("from deserializer");
- }
-
Assert.assertEquals(expectedSchema, hmsSchema);
}
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc.q b/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc.q
index b171157..2d62dd8 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/llap_iceberg_read_orc.q
@@ -59,7 +59,7 @@ INSERT INTO llap_items VALUES
SELECT cat, min(to60) from llap_items group by cat;
--removing a column
-ALTER TABLE llap_items REPLACE COLUMNS (itemid int, name string, cost int COMMENT 'from deserializer', description string, to60 float);
+ALTER TABLE llap_items REPLACE COLUMNS (itemid int, name string, cost int, description string, to60 float);
INSERT INTO llap_items VALUES
(8, 'Cybertruck', 40000, 'Single Motor RWD', 6.5),
(9, 'Cybertruck', 50000, 'Dual Motor AWD', 4.5);
@@ -96,7 +96,7 @@ INSERT INTO llap_orders VALUES
SELECT state, max(city), avg(itemid) from llap_orders WHERE region = 'EU' GROUP BY state;
--removing a column from schema
-ALTER TABLE llap_orders REPLACE COLUMNS (quantity int, itemid int, region string COMMENT 'from deserializer', state string COMMENT 'from deserializer', ordertime timestamp COMMENT 'from deserializer', city string);
+ALTER TABLE llap_orders REPLACE COLUMNS (quantity int, itemid int, region string, state string, ordertime timestamp, city string);
INSERT INTO llap_orders VALUES
(88, 5, 'EU', 'FR', timestamp('2006-02-04 19:55:46.129'), 'Paris');
SELECT state, max(city), avg(itemid) from llap_orders WHERE region = 'EU' GROUP BY state;
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
index 53603e2..d6004e4 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
@@ -166,9 +166,9 @@ POSTHOOK: query: describe formatted tbl_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_orc
# col_name data_type comment
-a int from deserializer
-b string from deserializer
-c string from deserializer
+a int
+b string
+c string
# Partition Transform Information
# col_name transform_type
@@ -415,9 +415,9 @@ POSTHOOK: query: describe formatted tbl_parquet
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet
# col_name data_type comment
-a int from deserializer
-b string from deserializer
-c string from deserializer
+a int
+b string
+c string
# Partition Transform Information
# col_name transform_type
@@ -664,9 +664,9 @@ POSTHOOK: query: describe formatted tbl_avro
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_avro
# col_name data_type comment
-a int from deserializer
-b string from deserializer
-c string from deserializer
+a int
+b string
+c string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
index 94f01bf..27e2585 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
@@ -126,8 +126,8 @@ POSTHOOK: query: describe formatted tbl_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_orc
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Partition Transform Information
# col_name transform_type
@@ -324,8 +324,8 @@ POSTHOOK: query: describe formatted tbl_parquet
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Partition Transform Information
# col_name transform_type
@@ -522,8 +522,8 @@ POSTHOOK: query: describe formatted tbl_avro
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_avro
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
index 97631ca..7748049 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
@@ -84,8 +84,8 @@ POSTHOOK: query: describe formatted tbl_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_orc
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Detailed Table Information
Database: default
@@ -234,8 +234,8 @@ POSTHOOK: query: describe formatted tbl_parquet
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_parquet
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Detailed Table Information
Database: default
@@ -384,8 +384,8 @@ POSTHOOK: query: describe formatted tbl_avro
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_avro
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
index 42a347a..3ef381f 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table.q.out
@@ -13,10 +13,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
index c7393a6..8dd4842 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_as_fileformat.q.out
@@ -17,10 +17,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_orc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_orc
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
@@ -80,10 +80,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_parquet
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_parquet
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
@@ -143,10 +143,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_avro
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_avro
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
@@ -206,10 +206,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
@@ -265,10 +265,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
index 42a347a..3ef381f 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg.q.out
@@ -13,10 +13,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
index d4a62cd..a386b88 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/create_iceberg_table_stored_by_iceberg_with_serdeproperties.q.out
@@ -13,10 +13,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_metadata_tables.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_metadata_tables.q.out
index 30fd9cf..c881c7b 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_metadata_tables.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_metadata_tables.q.out
@@ -32,24 +32,24 @@ PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-id int from deserializer
-value string from deserializer
+id int
+value string
PREHOOK: query: describe default.ice_meta_desc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-id int from deserializer
-value string from deserializer
+id int
+value string
PREHOOK: query: describe default.ice_meta_desc
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-id int from deserializer
-value string from deserializer
+id int
+value string
PREHOOK: query: describe default.ice_meta_desc.files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -77,68 +77,68 @@ PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.entries
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-status int from deserializer
-snapshot_id bigint from deserializer
-sequence_number bigint from deserializer
-data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int> from deserializer
+status int
+snapshot_id bigint
+sequence_number bigint
+data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int>
PREHOOK: query: describe default.ice_meta_desc.history
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.history
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-made_current_at timestamp with local time zone from deserializer
-snapshot_id bigint from deserializer
-parent_id bigint from deserializer
-is_current_ancestor boolean from deserializer
+made_current_at timestamp with local time zone
+snapshot_id bigint
+parent_id bigint
+is_current_ancestor boolean
PREHOOK: query: describe default.ice_meta_desc.manifests
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.manifests
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-path string from deserializer
-length bigint from deserializer
-partition_spec_id int from deserializer
-added_snapshot_id bigint from deserializer
-added_data_files_count int from deserializer
-existing_data_files_count int from deserializer
-deleted_data_files_count int from deserializer
-partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>> from deserializer
+path string
+length bigint
+partition_spec_id int
+added_snapshot_id bigint
+added_data_files_count int
+existing_data_files_count int
+deleted_data_files_count int
+partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>>
PREHOOK: query: describe default.ice_meta_desc.snapshots
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.snapshots
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-committed_at timestamp with local time zone from deserializer
-snapshot_id bigint from deserializer
-parent_id bigint from deserializer
-operation string from deserializer
-manifest_list string from deserializer
-summary map<string,string> from deserializer
+committed_at timestamp with local time zone
+snapshot_id bigint
+parent_id bigint
+operation string
+manifest_list string
+summary map<string,string>
PREHOOK: query: describe default.ice_meta_desc.partitions
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.partitions
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-record_count bigint from deserializer
-file_count int from deserializer
+record_count bigint
+file_count int
PREHOOK: query: describe default.ice_meta_desc.all_manifests
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.all_manifests
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-path string from deserializer
-length bigint from deserializer
-partition_spec_id int from deserializer
-added_snapshot_id bigint from deserializer
-added_data_files_count int from deserializer
-existing_data_files_count int from deserializer
-deleted_data_files_count int from deserializer
-partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>> from deserializer
+path string
+length bigint
+partition_spec_id int
+added_snapshot_id bigint
+added_data_files_count int
+existing_data_files_count int
+deleted_data_files_count int
+partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>>
PREHOOK: query: describe default.ice_meta_desc.all_data_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -166,10 +166,10 @@ PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe default.ice_meta_desc.all_entries
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-status int from deserializer
-snapshot_id bigint from deserializer
-sequence_number bigint from deserializer
-data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int> from deserializer
+status int
+snapshot_id bigint
+sequence_number bigint
+data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int>
PREHOOK: query: describe formatted default.ice_meta_desc.files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -199,10 +199,10 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.entries
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-status int from deserializer
-snapshot_id bigint from deserializer
-sequence_number bigint from deserializer
-data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int> from deserializer
+status int
+snapshot_id bigint
+sequence_number bigint
+data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int>
PREHOOK: query: describe formatted default.ice_meta_desc.history
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -210,10 +210,10 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.history
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-made_current_at timestamp with local time zone from deserializer
-snapshot_id bigint from deserializer
-parent_id bigint from deserializer
-is_current_ancestor boolean from deserializer
+made_current_at timestamp with local time zone
+snapshot_id bigint
+parent_id bigint
+is_current_ancestor boolean
PREHOOK: query: describe formatted default.ice_meta_desc.manifests
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -221,14 +221,14 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.manifests
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-path string from deserializer
-length bigint from deserializer
-partition_spec_id int from deserializer
-added_snapshot_id bigint from deserializer
-added_data_files_count int from deserializer
-existing_data_files_count int from deserializer
-deleted_data_files_count int from deserializer
-partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>> from deserializer
+path string
+length bigint
+partition_spec_id int
+added_snapshot_id bigint
+added_data_files_count int
+existing_data_files_count int
+deleted_data_files_count int
+partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>>
PREHOOK: query: describe formatted default.ice_meta_desc.snapshots
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -236,12 +236,12 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.snapshots
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-committed_at timestamp with local time zone from deserializer
-snapshot_id bigint from deserializer
-parent_id bigint from deserializer
-operation string from deserializer
-manifest_list string from deserializer
-summary map<string,string> from deserializer
+committed_at timestamp with local time zone
+snapshot_id bigint
+parent_id bigint
+operation string
+manifest_list string
+summary map<string,string>
PREHOOK: query: describe formatted default.ice_meta_desc.partitions
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -249,8 +249,8 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.partitions
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-record_count bigint from deserializer
-file_count int from deserializer
+record_count bigint
+file_count int
PREHOOK: query: describe formatted default.ice_meta_desc.all_manifests
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -258,14 +258,14 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.all_manifests
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-path string from deserializer
-length bigint from deserializer
-partition_spec_id int from deserializer
-added_snapshot_id bigint from deserializer
-added_data_files_count int from deserializer
-existing_data_files_count int from deserializer
-deleted_data_files_count int from deserializer
-partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>> from deserializer
+path string
+length bigint
+partition_spec_id int
+added_snapshot_id bigint
+added_data_files_count int
+existing_data_files_count int
+deleted_data_files_count int
+partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>>
PREHOOK: query: describe formatted default.ice_meta_desc.all_data_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -295,10 +295,10 @@ POSTHOOK: query: describe formatted default.ice_meta_desc.all_entries
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
# col_name data_type comment
-status int from deserializer
-snapshot_id bigint from deserializer
-sequence_number bigint from deserializer
-data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int> from deserializer
+status int
+snapshot_id bigint
+sequence_number bigint
+data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int>
PREHOOK: query: describe extended default.ice_meta_desc.files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -326,68 +326,68 @@ PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.entries
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-status int from deserializer
-snapshot_id bigint from deserializer
-sequence_number bigint from deserializer
-data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int> from deserializer
+status int
+snapshot_id bigint
+sequence_number bigint
+data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int>
PREHOOK: query: describe extended default.ice_meta_desc.history
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.history
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-made_current_at timestamp with local time zone from deserializer
-snapshot_id bigint from deserializer
-parent_id bigint from deserializer
-is_current_ancestor boolean from deserializer
+made_current_at timestamp with local time zone
+snapshot_id bigint
+parent_id bigint
+is_current_ancestor boolean
PREHOOK: query: describe extended default.ice_meta_desc.manifests
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.manifests
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-path string from deserializer
-length bigint from deserializer
-partition_spec_id int from deserializer
-added_snapshot_id bigint from deserializer
-added_data_files_count int from deserializer
-existing_data_files_count int from deserializer
-deleted_data_files_count int from deserializer
-partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>> from deserializer
+path string
+length bigint
+partition_spec_id int
+added_snapshot_id bigint
+added_data_files_count int
+existing_data_files_count int
+deleted_data_files_count int
+partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>>
PREHOOK: query: describe extended default.ice_meta_desc.snapshots
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.snapshots
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-committed_at timestamp with local time zone from deserializer
-snapshot_id bigint from deserializer
-parent_id bigint from deserializer
-operation string from deserializer
-manifest_list string from deserializer
-summary map<string,string> from deserializer
+committed_at timestamp with local time zone
+snapshot_id bigint
+parent_id bigint
+operation string
+manifest_list string
+summary map<string,string>
PREHOOK: query: describe extended default.ice_meta_desc.partitions
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.partitions
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-record_count bigint from deserializer
-file_count int from deserializer
+record_count bigint
+file_count int
PREHOOK: query: describe extended default.ice_meta_desc.all_manifests
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.all_manifests
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-path string from deserializer
-length bigint from deserializer
-partition_spec_id int from deserializer
-added_snapshot_id bigint from deserializer
-added_data_files_count int from deserializer
-existing_data_files_count int from deserializer
-deleted_data_files_count int from deserializer
-partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>> from deserializer
+path string
+length bigint
+partition_spec_id int
+added_snapshot_id bigint
+added_data_files_count int
+existing_data_files_count int
+deleted_data_files_count int
+partition_summaries array<struct<contains_null:boolean,contains_nan:boolean,lower_bound:string,upper_bound:string>>
PREHOOK: query: describe extended default.ice_meta_desc.all_data_files
PREHOOK: type: DESCTABLE
PREHOOK: Input: default@ice_meta_desc
@@ -415,10 +415,10 @@ PREHOOK: Input: default@ice_meta_desc
POSTHOOK: query: describe extended default.ice_meta_desc.all_entries
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_meta_desc
-status int from deserializer
-snapshot_id bigint from deserializer
-sequence_number bigint from deserializer
-data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int> from deserializer
+status int
+snapshot_id bigint
+sequence_number bigint
+data_file struct<content:int,file_path:string,file_format:string,spec_id:int,record_count:bigint,file_size_in_bytes:bigint,column_sizes:map<int,bigint>,value_counts:map<int,bigint>,null_value_counts:map<int,bigint>,nan_value_counts:map<int,bigint>,lower_bounds:map<int,binary>,upper_bounds:map<int,binary>,key_metadata:binary,split_offsets:array<bigint>,equality_ids:array<int>,sort_order_id:int>
PREHOOK: query: drop table ice_meta_desc
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@ice_meta_desc
diff --git a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
index 96f72d1..3567116 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/describe_iceberg_table.q.out
@@ -53,10 +53,10 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t
# col_name data_type comment
-i int from deserializer
-s string from deserializer
-ts timestamp from deserializer
-d date from deserializer
+i int
+s string
+ts timestamp
+d date
# Detailed Table Information
Database: default
@@ -95,13 +95,13 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t_transform
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t_transform
# col_name data_type comment
-year_field date from deserializer
-month_field date from deserializer
-day_field date from deserializer
-hour_field timestamp from deserializer
-truncate_field string from deserializer
-bucket_field int from deserializer
-identity_field int from deserializer
+year_field date
+month_field date
+day_field date
+hour_field timestamp
+truncate_field string
+bucket_field int
+identity_field int
# Partition Transform Information
# col_name transform_type
@@ -150,14 +150,14 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t_transform_prop
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t_transform_prop
# col_name data_type comment
-id int from deserializer
-year_field date from deserializer
-month_field date from deserializer
-day_field date from deserializer
-hour_field timestamp from deserializer
-truncate_field string from deserializer
-bucket_field int from deserializer
-identity_field int from deserializer
+id int
+year_field date
+month_field date
+day_field date
+hour_field timestamp
+truncate_field string
+bucket_field int
+identity_field int
# Partition Transform Information
# col_name transform_type
@@ -206,8 +206,8 @@ POSTHOOK: query: DESCRIBE FORMATTED ice_t_identity_part
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@ice_t_identity_part
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Partition Transform Information
# col_name transform_type
@@ -260,7 +260,7 @@ max_col_len
num_trues
num_falses
bit_vector
-comment from deserializer
+comment
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"d\":\"true\",\"i\":\"true\",\"s\":\"true\",\"ts\":\"true\"}}
PREHOOK: query: DESCRIBE FORMATTED ice_t_transform year_field
PREHOOK: type: DESCTABLE
@@ -279,7 +279,7 @@ max_col_len
num_trues
num_falses
bit_vector
-comment from deserializer
+comment
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bucket_field\":\"true\",\"day_field\":\"true\",\"hour_field\":\"true\",\"identity_field\":\"true\",\"month_field\":\"true\",\"truncate_field\":\"true\",\"year_field\":\"true\"}}
PREHOOK: query: DESCRIBE FORMATTED ice_t_transform_prop id
PREHOOK: type: DESCTABLE
@@ -298,7 +298,7 @@ max_col_len
num_trues
num_falses
bit_vector
-comment from deserializer
+comment
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"bucket_field\":\"true\",\"day_field\":\"true\",\"hour_field\":\"true\",\"id\":\"true\",\"identity_field\":\"true\",\"month_field\":\"true\",\"truncate_field\":\"true\",\"year_field\":\"true\"}}
PREHOOK: query: DESCRIBE FORMATTED ice_t_identity_part a
PREHOOK: type: DESCTABLE
@@ -317,5 +317,5 @@ max_col_len
num_trues
num_falses
bit_vector
-comment from deserializer
+comment
COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"a\":\"true\",\"b\":\"true\"}}
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out
index 585d8a0..625c180 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/llap_iceberg_read_orc.q.out
@@ -171,11 +171,11 @@ Crossover NULL
SUV 2.5
Sedan NULL
Sports NULL
-PREHOOK: query: ALTER TABLE llap_items REPLACE COLUMNS (itemid int, name string, cost int COMMENT 'from deserializer', description string, to60 float)
+PREHOOK: query: ALTER TABLE llap_items REPLACE COLUMNS (itemid int, name string, cost int, description string, to60 float)
PREHOOK: type: ALTERTABLE_REPLACECOLS
PREHOOK: Input: default@llap_items
PREHOOK: Output: default@llap_items
-POSTHOOK: query: ALTER TABLE llap_items REPLACE COLUMNS (itemid int, name string, cost int COMMENT 'from deserializer', description string, to60 float)
+POSTHOOK: query: ALTER TABLE llap_items REPLACE COLUMNS (itemid int, name string, cost int, description string, to60 float)
POSTHOOK: type: ALTERTABLE_REPLACECOLS
POSTHOOK: Input: default@llap_items
POSTHOOK: Output: default@llap_items
@@ -353,11 +353,11 @@ FR NULL 2.75
HU NULL 7.0
IT Venezia 6.0
UK London 2.0
-PREHOOK: query: ALTER TABLE llap_orders REPLACE COLUMNS (quantity int, itemid int, region string COMMENT 'from deserializer', state string COMMENT 'from deserializer', ordertime timestamp COMMENT 'from deserializer', city string)
+PREHOOK: query: ALTER TABLE llap_orders REPLACE COLUMNS (quantity int, itemid int, region string, state string, ordertime timestamp, city string)
PREHOOK: type: ALTERTABLE_REPLACECOLS
PREHOOK: Input: default@llap_orders
PREHOOK: Output: default@llap_orders
-POSTHOOK: query: ALTER TABLE llap_orders REPLACE COLUMNS (quantity int, itemid int, region string COMMENT 'from deserializer', state string COMMENT 'from deserializer', ordertime timestamp COMMENT 'from deserializer', city string)
+POSTHOOK: query: ALTER TABLE llap_orders REPLACE COLUMNS (quantity int, itemid int, region string, state string, ordertime timestamp, city string)
POSTHOOK: type: ALTERTABLE_REPLACECOLS
POSTHOOK: Input: default@llap_orders
POSTHOOK: Output: default@llap_orders
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
index e1e0eb9..b3d9d7f 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_mixed.q.out
@@ -366,10 +366,10 @@ PREHOOK: Input: default@tbl_ice_mixed_parted
POSTHOOK: query: describe tbl_ice_mixed_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_mixed_parted
-p1 string from deserializer
-b string from deserializer
-a int from deserializer
-p2 string from deserializer
+p1 string
+b string
+a int
+p2 string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
index c5935a3..f1207fb 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_orc.q.out
@@ -300,10 +300,10 @@ PREHOOK: Input: default@tbl_ice_orc_parted
POSTHOOK: query: describe tbl_ice_orc_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_orc_parted
-p1 string from deserializer
-b string from deserializer
-a int from deserializer
-p2 string from deserializer
+p1 string
+b string
+a int
+p2 string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
index df57457..e1d6dfa 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/llap/vectorized_iceberg_read_parquet.q.out
@@ -300,10 +300,10 @@ PREHOOK: Input: default@tbl_ice_parquet_parted
POSTHOOK: query: describe tbl_ice_parquet_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_parquet_parted
-p1 string from deserializer
-b string from deserializer
-a int from deserializer
-p2 string from deserializer
+p1 string
+b string
+a int
+p2 string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
index 0684512..a3640d7 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
@@ -17,10 +17,10 @@ POSTHOOK: query: SHOW CREATE TABLE ice_t
POSTHOOK: type: SHOW_CREATETABLE
POSTHOOK: Input: default@ice_t
CREATE EXTERNAL TABLE `ice_t`(
- `i` int COMMENT 'from deserializer',
- `s` string COMMENT 'from deserializer',
- `ts` timestamp COMMENT 'from deserializer',
- `d` date COMMENT 'from deserializer')
+ `i` int,
+ `s` string,
+ `ts` timestamp,
+ `d` date)
ROW FORMAT SERDE
'org.apache.iceberg.mr.hive.HiveIcebergSerDe'
STORED BY
@@ -55,13 +55,13 @@ POSTHOOK: query: SHOW CREATE TABLE ice_t_transform
POSTHOOK: type: SHOW_CREATETABLE
POSTHOOK: Input: default@ice_t_transform
CREATE EXTERNAL TABLE `ice_t_transform`(
- `year_field` date COMMENT 'from deserializer',
- `month_field` date COMMENT 'from deserializer',
- `day_field` date COMMENT 'from deserializer',
- `hour_field` timestamp COMMENT 'from deserializer',
- `truncate_field` string COMMENT 'from deserializer',
- `bucket_field` int COMMENT 'from deserializer',
- `identity_field` int COMMENT 'from deserializer')
+ `year_field` date,
+ `month_field` date,
+ `day_field` date,
+ `hour_field` timestamp,
+ `truncate_field` string,
+ `bucket_field` int,
+ `identity_field` int)
PARTITIONED BY SPEC (
YEAR(year_field),
MONTH(month_field),
@@ -104,14 +104,14 @@ POSTHOOK: query: SHOW CREATE TABLE ice_t_transform_prop
POSTHOOK: type: SHOW_CREATETABLE
POSTHOOK: Input: default@ice_t_transform_prop
CREATE EXTERNAL TABLE `ice_t_transform_prop`(
- `id` int COMMENT 'from deserializer',
- `year_field` date COMMENT 'from deserializer',
- `month_field` date COMMENT 'from deserializer',
- `day_field` date COMMENT 'from deserializer',
- `hour_field` timestamp COMMENT 'from deserializer',
- `truncate_field` string COMMENT 'from deserializer',
- `bucket_field` int COMMENT 'from deserializer',
- `identity_field` int COMMENT 'from deserializer')
+ `id` int,
+ `year_field` date,
+ `month_field` date,
+ `day_field` date,
+ `hour_field` timestamp,
+ `truncate_field` string,
+ `bucket_field` int,
+ `identity_field` int)
PARTITIONED BY SPEC (
YEAR(year_field),
MONTH(month_field),
@@ -154,8 +154,8 @@ POSTHOOK: query: SHOW CREATE TABLE ice_t_identity_part
POSTHOOK: type: SHOW_CREATETABLE
POSTHOOK: Input: default@ice_t_identity_part
CREATE EXTERNAL TABLE `ice_t_identity_part`(
- `a` int COMMENT 'from deserializer',
- `b` string COMMENT 'from deserializer')
+ `a` int,
+ `b` string)
PARTITIONED BY SPEC (
b)
ROW FORMAT SERDE
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
index 1fd52a5..b51e1b8 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
@@ -75,8 +75,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-id int from deserializer
-value string from deserializer
+id int
+value string
# Detailed Table Information
Database: default
@@ -141,8 +141,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-id int from deserializer
-value string from deserializer
+id int
+value string
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
index af44a6d..42925ee 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
@@ -75,8 +75,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-id int from deserializer
-value string from deserializer
+id int
+value string
# Detailed Table Information
Database: default
@@ -141,8 +141,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-id int from deserializer
-value string from deserializer
+id int
+value string
# Detailed Table Information
Database: default
@@ -205,8 +205,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-id int from deserializer
-value string from deserializer
+id int
+value string
# Detailed Table Information
Database: default
@@ -271,8 +271,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-id int from deserializer
-value string from deserializer
+id int
+value string
# Detailed Table Information
Database: default
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
index ec48682..2c8dbab 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
@@ -77,8 +77,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Partition Transform Information
# col_name transform_type
@@ -170,8 +170,8 @@ POSTHOOK: query: describe formatted test_truncate
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@test_truncate
# col_name data_type comment
-a int from deserializer
-b string from deserializer
+a int
+b string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
index 78a0544..e6d990c 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_mixed.q.out
@@ -294,10 +294,10 @@ PREHOOK: Input: default@tbl_ice_mixed_parted
POSTHOOK: query: describe tbl_ice_mixed_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_mixed_parted
-p1 string from deserializer
-b string from deserializer
-a int from deserializer
-p2 string from deserializer
+p1 string
+b string
+a int
+p2 string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
index cfdef05..25764be 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_orc.q.out
@@ -228,10 +228,10 @@ PREHOOK: Input: default@tbl_ice_orc_parted
POSTHOOK: query: describe tbl_ice_orc_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_orc_parted
-p1 string from deserializer
-b string from deserializer
-a int from deserializer
-p2 string from deserializer
+p1 string
+b string
+a int
+p2 string
# Partition Transform Information
# col_name transform_type
diff --git a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
index 38ab7f4..201a5ef 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/vectorized_iceberg_read_parquet.q.out
@@ -228,10 +228,10 @@ PREHOOK: Input: default@tbl_ice_parquet_parted
POSTHOOK: query: describe tbl_ice_parquet_parted
POSTHOOK: type: DESCTABLE
POSTHOOK: Input: default@tbl_ice_parquet_parted
-p1 string from deserializer
-b string from deserializer
-a int from deserializer
-p2 string from deserializer
+p1 string
+b string
+a int
+p2 string
# Partition Transform Information
# col_name transform_type
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
index b31e5b8..ae97122 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/serde2/TestSerdeWithFieldComments.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.serde2;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -67,7 +68,7 @@ public class TestSerdeWithFieldComments {
Deserializer mockDe = mock(Deserializer.class);
when(mockDe.getObjectInspector()).thenReturn(mockSOI);
List<FieldSchema> result =
- HiveMetaStoreUtils.getFieldsFromDeserializer("testTable", mockDe);
+ HiveMetaStoreUtils.getFieldsFromDeserializer("testTable", mockDe, new HiveConf());
assertEquals(2, result.size());
assertEquals("first", result.get(0).getName());
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java
index d30ca37..d1875cf 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreUtils.java
@@ -20,12 +20,14 @@ package org.apache.hadoop.hive.metastore;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Properties;
import javax.security.auth.login.LoginException;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -148,12 +150,17 @@ public class HiveMetaStoreUtils {
/**
* @param tableName name of the table
* @param deserializer deserializer to use
+ * @param conf the configuration - used to determine if the deserializer needs to generate the 'from deserializer'
+ * comments
* @return the list of fields
* @throws SerDeException if the serde throws an exception
* @throws MetaException if one of the fields or types in the table is invalid
*/
public static List<FieldSchema> getFieldsFromDeserializer(String tableName,
- Deserializer deserializer) throws SerDeException, MetaException {
+ Deserializer deserializer, Configuration conf) throws SerDeException, MetaException {
+ Collection<String> noCommentSerdes =
+ MetastoreConf.getStringCollection(conf, MetastoreConf.ConfVars.SERDES_WITHOUT_FROM_DESERIALIZER);
+ boolean noCommentFromDeserializer = noCommentSerdes.contains(deserializer.getClass().getName());
ObjectInspector oi = deserializer.getObjectInspector();
String[] names = tableName.split("\\.");
String last_name = names[names.length - 1];
@@ -188,7 +195,7 @@ public class HiveMetaStoreUtils {
// rules on how to recurse the ObjectInspector based on its type
if (oi.getCategory() != Category.STRUCT) {
str_fields.add(new FieldSchema(last_name, oi.getTypeName(),
- FROM_SERIALIZER));
+ determineFieldComment(null, noCommentFromDeserializer)));
} else {
List<? extends StructField> fields = ((StructObjectInspector) oi)
.getAllStructFieldRefs();
@@ -196,7 +203,7 @@ public class HiveMetaStoreUtils {
StructField structField = fields.get(i);
String fieldName = structField.getFieldName();
String fieldTypeName = structField.getFieldObjectInspector().getTypeName();
- String fieldComment = determineFieldComment(structField.getFieldComment());
+ String fieldComment = determineFieldComment(structField.getFieldComment(), noCommentFromDeserializer);
str_fields.add(new FieldSchema(fieldName, fieldTypeName, fieldComment));
}
@@ -205,8 +212,8 @@ public class HiveMetaStoreUtils {
}
private static final String FROM_SERIALIZER = "from deserializer";
- private static String determineFieldComment(String comment) {
- return (comment == null) ? FROM_SERIALIZER : comment;
+ private static String determineFieldComment(String comment, boolean noCommentFromDeserializer) {
+ return (comment != null || noCommentFromDeserializer) ? comment : FROM_SERIALIZER;
}
/**
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java b/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java
index 6a8902f..958f21b 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/SerDeStorageSchemaReader.java
@@ -45,7 +45,7 @@ public class SerDeStorageSchemaReader implements StorageSchemaReader {
}
Deserializer s = HiveMetaStoreUtils.getDeserializer(conf, tbl, null, false);
- return HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), s);
+ return HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), s, conf);
} catch (Exception e) {
StringUtils.stringifyException(e);
throw new MetaException(e.getMessage());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java b/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
index c556139..0a0f7df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Compiler.java
@@ -400,7 +400,8 @@ public class Compiler {
String tableName = "result";
List<FieldSchema> lst = null;
try {
- lst = HiveMetaStoreUtils.getFieldsFromDeserializer(tableName, td.getDeserializer(driverContext.getConf()));
+ lst = HiveMetaStoreUtils.getFieldsFromDeserializer(tableName, td.getDeserializer(driverContext.getConf()),
+ driverContext.getConf());
} catch (Exception e) {
LOG.warn("Error getting schema", e);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/update/AlterTableUpdateColumnsOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/update/AlterTableUpdateColumnsOperation.java
index c84d6d7..c8885d0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/update/AlterTableUpdateColumnsOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/column/update/AlterTableUpdateColumnsOperation.java
@@ -57,7 +57,8 @@ public class AlterTableUpdateColumnsOperation extends AbstractAlterTableOperatio
Deserializer deserializer = table.getDeserializer(true);
try {
LOG.info("Updating metastore columns for table: {}", table.getTableName());
- List<FieldSchema> fields = HiveMetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), deserializer);
+ List<FieldSchema> fields = HiveMetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), deserializer,
+ context.getConf());
StorageDescriptor sd = getStorageDescriptor(table, partition);
sd.setCols(fields);
} catch (org.apache.hadoop.hive.serde2.SerDeException | MetaException e) {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
index 17d8613..d21673c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
@@ -93,7 +93,7 @@ public class DescTableOperation extends DDLOperation<DescTableDesc> {
if (desc.isFormatted()) {
getColumnDataColPathSpecified(table, part, cols, colStats, deserializer);
} else {
- cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer));
+ cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer, context.getConf()));
}
}
fixDecimalColumnTypeName(cols);
@@ -208,7 +208,7 @@ public class DescTableOperation extends DDLOperation<DescTableDesc> {
}
table.setParameters(tableProps);
} else {
- cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer));
+ cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer, context.getConf()));
colStats.addAll(
context.getDb().getTableColumnStatistics(tableName.getDb().toLowerCase(),
tableName.getTable().toLowerCase(), colNames, false));
@@ -219,7 +219,7 @@ public class DescTableOperation extends DDLOperation<DescTableDesc> {
// lower case name to get the stats.
String partName = HMSHandler.lowerCaseConvertPartName(part.getName());
partitions.add(partName);
- cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer));
+ cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer, context.getConf()));
Map<String, List<ColumnStatisticsObj>> partitionColumnStatistics = context.getDb().getPartitionColumnStatistics(
tableName.getDb().toLowerCase(), tableName.getTable().toLowerCase(), partitions, colNames, false);
List<ColumnStatisticsObj> partitionColStat = partitionColumnStatistics.get(partName);
@@ -254,7 +254,7 @@ public class DescTableOperation extends DDLOperation<DescTableDesc> {
private void getColumnsForNotPartitionKeyColumn(List<FieldSchema> cols, List<ColumnStatisticsObj> colStats,
Deserializer deserializer, List<String> colNames, TableName tableName, Map<String, String> tableProps)
throws HiveException {
- cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer));
+ cols.addAll(Hive.getFieldsFromDeserializer(desc.getColumnPath(), deserializer, context.getConf()));
List<String> parts = context.getDb().getPartitionNames(tableName.getDb().toLowerCase(),
tableName.getTable().toLowerCase(), (short) -1);
AggrStats aggrStats = context.getDb().getAggrColStatsFor(
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/serde/AlterTableSetSerdeOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/serde/AlterTableSetSerdeOperation.java
index 48be90b..0821fb3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/serde/AlterTableSetSerdeOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/serde/AlterTableSetSerdeOperation.java
@@ -69,7 +69,7 @@ public class AlterTableSetSerdeOperation extends AbstractAlterTableOperation<Alt
try {
Deserializer oldSerde = HiveMetaStoreUtils.getDeserializer(context.getConf(), table.getTTable(), null,
false, oldSerdeName);
- table.setFields(Hive.getFieldsFromDeserializer(table.getTableName(), oldSerde));
+ table.setFields(Hive.getFieldsFromDeserializer(table.getTableName(), oldSerde, context.getConf()));
} catch (MetaException ex) {
throw new HiveException(ex);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 773d2b7..f211ddc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1258,8 +1258,7 @@ public class Hive {
tbl.setDbName(SessionState.get().getCurrentDatabase());
}
if (tbl.getCols().size() == 0 || tbl.getSd().getColsSize() == 0) {
- tbl.setFields(HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(),
- tbl.getDeserializer()));
+ tbl.setFields(HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), tbl.getDeserializer(), conf));
}
tbl.checkValidity(conf);
if (tbl.getParameters() != null) {
@@ -1320,9 +1319,8 @@ public class Hive {
}
public static List<FieldSchema> getFieldsFromDeserializerForMsStorage(
- Table tbl, Deserializer deserializer) throws SerDeException, MetaException {
- List<FieldSchema> schema = HiveMetaStoreUtils.getFieldsFromDeserializer(
- tbl.getTableName(), deserializer);
+ Table tbl, Deserializer deserializer, Configuration conf) throws SerDeException, MetaException {
+ List<FieldSchema> schema = HiveMetaStoreUtils.getFieldsFromDeserializer(tbl.getTableName(), deserializer, conf);
for (FieldSchema field : schema) {
field.setType(MetaStoreUtils.TYPE_FROM_DESERIALIZER);
}
@@ -5618,16 +5616,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
return null;
}
- public static List<FieldSchema> getFieldsFromDeserializer(String name,
- Deserializer serde) throws HiveException {
+ public static List<FieldSchema> getFieldsFromDeserializer(String name, Deserializer serde, Configuration conf)
+ throws HiveException {
try {
- return HiveMetaStoreUtils.getFieldsFromDeserializer(name, serde);
+ return HiveMetaStoreUtils.getFieldsFromDeserializer(name, serde, conf);
} catch (SerDeException e) {
- throw new HiveException("Error in getting fields from serde. "
- + e.getMessage(), e);
+ throw new HiveException("Error in getting fields from serde. " + e.getMessage(), e);
} catch (MetaException e) {
- throw new HiveException("Error in getting fields from serde."
- + e.getMessage(), e);
+ throw new HiveException("Error in getting fields from serde." + e.getMessage(), e);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
index 4e4e511..b7940f8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java
@@ -507,9 +507,10 @@ public class Partition implements Serializable {
return tPartition.getSd().getCols();
} else if (forMs && !Table.shouldStoreFieldsInMetastore(
SessionState.getSessionConf(), serializationLib, table.getParameters())) {
- return Hive.getFieldsFromDeserializerForMsStorage(table, getDeserializer());
+ return Hive.getFieldsFromDeserializerForMsStorage(table, getDeserializer(), SessionState.getSessionConf());
}
- return HiveMetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), getDeserializer());
+ return HiveMetaStoreUtils.getFieldsFromDeserializer(table.getTableName(), getDeserializer(),
+ SessionState.getSessionConf());
} catch (Exception e) {
LOG.error("Unable to get cols from serde: " +
tPartition.getSd().getSerdeInfo().getSerializationLib(), e);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index 00ba5a2..493cf62 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -737,9 +737,10 @@ public class Table implements Serializable {
return tTable.getSd().getCols();
} else if (forMs && !shouldStoreFieldsInMetastore(
SessionState.getSessionConf(), serializationLib, tTable.getParameters())) {
- return Hive.getFieldsFromDeserializerForMsStorage(this, getDeserializer());
+ return Hive.getFieldsFromDeserializerForMsStorage(this, getDeserializer(), SessionState.getSessionConf());
} else {
- return HiveMetaStoreUtils.getFieldsFromDeserializer(getTableName(), getDeserializer());
+ return HiveMetaStoreUtils.getFieldsFromDeserializer(getTableName(), getDeserializer(),
+ SessionState.getSessionConf());
}
} catch (Exception e) {
LOG.error("Unable to get field from serde: " + serializationLib, e);
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index ca6e83d..e46a6e9 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1268,6 +1268,10 @@ public class MetastoreConf {
"org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe," +
"org.apache.hadoop.hive.serde2.OpenCSVSerde",
"SerDes retrieving schema from metastore. This is an internal parameter."),
+ SERDES_WITHOUT_FROM_DESERIALIZER("metastore.serdes.without.from.deserializer",
+ "hive.metastore.serdes.without.from.deserializer",
+ "org.apache.iceberg.mr.hive.HiveIcebergSerDe",
+ "SerDes which are providing the schema but do not need the 'from deserializer' comment for the columns."),
SERVER_MAX_MESSAGE_SIZE("metastore.server.max.message.size",
"hive.metastore.server.max.message.size", 100*1024*1024L,
"Maximum message size in bytes a HMS will accept."),