You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ay...@apache.org on 2023/08/07 20:27:37 UTC
[hive] branch master updated: HIVE-27562: Iceberg: Fetching virtual columns failing. (#4546). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 93ae19b9647 HIVE-27562: Iceberg: Fetching virtual columns failing. (#4546). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)
93ae19b9647 is described below
commit 93ae19b96476bb69fd18cf21d458e434cf3ef84e
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Tue Aug 8 01:57:31 2023 +0530
HIVE-27562: Iceberg: Fetching virtual columns failing. (#4546). (Ayush Saxena, reviewed by Denys Kuzmenko, zhangbutao)
---
.../iceberg/mr/mapreduce/IcebergInputFormat.java | 10 +++---
.../queries/positive/query_iceberg_virtualcol.q | 9 +++++
.../positive/query_iceberg_virtualcol.q.out | 41 ++++++++++++++++++++++
3 files changed, 55 insertions(+), 5 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
index e438879c1f7..7020d705cac 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/mapreduce/IcebergInputFormat.java
@@ -534,12 +534,12 @@ public class IcebergInputFormat<T> extends InputFormat<Void, T> {
}
String[] selectedColumns = InputFormatConfig.selectedColumns(conf);
- if (selectedColumns == null) {
- return table.schema();
- }
+ readSchema = table.schema();
- readSchema = caseSensitive ? table.schema().select(selectedColumns) :
- table.schema().caseInsensitiveSelect(selectedColumns);
+ if (selectedColumns != null) {
+ readSchema =
+ caseSensitive ? readSchema.select(selectedColumns) : readSchema.caseInsensitiveSelect(selectedColumns);
+ }
if (InputFormatConfig.fetchVirtualColumns(conf)) {
return IcebergAcidUtil.createFileReadSchemaWithVirtualColums(readSchema.columns(), table);
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_virtualcol.q b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_virtualcol.q
index bf27fa291b7..c706439ce76 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_virtualcol.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/query_iceberg_virtualcol.q
@@ -2,8 +2,17 @@ drop table if exists tbl_ice;
create external table tbl_ice(a int, b string) partitioned by (c int) stored by iceberg stored as orc tblproperties ('format-version'='2');
insert into tbl_ice values (1, 'one', 50), (2, 'two', 50), (3, 'three', 50), (4, 'four', 52), (5, 'five', 54), (111, 'one', 52), (333, 'two', 56);
+select tbl_ice.ROW__POSITION from tbl_ice;
+
select a, c, tbl_ice.PARTITION__SPEC__ID, tbl_ice.PARTITION__HASH, tbl_ice.ROW__POSITION from tbl_ice
order by tbl_ice.PARTITION__HASH, tbl_ice.ROW__POSITION desc;
select a, c, tbl_ice.PARTITION__SPEC__ID, tbl_ice.PARTITION__HASH, tbl_ice.ROW__POSITION from tbl_ice
sort by tbl_ice.PARTITION__HASH, tbl_ice.ROW__POSITION desc;
+
+
+-- create a table with more than 4 columns
+create table ice01 (a int, b int, c int, d int, e int) stored by iceberg tblproperties ('format-version'='2');
+insert into ice01 values (1,2,3,4,5), (6,7,8,9,10);
+
+select ice01.ROW__POSITION from ice01;
\ No newline at end of file
diff --git a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_virtualcol.q.out b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_virtualcol.q.out
index 2cef50bede6..7f50dc0eafb 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_virtualcol.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/query_iceberg_virtualcol.q.out
@@ -20,6 +20,21 @@ POSTHOOK: query: insert into tbl_ice values (1, 'one', 50), (2, 'two', 50), (3,
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@tbl_ice
+PREHOOK: query: select tbl_ice.ROW__POSITION from tbl_ice
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tbl_ice
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select tbl_ice.ROW__POSITION from tbl_ice
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tbl_ice
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+1
+2
+0
+1
+0
+0
PREHOOK: query: select a, c, tbl_ice.PARTITION__SPEC__ID, tbl_ice.PARTITION__HASH, tbl_ice.ROW__POSITION from tbl_ice
order by tbl_ice.PARTITION__HASH, tbl_ice.ROW__POSITION desc
PREHOOK: type: QUERY
@@ -54,3 +69,29 @@ POSTHOOK: Output: hdfs://### HDFS PATH ###
4 52 0 83 0
5 54 0 85 0
333 56 0 87 0
+PREHOOK: query: create table ice01 (a int, b int, c int, d int, e int) stored by iceberg tblproperties ('format-version'='2')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice01
+POSTHOOK: query: create table ice01 (a int, b int, c int, d int, e int) stored by iceberg tblproperties ('format-version'='2')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice01
+PREHOOK: query: insert into ice01 values (1,2,3,4,5), (6,7,8,9,10)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice01
+POSTHOOK: query: insert into ice01 values (1,2,3,4,5), (6,7,8,9,10)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice01
+PREHOOK: query: select ice01.ROW__POSITION from ice01
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice01
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select ice01.ROW__POSITION from ice01
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice01
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+0
+1