You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mb...@apache.org on 2021/10/12 15:30:59 UTC
[hive] branch master updated: HIVE-25607: Mask totalSize table
property in Iceberg q-tests (Marton Bod,
reviewed by Peter Vary and Adam Szita)
This is an automated email from the ASF dual-hosted git repository.
mbod pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0df73fb HIVE-25607: Mask totalSize table property in Iceberg q-tests (Marton Bod, reviewed by Peter Vary and Adam Szita)
0df73fb is described below
commit 0df73fb6f1ac79ec2246ab5e8512f775e4baf03c
Author: Marton Bod <ma...@gmail.com>
AuthorDate: Tue Oct 12 17:30:44 2021 +0200
HIVE-25607: Mask totalSize table property in Iceberg q-tests (Marton Bod, reviewed by Peter Vary and Adam Szita)
The totalSize table property can change whenever the file format version changes, therefore potentially causing the q tests to be flaky when issuing describe formatted commands. We should mask this and not test against the exact value.
---
.../src/test/queries/positive/alter_multi_part_table_to_iceberg.q | 3 +++
.../src/test/queries/positive/alter_part_table_to_iceberg.q | 3 +++
.../src/test/queries/positive/alter_table_to_iceberg.q | 3 +++
.../src/test/queries/positive/describe_iceberg_metadata_tables.q | 6 ------
.../src/test/queries/positive/truncate_force_iceberg_table.q | 2 ++
.../src/test/queries/positive/truncate_iceberg_table.q | 2 ++
.../src/test/queries/positive/truncate_partitioned_iceberg_table.q | 2 ++
.../test/results/positive/alter_multi_part_table_to_iceberg.q.out | 6 +++---
.../src/test/results/positive/alter_part_table_to_iceberg.q.out | 6 +++---
.../src/test/results/positive/alter_table_to_iceberg.q.out | 6 +++---
.../src/test/results/positive/truncate_force_iceberg_table.q.out | 2 +-
.../src/test/results/positive/truncate_iceberg_table.q.out | 4 ++--
.../test/results/positive/truncate_partitioned_iceberg_table.q.out | 2 +-
.../java/{ => org/apache/hadoop/hive/cli}/TestIcebergCliDriver.java | 1 +
.../apache/hadoop/hive/cli}/TestIcebergLlapLocalCliDriver.java | 1 +
.../apache/hadoop/hive/cli}/TestIcebergNegativeCliDriver.java | 1 +
16 files changed, 31 insertions(+), 19 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/alter_multi_part_table_to_iceberg.q b/iceberg/iceberg-handler/src/test/queries/positive/alter_multi_part_table_to_iceberg.q
index 56fedb8..99ba2f3 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/alter_multi_part_table_to_iceberg.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/alter_multi_part_table_to_iceberg.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes
+--! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
drop table if exists tbl_orc;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/alter_part_table_to_iceberg.q b/iceberg/iceberg-handler/src/test/queries/positive/alter_part_table_to_iceberg.q
index e108525..6105ef2 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/alter_part_table_to_iceberg.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/alter_part_table_to_iceberg.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes
+--! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
drop table if exists tbl_orc;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/alter_table_to_iceberg.q b/iceberg/iceberg-handler/src/test/queries/positive/alter_table_to_iceberg.q
index 8dcaaf7..58afecb 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/alter_table_to_iceberg.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/alter_table_to_iceberg.q
@@ -1,3 +1,6 @@
+-- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes
+--! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
+
set hive.vectorized.execution.enabled=false;
drop table if exists tbl_orc;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
index ec50e43..6fd7daa 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/describe_iceberg_metadata_tables.q
@@ -1,9 +1,3 @@
-set hive.vectorized.execution.enabled = false;
-set tez.mrreader.config.update.properties=hive.io.file.readcolumn.names,hive.io.file.readcolumn.ids;
-set hive.query.results.cache.enabled=false;
-set hive.fetch.task.conversion=none;
-set hive.cbo.enable=true;
-
drop table if exists ice_meta_desc;
create external table ice_meta_desc (id int, value string) stored by iceberg stored as orc;
insert into ice_meta_desc values (1, 'one'),(2,'two'),(3,'three'),(4,'four'),(5,'five');
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
index 6cf6004..77e394d 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/truncate_force_iceberg_table.q
@@ -1,4 +1,6 @@
-- SORT_QUERY_RESULTS
+-- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes
+--! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
set hive.vectorized.execution.enabled=false;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
index 485ee4a..277f1c1 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/truncate_iceberg_table.q
@@ -1,4 +1,6 @@
-- SORT_QUERY_RESULTS
+-- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes
+--! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
set hive.vectorized.execution.enabled=false;
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/truncate_partitioned_iceberg_table.q b/iceberg/iceberg-handler/src/test/queries/positive/truncate_partitioned_iceberg_table.q
index dc52fcc..4ac492c 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/truncate_partitioned_iceberg_table.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/truncate_partitioned_iceberg_table.q
@@ -1,4 +1,6 @@
-- SORT_QUERY_RESULTS
+-- Mask the totalSize value as it can change whenever the file format (e.g. ORC) version changes
+--! qt:replace:/(\s+totalSize\s+)[\S+](\s+)/$1#Masked#$2/
set hive.vectorized.execution.enabled=false;
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
index 4e73c70..d11b826 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_multi_part_table_to_iceberg.q.out
@@ -38,7 +38,7 @@ Table Parameters:
numPartitions 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -276,7 +276,7 @@ Table Parameters:
numPartitions 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -514,7 +514,7 @@ Table Parameters:
numPartitions 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
index 97021fa..5a6181b 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_part_table_to_iceberg.q.out
@@ -37,7 +37,7 @@ Table Parameters:
numPartitions 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -227,7 +227,7 @@ Table Parameters:
numPartitions 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -417,7 +417,7 @@ Table Parameters:
numPartitions 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
index 572df30..4b44d68 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/alter_table_to_iceberg.q.out
@@ -33,7 +33,7 @@ Table Parameters:
numFiles 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -175,7 +175,7 @@ Table Parameters:
numFiles 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
@@ -317,7 +317,7 @@ Table Parameters:
numFiles 0
numRows 0
rawDataSize 0
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
# Storage Information
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
index 50a1180..9ad42d5 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_force_iceberg_table.q.out
@@ -165,7 +165,7 @@ Table Parameters:
serialization.format 1
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default parquet
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
index 67f19ca..a170985 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_iceberg_table.q.out
@@ -165,7 +165,7 @@ Table Parameters:
serialization.format 1
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default orc
@@ -295,7 +295,7 @@ Table Parameters:
serialization.format 1
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default orc
diff --git a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
index f0e2cac..caac41e 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/truncate_partitioned_iceberg_table.q.out
@@ -188,7 +188,7 @@ Table Parameters:
previous_metadata_location hdfs://### HDFS PATH ###
storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
table_type ICEBERG
- totalSize 0
+ totalSize #Masked#
#### A masked pattern was here ####
write.format.default avro
diff --git a/itests/qtest-iceberg/src/test/java/TestIcebergCliDriver.java b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergCliDriver.java
similarity index 98%
rename from itests/qtest-iceberg/src/test/java/TestIcebergCliDriver.java
rename to itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergCliDriver.java
index 0899185..c32fa24 100644
--- a/itests/qtest-iceberg/src/test/java/TestIcebergCliDriver.java
+++ b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergCliDriver.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.hadoop.hive.cli;
import java.io.File;
import java.util.List;
diff --git a/itests/qtest-iceberg/src/test/java/TestIcebergLlapLocalCliDriver.java b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergLlapLocalCliDriver.java
similarity index 98%
rename from itests/qtest-iceberg/src/test/java/TestIcebergLlapLocalCliDriver.java
rename to itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergLlapLocalCliDriver.java
index 46c3f44..a0e65eb 100644
--- a/itests/qtest-iceberg/src/test/java/TestIcebergLlapLocalCliDriver.java
+++ b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergLlapLocalCliDriver.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.hadoop.hive.cli;
import java.io.File;
import java.util.List;
diff --git a/itests/qtest-iceberg/src/test/java/TestIcebergNegativeCliDriver.java b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergNegativeCliDriver.java
similarity index 98%
rename from itests/qtest-iceberg/src/test/java/TestIcebergNegativeCliDriver.java
rename to itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergNegativeCliDriver.java
index bd61e03..adbc0a2 100644
--- a/itests/qtest-iceberg/src/test/java/TestIcebergNegativeCliDriver.java
+++ b/itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/TestIcebergNegativeCliDriver.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.hadoop.hive.cli;
import java.io.File;
import java.util.List;