You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kg...@apache.org on 2019/05/27 14:00:41 UTC
[hive] 01/02: HIVE-21715: Adding a new partition specified by
location (which is empty) leads to Exceptions (Zoltan Haindrich reviewed by
Laszlo Bodor, Ashutosh Chauhan)
This is an automated email from the ASF dual-hosted git repository.
kgyrtkirk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
commit a3f70bf8bf78cf6d5fddb876ec78e6ee0329db4d
Author: Zoltan Haindrich <ki...@rxd.hu>
AuthorDate: Mon May 27 15:42:11 2019 +0200
HIVE-21715: Adding a new partition specified by location (which is empty) leads to Exceptions (Zoltan Haindrich reviewed by Laszlo Bodor, Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich <ki...@rxd.hu>
---
.../test/resources/testconfiguration.properties | 1 +
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 3 ++
.../hadoop/hive/ql/io/HiveFileFormatUtils.java | 8 ++-
.../queries/clientpositive/add_part_with_loc.q | 23 +++++++++
.../clientpositive/llap/add_part_with_loc.q.out | 59 ++++++++++++++++++++++
5 files changed, 92 insertions(+), 2 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 3d8bae8..ad00b45 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -379,6 +379,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
vectorized_timestamp_ints_casts.q
minillap.query.files=acid_bucket_pruning.q,\
+ add_part_with_loc.q,\
alter_table_location2.q,\
alter_table_location3.q,\
bucket5.q,\
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 98adebb..295fe7c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -2597,6 +2597,9 @@ public class AcidUtils {
private static void validateAcidFiles(FileStatus[] srcs, FileSystem fs) throws SemanticException {
try {
+ if (srcs == null) {
+ return;
+ }
for (FileStatus oneSrc : srcs) {
if (!AcidUtils.MetaDataFile.isRawFormatFile(oneSrc.getPath(), fs)) {
throw new SemanticException(ErrorMsg.LOAD_DATA_ACID_FILE, oneSrc.getPath().toString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
index bd15fa4..33a6101 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
@@ -177,13 +177,17 @@ public final class HiveFileFormatUtils {
}
/**
- * checks if files are in same format as the given input format.
+ * Checks if files are in same format as the given input format.
+ *
+ * Note: an empty set of files is considered compliant.
*/
@SuppressWarnings("unchecked")
public static boolean checkInputFormat(FileSystem fs, HiveConf conf,
Class<? extends InputFormat> inputFormatCls, List<FileStatus> files)
throws HiveException {
- if (files.isEmpty()) return false;
+ if (files.isEmpty()) {
+ return true;
+ }
Class<? extends InputFormatChecker> checkerCls = FileChecker.getInstance()
.getInputFormatCheckerClass(inputFormatCls);
if (checkerCls == null
diff --git a/ql/src/test/queries/clientpositive/add_part_with_loc.q b/ql/src/test/queries/clientpositive/add_part_with_loc.q
new file mode 100644
index 0000000..e674f0d
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/add_part_with_loc.q
@@ -0,0 +1,23 @@
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+set hive.exec.dynamic.partition.mode=nonstrict;
+set hive.exec.dynamic.partition=true;
+set hive.vectorized.execution.enabled=true;
+
+
+set hive.fileformat.check=true;
+
+create table supply (id int, part string, quantity int) partitioned by (day int)
+ stored as orc
+ location 'hdfs:///tmp/a1'
+ TBLPROPERTIES ('transactional'='true')
+;
+
+
+explain alter table supply add partition (day=20110102) location
+ 'hdfs:///tmp/a2';
+
+alter table supply add partition (day=20110103) location
+ 'hdfs:///tmp/a3';
+
+
diff --git a/ql/src/test/results/clientpositive/llap/add_part_with_loc.q.out b/ql/src/test/results/clientpositive/llap/add_part_with_loc.q.out
new file mode 100644
index 0000000..b307c02
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/add_part_with_loc.q.out
@@ -0,0 +1,59 @@
+PREHOOK: query: create table supply (id int, part string, quantity int) partitioned by (day int)
+ stored as orc
+ location 'hdfs://### HDFS PATH ###'
+ TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: database:default
+PREHOOK: Output: default@supply
+POSTHOOK: query: create table supply (id int, part string, quantity int) partitioned by (day int)
+ stored as orc
+ location 'hdfs://### HDFS PATH ###'
+ TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@supply
+PREHOOK: query: explain alter table supply add partition (day=20110102) location
+ 'hdfs://### HDFS PATH ###'
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: default@supply
+POSTHOOK: query: explain alter table supply add partition (day=20110102) location
+ 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: default@supply
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+ Stage-1 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-0
+ Add Partition
+#### A masked pattern was here ####
+ Spec: {day=20110102}
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ partition:
+ day 20110102
+ replace: false
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.supply
+
+PREHOOK: query: alter table supply add partition (day=20110103) location
+ 'hdfs://### HDFS PATH ###'
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Input: hdfs://### HDFS PATH ###
+PREHOOK: Output: default@supply
+POSTHOOK: query: alter table supply add partition (day=20110103) location
+ 'hdfs://### HDFS PATH ###'
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Input: hdfs://### HDFS PATH ###
+POSTHOOK: Output: default@supply
+POSTHOOK: Output: default@supply@day=20110103