You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/07/16 06:40:59 UTC
[12/50] [abbrv] hive git commit: HIVE-11198: Fix load data query file
format check for partitioned tables (Prasanth Jayachandran reviewed by
Sushanth Sowmyan)
HIVE-11198: Fix load data query file format check for partitioned tables (Prasanth Jayachandran reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a2dabcb8
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a2dabcb8
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a2dabcb8
Branch: refs/heads/beeline-cli
Commit: a2dabcb8c7bc52bd8def1402fa649420ef41e3bd
Parents: be89eac
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Thu Jul 9 11:20:29 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Thu Jul 9 11:20:58 2015 -0700
----------------------------------------------------------------------
.../hive/ql/parse/LoadSemanticAnalyzer.java | 12 +++-
.../clientnegative/load_orc_negative_part.q | 14 ++++
.../test/queries/clientpositive/load_orc_part.q | 15 +++++
.../clientnegative/load_orc_negative_part.q.out | 52 +++++++++++++++
.../results/clientpositive/load_orc_part.q.out | 70 ++++++++++++++++++++
5 files changed, 162 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
index 187dc20..944cee4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
@@ -335,7 +335,17 @@ public class LoadSemanticAnalyzer extends BaseSemanticAnalyzer {
}
private void ensureFileFormatsMatch(TableSpec ts, URI fromURI) throws SemanticException {
- Class<? extends InputFormat> destInputFormat = ts.tableHandle.getInputFormatClass();
+ final Class<? extends InputFormat> destInputFormat;
+ try {
+ if (ts.getPartSpec() == null || ts.getPartSpec().isEmpty()) {
+ destInputFormat = ts.tableHandle.getInputFormatClass();
+ } else {
+ destInputFormat = ts.partHandle.getInputFormatClass();
+ }
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
+
// Other file formats should do similar check to make sure file formats match
// when doing LOAD DATA .. INTO TABLE
if (OrcInputFormat.class.equals(destInputFormat)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/queries/clientnegative/load_orc_negative_part.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/load_orc_negative_part.q b/ql/src/test/queries/clientnegative/load_orc_negative_part.q
new file mode 100644
index 0000000..5de4917
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/load_orc_negative_part.q
@@ -0,0 +1,14 @@
+set hive.default.fileformat=ORC;
+create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp);
+create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string);
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_staging/;
+
+load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split_elim.orc' into table orc_test partition (ds='10');
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/;
+
+alter table orc_test add partition(ds='11');
+load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=11/;
http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/queries/clientpositive/load_orc_part.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/load_orc_part.q b/ql/src/test/queries/clientpositive/load_orc_part.q
new file mode 100644
index 0000000..0927ea4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/load_orc_part.q
@@ -0,0 +1,15 @@
+set hive.default.fileformat=ORC;
+create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp);
+create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string);
+
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging;
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_staging/;
+
+load data inpath '${hiveconf:hive.metastore.warehouse.dir}/orc_staging/orc_split_elim.orc' into table orc_test partition (ds='10');
+load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=10/;
+
+alter table orc_test add partition(ds='11');
+alter table orc_test partition(ds='11') set fileformat textfile;
+load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11');
+dfs -ls ${hiveconf:hive.metastore.warehouse.dir}/orc_test/ds=11/;
http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/results/clientnegative/load_orc_negative_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/load_orc_negative_part.q.out b/ql/src/test/results/clientnegative/load_orc_negative_part.q.out
new file mode 100644
index 0000000..32dd627
--- /dev/null
+++ b/ql/src/test/results/clientnegative/load_orc_negative_part.q.out
@@ -0,0 +1,52 @@
+PREHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_staging
+PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_test
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_staging
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=10
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=10
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=10
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: alter table orc_test add partition(ds='11')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: alter table orc_test add partition(ds='11')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=11
+FAILED: SemanticException [Error 30019]: The file that you are trying to load does not match the file format of the destination table. Destination table is stored as ORC but the file being loaded is not a valid ORC file.
http://git-wip-us.apache.org/repos/asf/hive/blob/a2dabcb8/ql/src/test/results/clientpositive/load_orc_part.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/load_orc_part.q.out b/ql/src/test/results/clientpositive/load_orc_part.q.out
new file mode 100644
index 0000000..34ca493
--- /dev/null
+++ b/ql/src/test/results/clientpositive/load_orc_part.q.out
@@ -0,0 +1,70 @@
+PREHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: create table orc_staging (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_staging
+PREHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: create table orc_test (userid bigint, string1 string, subtype double, decimal1 decimal, ts timestamp) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orc_test
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_staging
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_staging
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_staging
+Found 1 items
+#### A masked pattern was here ####
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test
+#### A masked pattern was here ####
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=10
+PREHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=10
+POSTHOOK: query: load data local inpath '../../data/files/orc_split_elim.orc' into table orc_test partition (ds='10')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=10
+Found 2 items
+#### A masked pattern was here ####
+PREHOOK: query: alter table orc_test add partition(ds='11')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+PREHOOK: Output: default@orc_test
+POSTHOOK: query: alter table orc_test add partition(ds='11')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@orc_test
+POSTHOOK: Output: default@orc_test@ds=11
+PREHOOK: query: alter table orc_test partition(ds='11') set fileformat textfile
+PREHOOK: type: ALTERPARTITION_FILEFORMAT
+PREHOOK: Input: default@orc_test
+PREHOOK: Output: default@orc_test@ds=11
+POSTHOOK: query: alter table orc_test partition(ds='11') set fileformat textfile
+POSTHOOK: type: ALTERPARTITION_FILEFORMAT
+POSTHOOK: Input: default@orc_test
+POSTHOOK: Input: default@orc_test@ds=11
+POSTHOOK: Output: default@orc_test@ds=11
+PREHOOK: query: load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11')
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@orc_test@ds=11
+POSTHOOK: query: load data local inpath '../../data/files/kv1.txt' into table orc_test partition(ds='11')
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@orc_test@ds=11
+Found 1 items
+#### A masked pattern was here ####