You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/09/23 08:02:03 UTC
[1/2] hive git commit: HIVE-11319: CTAS with location qualifier
overwrites directories (Yongzhi Chen reviewed by Szehon Ho)
Repository: hive
Updated Branches:
refs/heads/branch-1 4bcf1d7e8 -> 83950c7fb
HIVE-11319: CTAS with location qualifier overwrites directories (Yongzhi Chen reviewed by Szehon Ho)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/88ef564a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/88ef564a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/88ef564a
Branch: refs/heads/branch-1
Commit: 88ef564ae74040effbaeac6cd020cfa029df5a24
Parents: 4bcf1d7
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Sep 23 00:57:32 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Wed Sep 23 00:57:32 2015 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 26 ++++++++++++++++++++
.../queries/clientnegative/ctas_noemptyfolder.q | 10 ++++++++
.../clientnegative/ctas_noemptyfolder.q.out | 19 ++++++++++++++
4 files changed, 56 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/88ef564a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index c2c99c5..6ada2d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -431,6 +431,7 @@ public enum ErrorMsg {
DROP_NATIVE_FUNCTION(10301, "Cannot drop native function"),
UPDATE_CANNOT_UPDATE_BUCKET_VALUE(10302, "Updating values of bucketing columns is not supported. Column {0}.", true),
IMPORT_INTO_STRICT_REPL_TABLE(10303,"Non-repl import disallowed against table that is a destination of replication."),
+ CTAS_LOCATION_NONEMPTY(10304, "CREATE-TABLE-AS-SELECT cannot create table with location to a non-empty directory."),
//========================== 20000 range starts here ========================//
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
http://git-wip-us.apache.org/repos/asf/hive/blob/88ef564a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 9b09142..affedd4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.parse;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Serializable;
import java.security.AccessControlException;
@@ -10999,6 +11000,31 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException(e);
}
+ if(location != null && location.length() != 0) {
+ Path locPath = new Path(location);
+ FileSystem curFs = null;
+ FileStatus locStats = null;
+ try {
+ curFs = locPath.getFileSystem(conf);
+ if(curFs != null) {
+ locStats = curFs.getFileStatus(locPath);
+ }
+ if(locStats != null && locStats.isDir()) {
+ FileStatus[] lStats = curFs.listStatus(locPath);
+ if(lStats != null && lStats.length != 0) {
+ throw new SemanticException(ErrorMsg.CTAS_LOCATION_NONEMPTY.getMsg(location));
+ }
+ }
+ } catch (FileNotFoundException nfe) {
+ //we will create the folder if it does not exist.
+ } catch (IOException ioE) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exception when validate folder ",ioE);
+ }
+
+ }
+ }
+
tblProps = addDefaultProperties(tblProps);
tableDesc = new CreateTableDesc(qualifiedTabName[0], dbDotTab, isExt, isTemporary, cols,
http://git-wip-us.apache.org/repos/asf/hive/blob/88ef564a/ql/src/test/queries/clientnegative/ctas_noemptyfolder.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/ctas_noemptyfolder.q b/ql/src/test/queries/clientnegative/ctas_noemptyfolder.q
new file mode 100644
index 0000000..18c9086
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/ctas_noemptyfolder.q
@@ -0,0 +1,10 @@
+create table ctas1
+location 'file:${system:test.tmp.dir}/ctastmpfolder'
+as
+select * from src limit 3;
+
+create table ctas2
+location 'file:${system:test.tmp.dir}/ctastmpfolder'
+as
+select * from src limit 2;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/88ef564a/ql/src/test/results/clientnegative/ctas_noemptyfolder.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ctas_noemptyfolder.q.out b/ql/src/test/results/clientnegative/ctas_noemptyfolder.q.out
new file mode 100644
index 0000000..76d7b86
--- /dev/null
+++ b/ql/src/test/results/clientnegative/ctas_noemptyfolder.q.out
@@ -0,0 +1,19 @@
+PREHOOK: query: create table ctas1
+#### A masked pattern was here ####
+as
+select * from src limit 3
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ctas1
+POSTHOOK: query: create table ctas1
+#### A masked pattern was here ####
+as
+select * from src limit 3
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ctas1
+#### A masked pattern was here ####
[2/2] hive git commit: HIVE-11217: CTAS statements throws error,
when the table is stored as ORC File format and select clause has
NULL/VOID type column (Yongzhi Chen reviewed by Prasanth Jayachandran)
Posted by pr...@apache.org.
HIVE-11217: CTAS statements throws error, when the table is stored as ORC File format and select clause has NULL/VOID type column (Yongzhi Chen reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/83950c7f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/83950c7f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/83950c7f
Branch: refs/heads/branch-1
Commit: 83950c7fbb2963817ca9476ec39be565b453c69d
Parents: 88ef564
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Wed Sep 23 01:01:48 2015 -0500
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Wed Sep 23 01:01:48 2015 -0500
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +-
.../org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 8 +++++++-
ql/src/test/queries/clientnegative/ctasnullcol.q | 2 ++
ql/src/test/results/clientnegative/ctasnullcol.q.out | 5 +++++
4 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/83950c7f/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 6ada2d3..5e89000 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -432,7 +432,7 @@ public enum ErrorMsg {
UPDATE_CANNOT_UPDATE_BUCKET_VALUE(10302, "Updating values of bucketing columns is not supported. Column {0}.", true),
IMPORT_INTO_STRICT_REPL_TABLE(10303,"Non-repl import disallowed against table that is a destination of replication."),
CTAS_LOCATION_NONEMPTY(10304, "CREATE-TABLE-AS-SELECT cannot create table with location to a non-empty directory."),
-
+ CTAS_CREATES_VOID_TYPE(10305, "CREATE-TABLE-AS-SELECT creates a VOID type, please use CAST to specify the type, near field: "),
//========================== 20000 range starts here ========================//
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
http://git-wip-us.apache.org/repos/asf/hive/blob/83950c7f/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index affedd4..5864c35 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -6489,7 +6489,13 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
}
colName = fixCtasColumnName(colName);
col.setName(colName);
- col.setType(colInfo.getType().getTypeName());
+ String typeName = colInfo.getType().getTypeName();
+ // CTAS should NOT create a VOID type
+ if (typeName.equals(serdeConstants.VOID_TYPE_NAME)) {
+ throw new SemanticException(ErrorMsg.CTAS_CREATES_VOID_TYPE
+ .getMsg(colName));
+ }
+ col.setType(typeName);
field_schemas.add(col);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/83950c7f/ql/src/test/queries/clientnegative/ctasnullcol.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/ctasnullcol.q b/ql/src/test/queries/clientnegative/ctasnullcol.q
new file mode 100644
index 0000000..b03c172
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/ctasnullcol.q
@@ -0,0 +1,2 @@
+drop table if exists orc_table_with_null;
+CREATE TABLE orc_table_with_null STORED AS ORC AS SELECT key, null FROM src;
http://git-wip-us.apache.org/repos/asf/hive/blob/83950c7f/ql/src/test/results/clientnegative/ctasnullcol.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/ctasnullcol.q.out b/ql/src/test/results/clientnegative/ctasnullcol.q.out
new file mode 100644
index 0000000..6d36bb8
--- /dev/null
+++ b/ql/src/test/results/clientnegative/ctasnullcol.q.out
@@ -0,0 +1,5 @@
+PREHOOK: query: drop table if exists orc_table_with_null
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists orc_table_with_null
+POSTHOOK: type: DROPTABLE
+FAILED: SemanticException [Error 10305]: CREATE-TABLE-AS-SELECT creates a VOID type, please use CAST to specify the type, near field: c1