You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/10/01 23:15:48 UTC
[1/4] hive git commit: HIVE-11925 : Hive file format checking breaks
load from named pipes (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 24988f77f -> 947871a3b
HIVE-11925 : Hive file format checking breaks load from named pipes (Sergey Shelukhin, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/82bc0e1c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/82bc0e1c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/82bc0e1c
Branch: refs/heads/master
Commit: 82bc0e1c79ca656ec34a43efe4a8807f0f655e30
Parents: 24988f7
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 1 12:42:28 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 1 12:42:28 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/io/HiveFileFormatUtils.java | 95 ++++++++++++++------
.../hadoop/hive/ql/io/InputFormatChecker.java | 5 +-
.../hadoop/hive/ql/io/RCFileInputFormat.java | 3 +-
.../ql/io/SequenceFileInputFormatChecker.java | 3 +-
.../hive/ql/io/VectorizedRCFileInputFormat.java | 3 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 +-
.../ql/io/orc/VectorizedOrcInputFormat.java | 2 +-
.../hive/ql/exec/TestFileSinkOperator.java | 2 +-
.../hive/ql/txn/compactor/CompactorTest.java | 2 +-
9 files changed, 80 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
index 50ba740..06d3df7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveFileFormatUtils.java
@@ -19,8 +19,13 @@
package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
+import java.nio.file.FileSystemNotFoundException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -28,10 +33,13 @@ import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter;
@@ -69,6 +77,7 @@ import org.apache.hive.common.util.ReflectionUtil;
*
*/
public final class HiveFileFormatUtils {
+ private static final Log LOG = LogFactory.getLog(HiveFileFormatUtils.class);
static {
outputFormatSubstituteMap =
@@ -177,44 +186,51 @@ public final class HiveFileFormatUtils {
*/
@SuppressWarnings("unchecked")
public static boolean checkInputFormat(FileSystem fs, HiveConf conf,
- Class<? extends InputFormat> inputFormatCls, ArrayList<FileStatus> files)
+ Class<? extends InputFormat> inputFormatCls, List<FileStatus> files)
throws HiveException {
- if (files.size() > 0) {
- Class<? extends InputFormatChecker> checkerCls = getInputFormatChecker(inputFormatCls);
- if (checkerCls == null
- && inputFormatCls.isAssignableFrom(TextInputFormat.class)) {
- // we get a text input format here, we can not determine a file is text
- // according to its content, so we can do is to test if other file
- // format can accept it. If one other file format can accept this file,
- // we treat this file as text file, although it maybe not.
- return checkTextInputFormat(fs, conf, files);
- }
+ if (files.isEmpty()) return false;
+ Class<? extends InputFormatChecker> checkerCls = getInputFormatChecker(inputFormatCls);
+ if (checkerCls == null
+ && inputFormatCls.isAssignableFrom(TextInputFormat.class)) {
+ // we get a text input format here, we can not determine a file is text
+ // according to its content, so we can do is to test if other file
+ // format can accept it. If one other file format can accept this file,
+ // we treat this file as text file, although it maybe not.
+ return checkTextInputFormat(fs, conf, files);
+ }
- if (checkerCls != null) {
- InputFormatChecker checkerInstance = inputFormatCheckerInstanceCache
- .get(checkerCls);
- try {
- if (checkerInstance == null) {
- checkerInstance = checkerCls.newInstance();
- inputFormatCheckerInstanceCache.put(checkerCls, checkerInstance);
- }
- return checkerInstance.validateInput(fs, conf, files);
- } catch (Exception e) {
- throw new HiveException(e);
+ if (checkerCls != null) {
+ InputFormatChecker checkerInstance = inputFormatCheckerInstanceCache.get(checkerCls);
+ try {
+ if (checkerInstance == null) {
+ checkerInstance = checkerCls.newInstance();
+ inputFormatCheckerInstanceCache.put(checkerCls, checkerInstance);
}
+ return checkerInstance.validateInput(fs, conf, files);
+ } catch (Exception e) {
+ throw new HiveException(e);
}
- return true;
}
- return false;
+ return true;
}
@SuppressWarnings("unchecked")
private static boolean checkTextInputFormat(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files) throws HiveException {
- Set<Class<? extends InputFormat>> inputFormatter = inputFormatCheckerMap
- .keySet();
+ List<FileStatus> files) throws HiveException {
+ List<FileStatus> files2 = new LinkedList<>(files);
+ Iterator<FileStatus> iter = files2.iterator();
+ while (iter.hasNext()) {
+ FileStatus file = iter.next();
+ if (file == null) continue;
+ if (isPipe(fs, file)) {
+ LOG.info("Skipping format check for " + file.getPath() + " as it is a pipe");
+ iter.remove();
+ }
+ }
+ if (files2.isEmpty()) return true;
+ Set<Class<? extends InputFormat>> inputFormatter = inputFormatCheckerMap.keySet();
for (Class<? extends InputFormat> reg : inputFormatter) {
- boolean result = checkInputFormat(fs, conf, reg, files);
+ boolean result = checkInputFormat(fs, conf, reg, files2);
if (result) {
return false;
}
@@ -222,6 +238,29 @@ public final class HiveFileFormatUtils {
return true;
}
+ // See include/uapi/linux/stat.h
+ private static final int S_IFIFO = 0010000;
+ private static boolean isPipe(FileSystem fs, FileStatus file) {
+ if (fs instanceof DistributedFileSystem) {
+ return false; // Shortcut for HDFS.
+ }
+ int mode = 0;
+ Object pathToLog = file.getPath();
+ try {
+ java.nio.file.Path realPath = Paths.get(file.getPath().toUri());
+ pathToLog = realPath;
+ mode = (Integer)Files.getAttribute(realPath, "unix:mode");
+ } catch (FileSystemNotFoundException t) {
+ return false; // Probably not a local filesystem; no need to check.
+ } catch (UnsupportedOperationException | IOException
+ | SecurityException | IllegalArgumentException t) {
+ LOG.info("Failed to check mode for " + pathToLog + ": "
+ + t.getMessage() + " (" + t.getClass() + ")");
+ return false;
+ }
+ return (mode & S_IFIFO) != 0;
+ }
+
public static RecordWriter getHiveRecordWriter(JobConf jc,
TableDesc tableInfo, Class<? extends Writable> outputClass,
FileSinkDesc conf, Path outPath, Reporter reporter) throws HiveException {
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/InputFormatChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/InputFormatChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/io/InputFormatChecker.java
index 3945411..129b834 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/InputFormatChecker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/InputFormatChecker.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
-import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -34,7 +34,6 @@ public interface InputFormatChecker {
* This method is used to validate the input files.
*
*/
- boolean validateInput(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files) throws IOException;
+ boolean validateInput(FileSystem fs, HiveConf conf, List<FileStatus> files) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java
index 88198ed..6004db8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/RCFileInputFormat.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -59,7 +60,7 @@ public class RCFileInputFormat<K extends LongWritable, V extends BytesRefArrayWr
@Override
public boolean validateInput(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files) throws IOException {
+ List<FileStatus> files) throws IOException {
if (files.size() <= 0) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileInputFormatChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileInputFormatChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileInputFormatChecker.java
index e2666d7..6cb46c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileInputFormatChecker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/SequenceFileInputFormatChecker.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -35,7 +36,7 @@ public class SequenceFileInputFormatChecker implements InputFormatChecker {
@Override
public boolean validateInput(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files) throws IOException {
+ List<FileStatus> files) throws IOException {
if (files.size() <= 0) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java
index faad5f2..e9e1d5a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.io;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.List;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -56,7 +57,7 @@ public class VectorizedRCFileInputFormat extends FileInputFormat<NullWritable, V
@Override
public boolean validateInput(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files) throws IOException {
+ List<FileStatus> files) throws IOException {
if (files.size() <= 0) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
index c45b6e6..57bde3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
@@ -106,7 +106,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* that added this event. Insert and update events include the entire row, while
* delete events have null for row.
*/
-public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
+public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
InputFormatChecker, VectorizedInputFormatInterface,
AcidInputFormat<NullWritable, OrcStruct>, CombineHiveInputFormat.AvoidSplitCombination {
@@ -395,7 +395,7 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>,
}
@Override
public boolean validateInput(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files
+ List<FileStatus> files
) throws IOException {
if (Utilities.isVectorMode(conf)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
index 3992d8c..bf09001 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
@@ -158,7 +158,7 @@ public class VectorizedOrcInputFormat extends FileInputFormat<NullWritable, Vect
@Override
public boolean validateInput(FileSystem fs, HiveConf conf,
- ArrayList<FileStatus> files
+ List<FileStatus> files
) throws IOException {
if (files.size() <= 0) {
return false;
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
index 9e89376..4594836 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
@@ -704,7 +704,7 @@ public class TestFileSinkOperator {
}
@Override
- public boolean validateInput(FileSystem fs, HiveConf conf, ArrayList<FileStatus> files) throws
+ public boolean validateInput(FileSystem fs, HiveConf conf, List<FileStatus> files) throws
IOException {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/82bc0e1c/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
index 21adc9d..5a8c932 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
@@ -361,7 +361,7 @@ public abstract class CompactorTest {
}
@Override
- public boolean validateInput(FileSystem fs, HiveConf conf, ArrayList<FileStatus> files) throws
+ public boolean validateInput(FileSystem fs, HiveConf conf, List<FileStatus> files) throws
IOException {
return false;
}
[2/4] hive git commit: HIVE-11970 : COLUMNS_V2 table in metastore
should have a longer name field (Sergey Shelukhin, reviewed by Alan Gates)
Posted by se...@apache.org.
HIVE-11970 : COLUMNS_V2 table in metastore should have a longer name field (Sergey Shelukhin, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a953b436
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a953b436
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a953b436
Branch: refs/heads/master
Commit: a953b43628bf0dbbb9aadc4ae936e8c964534f3f
Parents: 82bc0e1
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 1 14:09:36 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 1 14:09:36 2015 -0700
----------------------------------------------------------------------
.../upgrade/derby/021-HIVE-11970.derby.sql | 6 +++++
.../upgrade/derby/hive-schema-1.3.0.derby.sql | 12 +++++-----
.../upgrade/derby/hive-schema-2.0.0.derby.sql | 12 +++++-----
.../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 1 +
.../derby/upgrade-1.2.0-to-2.0.0.derby.sql | 3 ++-
.../upgrade/mssql/007-HIVE-11970.mssql.sql | 6 +++++
.../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 12 +++++-----
.../upgrade/mssql/hive-schema-2.0.0.mssql.sql | 12 +++++-----
.../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 1 +
.../mssql/upgrade-1.2.0-to-2.0.0.mssql.sql | 7 +++---
.../upgrade/mysql/022-HIVE-11970.mysql.sql | 6 +++++
.../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 12 +++++-----
.../upgrade/mysql/hive-schema-2.0.0.mysql.sql | 12 +++++-----
.../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 1 +
.../mysql/upgrade-1.2.0-to-2.0.0.mysql.sql | 2 ++
.../upgrade/oracle/022-HIVE-11970.oracle.sql | 23 ++++++++++++++++++++
.../upgrade/oracle/hive-schema-1.3.0.oracle.sql | 12 +++++-----
.../upgrade/oracle/hive-schema-2.0.0.oracle.sql | 12 +++++-----
.../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 2 ++
.../oracle/upgrade-1.2.0-to-2.0.0.oracle.sql | 2 ++
.../postgres/021-HIVE-11970.postgres.sql | 6 +++++
.../postgres/hive-schema-1.3.0.postgres.sql | 12 +++++-----
.../postgres/hive-schema-2.0.0.postgres.sql | 12 +++++-----
.../upgrade-1.2.0-to-1.3.0.postgres.sql | 1 +
.../upgrade-1.2.0-to-2.0.0.postgres.sql | 1 +
25 files changed, 124 insertions(+), 64 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/derby/021-HIVE-11970.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/021-HIVE-11970.derby.sql b/metastore/scripts/upgrade/derby/021-HIVE-11970.derby.sql
new file mode 100644
index 0000000..6a01a53
--- /dev/null
+++ b/metastore/scripts/upgrade/derby/021-HIVE-11970.derby.sql
@@ -0,0 +1,6 @@
+ALTER TABLE "COLUMNS_V2" ALTER "COLUMN_NAME" SET DATA TYPE VARCHAR(1000);
+ALTER TABLE "PART_COL_PRIVS" ALTER "COLUMN_NAME" SET DATA TYPE VARCHAR(1000);
+ALTER TABLE "TBL_COL_PRIVS" ALTER "COLUMN_NAME" SET DATA TYPE VARCHAR(1000);
+ALTER TABLE "SORT_COLS" ALTER "COLUMN_NAME" SET DATA TYPE VARCHAR(1000);
+ALTER TABLE "TAB_COL_STATS" ALTER "COLUMN_NAME" SET DATA TYPE VARCHAR(1000);
+ALTER TABLE "PART_COL_STATS" ALTER "COLUMN_NAME" SET DATA TYPE VARCHAR(1000);
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql
index c50375f..20eb326 100644
--- a/metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/hive-schema-1.3.0.derby.sql
@@ -22,13 +22,13 @@ CREATE TABLE "APP"."TBL_PRIVS" ("TBL_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" IN
CREATE TABLE "APP"."DATABASE_PARAMS" ("DB_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(180) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT);
+CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(1000), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT);
CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(128) NOT NULL, "TYPE_NAME" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL);
+CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(1000) NOT NULL, "TYPE_NAME" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL);
-CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
+CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(1000), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
CREATE TABLE "APP"."CDS" ("CD_ID" BIGINT NOT NULL);
@@ -64,7 +64,7 @@ CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT N
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
-CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128));
+CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(1000), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128));
CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL);
@@ -94,9 +94,9 @@ CREATE TABLE "APP"."MASTER_KEYS" ("KEY_ID" INTEGER NOT NULL generated always as
CREATE TABLE "APP"."DELEGATION_TOKENS" ( "TOKEN_IDENT" VARCHAR(767) NOT NULL, "TOKEN" VARCHAR(767));
-CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "COLUMN_NAME" VARCHAR(1000) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL);
-CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(1000) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
index 1cc0a24..abc6bf4 100644
--- a/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/hive-schema-2.0.0.derby.sql
@@ -22,13 +22,13 @@ CREATE TABLE "APP"."TBL_PRIVS" ("TBL_GRANT_ID" BIGINT NOT NULL, "CREATE_TIME" IN
CREATE TABLE "APP"."DATABASE_PARAMS" ("DB_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(180) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT);
+CREATE TABLE "APP"."TBL_COL_PRIVS" ("TBL_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(1000), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "TBL_COL_PRIV" VARCHAR(128), "TBL_ID" BIGINT);
CREATE TABLE "APP"."SERDE_PARAMS" ("SERDE_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000));
-CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(128) NOT NULL, "TYPE_NAME" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL);
+CREATE TABLE "APP"."COLUMNS_V2" ("CD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(4000), "COLUMN_NAME" VARCHAR(1000) NOT NULL, "TYPE_NAME" VARCHAR(4000), "INTEGER_IDX" INTEGER NOT NULL);
-CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
+CREATE TABLE "APP"."SORT_COLS" ("SD_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(1000), "ORDER" INTEGER NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
CREATE TABLE "APP"."CDS" ("CD_ID" BIGINT NOT NULL);
@@ -64,7 +64,7 @@ CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT N
CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL);
-CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(128), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128));
+CREATE TABLE "APP"."PART_COL_PRIVS" ("PART_COLUMN_GRANT_ID" BIGINT NOT NULL, "COLUMN_NAME" VARCHAR(1000), "CREATE_TIME" INTEGER NOT NULL, "GRANT_OPTION" SMALLINT NOT NULL, "GRANTOR" VARCHAR(128), "GRANTOR_TYPE" VARCHAR(128), "PART_ID" BIGINT, "PRINCIPAL_NAME" VARCHAR(128), "PRINCIPAL_TYPE" VARCHAR(128), "PART_COL_PRIV" VARCHAR(128));
CREATE TABLE "APP"."SDS" ("SD_ID" BIGINT NOT NULL, "INPUT_FORMAT" VARCHAR(4000), "IS_COMPRESSED" CHAR(1) NOT NULL, "LOCATION" VARCHAR(4000), "NUM_BUCKETS" INTEGER NOT NULL, "OUTPUT_FORMAT" VARCHAR(4000), "SERDE_ID" BIGINT, "CD_ID" BIGINT, "IS_STOREDASSUBDIRECTORIES" CHAR(1) NOT NULL);
@@ -94,9 +94,9 @@ CREATE TABLE "APP"."MASTER_KEYS" ("KEY_ID" INTEGER NOT NULL generated always as
CREATE TABLE "APP"."DELEGATION_TOKENS" ( "TOKEN_IDENT" VARCHAR(767) NOT NULL, "TOKEN" VARCHAR(767));
-CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."TAB_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "COLUMN_NAME" VARCHAR(1000) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL);
-CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(128) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
+CREATE TABLE "APP"."PART_COL_STATS"("DB_NAME" VARCHAR(128) NOT NULL,"TABLE_NAME" VARCHAR(128) NOT NULL, "PARTITION_NAME" VARCHAR(767) NOT NULL, "COLUMN_NAME" VARCHAR(1000) NOT NULL, "COLUMN_TYPE" VARCHAR(128) NOT NULL, "LONG_LOW_VALUE" BIGINT, "LONG_HIGH_VALUE" BIGINT, "DOUBLE_LOW_VALUE" DOUBLE, "DOUBLE_HIGH_VALUE" DOUBLE, "BIG_DECIMAL_LOW_VALUE" VARCHAR(4000), "BIG_DECIMAL_HIGH_VALUE" VARCHAR(4000),"NUM_DISTINCTS" BIGINT, "NUM_NULLS" BIGINT NOT NULL, "AVG_COL_LEN" DOUBLE, "MAX_COL_LEN" BIGINT, "NUM_TRUES" BIGINT, "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "PART_ID" BIGINT NOT NULL);
CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255));
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql
index dee744b..6359de4 100644
--- a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql
@@ -1,3 +1,4 @@
-- Upgrade MetaStore schema from 1.2.0 to 1.3.0
+RUN '021-HIVE-11970.derby.sql';
UPDATE "APP".VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
index 976ebd1..adf3cb0 100644
--- a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
+++ b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-2.0.0.derby.sql
@@ -1,3 +1,4 @@
--- Upgrade MetaStore schema from 1.3.0 to 2.0.0
+-- Upgrade MetaStore schema from 1.2.0 to 2.0.0
+RUN '021-HIVE-11970.derby.sql';
UPDATE "APP".VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mssql/007-HIVE-11970.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/007-HIVE-11970.mssql.sql b/metastore/scripts/upgrade/mssql/007-HIVE-11970.mssql.sql
new file mode 100644
index 0000000..69b11a2
--- /dev/null
+++ b/metastore/scripts/upgrade/mssql/007-HIVE-11970.mssql.sql
@@ -0,0 +1,6 @@
+ALTER TABLE "COLUMNS_V2" ALTER COLUMN "COLUMN_NAME" VARCHAR(1000) NOT NULL;
+ALTER TABLE "PART_COL_PRIVS" ALTER COLUMN "COLUMN_NAME" VARCHAR(1000) NULL;
+ALTER TABLE "TBL_COL_PRIVS" ALTER COLUMN "COLUMN_NAME" VARCHAR(1000) NULL;
+ALTER TABLE "SORT_COLS" ALTER COLUMN "COLUMN_NAME" VARCHAR(1000) NULL;
+ALTER TABLE "TAB_COL_STATS" ALTER COLUMN "COLUMN_NAME" VARCHAR(1000) NOT NULL;
+ALTER TABLE "PART_COL_STATS" ALTER COLUMN "COLUMN_NAME" VARCHAR(1000) NOT NULL;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql
index 7165edd..01c1376 100644
--- a/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql
@@ -75,7 +75,7 @@ CREATE TABLE PART_COL_STATS
(
CS_ID bigint NOT NULL,
AVG_COL_LEN float NULL,
- "COLUMN_NAME" nvarchar(128) NOT NULL,
+ "COLUMN_NAME" nvarchar(1000) NOT NULL,
COLUMN_TYPE nvarchar(128) NOT NULL,
DB_NAME nvarchar(128) NOT NULL,
BIG_DECIMAL_HIGH_VALUE nvarchar(255) NULL,
@@ -184,7 +184,7 @@ ALTER TABLE GLOBAL_PRIVS ADD CONSTRAINT GLOBAL_PRIVS_PK PRIMARY KEY (USER_GRANT_
CREATE TABLE PART_COL_PRIVS
(
PART_COLUMN_GRANT_ID bigint NOT NULL,
- "COLUMN_NAME" nvarchar(128) NULL,
+ "COLUMN_NAME" nvarchar(1000) NULL,
CREATE_TIME int NOT NULL,
GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)),
GRANTOR nvarchar(128) NULL,
@@ -218,7 +218,7 @@ CREATE TABLE TAB_COL_STATS
(
CS_ID bigint NOT NULL,
AVG_COL_LEN float NULL,
- "COLUMN_NAME" nvarchar(128) NOT NULL,
+ "COLUMN_NAME" nvarchar(1000) NOT NULL,
COLUMN_TYPE nvarchar(128) NOT NULL,
DB_NAME nvarchar(128) NOT NULL,
BIG_DECIMAL_HIGH_VALUE nvarchar(255) NULL,
@@ -283,7 +283,7 @@ ALTER TABLE DBS ADD CONSTRAINT DBS_PK PRIMARY KEY (DB_ID);
CREATE TABLE TBL_COL_PRIVS
(
TBL_COLUMN_GRANT_ID bigint NOT NULL,
- "COLUMN_NAME" nvarchar(128) NULL,
+ "COLUMN_NAME" nvarchar(1000) NULL,
CREATE_TIME int NOT NULL,
GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)),
GRANTOR nvarchar(128) NULL,
@@ -396,7 +396,7 @@ ALTER TABLE PARTITION_EVENTS ADD CONSTRAINT PARTITION_EVENTS_PK PRIMARY KEY (PAR
CREATE TABLE SORT_COLS
(
SD_ID bigint NOT NULL,
- "COLUMN_NAME" nvarchar(128) NULL,
+ "COLUMN_NAME" nvarchar(1000) NULL,
"ORDER" int NOT NULL,
INTEGER_IDX int NOT NULL
);
@@ -533,7 +533,7 @@ CREATE TABLE COLUMNS_V2
(
CD_ID bigint NOT NULL,
COMMENT nvarchar(256) NULL,
- "COLUMN_NAME" nvarchar(128) NOT NULL,
+ "COLUMN_NAME" nvarchar(1000) NOT NULL,
TYPE_NAME nvarchar(4000) NOT NULL,
INTEGER_IDX int NOT NULL
);
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
index 36e7c1f..1ec8632 100644
--- a/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/hive-schema-2.0.0.mssql.sql
@@ -75,7 +75,7 @@ CREATE TABLE PART_COL_STATS
(
CS_ID bigint NOT NULL,
AVG_COL_LEN float NULL,
- "COLUMN_NAME" nvarchar(128) NOT NULL,
+ "COLUMN_NAME" nvarchar(1000) NOT NULL,
COLUMN_TYPE nvarchar(128) NOT NULL,
DB_NAME nvarchar(128) NOT NULL,
BIG_DECIMAL_HIGH_VALUE nvarchar(255) NULL,
@@ -184,7 +184,7 @@ ALTER TABLE GLOBAL_PRIVS ADD CONSTRAINT GLOBAL_PRIVS_PK PRIMARY KEY (USER_GRANT_
CREATE TABLE PART_COL_PRIVS
(
PART_COLUMN_GRANT_ID bigint NOT NULL,
- "COLUMN_NAME" nvarchar(128) NULL,
+ "COLUMN_NAME" nvarchar(1000) NULL,
CREATE_TIME int NOT NULL,
GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)),
GRANTOR nvarchar(128) NULL,
@@ -218,7 +218,7 @@ CREATE TABLE TAB_COL_STATS
(
CS_ID bigint NOT NULL,
AVG_COL_LEN float NULL,
- "COLUMN_NAME" nvarchar(128) NOT NULL,
+ "COLUMN_NAME" nvarchar(1000) NOT NULL,
COLUMN_TYPE nvarchar(128) NOT NULL,
DB_NAME nvarchar(128) NOT NULL,
BIG_DECIMAL_HIGH_VALUE nvarchar(255) NULL,
@@ -283,7 +283,7 @@ ALTER TABLE DBS ADD CONSTRAINT DBS_PK PRIMARY KEY (DB_ID);
CREATE TABLE TBL_COL_PRIVS
(
TBL_COLUMN_GRANT_ID bigint NOT NULL,
- "COLUMN_NAME" nvarchar(128) NULL,
+ "COLUMN_NAME" nvarchar(1000) NULL,
CREATE_TIME int NOT NULL,
GRANT_OPTION smallint NOT NULL CHECK (GRANT_OPTION IN (0,1)),
GRANTOR nvarchar(128) NULL,
@@ -396,7 +396,7 @@ ALTER TABLE PARTITION_EVENTS ADD CONSTRAINT PARTITION_EVENTS_PK PRIMARY KEY (PAR
CREATE TABLE SORT_COLS
(
SD_ID bigint NOT NULL,
- "COLUMN_NAME" nvarchar(128) NULL,
+ "COLUMN_NAME" nvarchar(1000) NULL,
"ORDER" int NOT NULL,
INTEGER_IDX int NOT NULL
);
@@ -533,7 +533,7 @@ CREATE TABLE COLUMNS_V2
(
CD_ID bigint NOT NULL,
COMMENT nvarchar(256) NULL,
- "COLUMN_NAME" nvarchar(128) NOT NULL,
+ "COLUMN_NAME" nvarchar(1000) NOT NULL,
TYPE_NAME nvarchar(4000) NOT NULL,
INTEGER_IDX int NOT NULL
);
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql
index fd12a92..9cf9d25 100644
--- a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql
@@ -1,5 +1,6 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE;
+:r 007-HIVE-11970.mssql.sql;
UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql
index fd12a92..de3d29d 100644
--- a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql
+++ b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-2.0.0.mssql.sql
@@ -1,5 +1,6 @@
-SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE;
+SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS MESSAGE;
+:r 007-HIVE-11970.mssql.sql;
-UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
-SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE;
+UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
+SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS MESSAGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mysql/022-HIVE-11970.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/022-HIVE-11970.mysql.sql b/metastore/scripts/upgrade/mysql/022-HIVE-11970.mysql.sql
new file mode 100644
index 0000000..4517e00
--- /dev/null
+++ b/metastore/scripts/upgrade/mysql/022-HIVE-11970.mysql.sql
@@ -0,0 +1,6 @@
+ALTER TABLE `COLUMNS_V2` MODIFY `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL;
+ALTER TABLE `PART_COL_PRIVS` MODIFY `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
+ALTER TABLE `TBL_COL_PRIVS` MODIFY `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
+ALTER TABLE `SORT_COLS` MODIFY `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL;
+ALTER TABLE `TAB_COL_STATS` MODIFY `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL;
+ALTER TABLE `PART_COL_STATS` MODIFY `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
index 71de138..ce0ac54 100644
--- a/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-1.3.0.mysql.sql
@@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS `CDS` (
CREATE TABLE IF NOT EXISTS `COLUMNS_V2` (
`CD_ID` bigint(20) NOT NULL,
`COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TYPE_NAME` varchar(4000) DEFAULT NULL,
`INTEGER_IDX` int(11) NOT NULL,
PRIMARY KEY (`CD_ID`,`COLUMN_NAME`),
@@ -296,7 +296,7 @@ CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` (
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` (
`PART_COLUMN_GRANT_ID` bigint(20) NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`CREATE_TIME` int(11) NOT NULL,
`GRANT_OPTION` smallint(6) NOT NULL,
`GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
@@ -544,7 +544,7 @@ CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` (
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE IF NOT EXISTS `SORT_COLS` (
`SD_ID` bigint(20) NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`ORDER` int(11) NOT NULL,
`INTEGER_IDX` int(11) NOT NULL,
PRIMARY KEY (`SD_ID`,`INTEGER_IDX`),
@@ -604,7 +604,7 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` (
`TBL_COLUMN_GRANT_ID` bigint(20) NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`CREATE_TIME` int(11) NOT NULL,
`GRANT_OPTION` smallint(6) NOT NULL,
`GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
@@ -650,7 +650,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` (
`CS_ID` bigint(20) NOT NULL,
`DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TBL_ID` bigint(20) NOT NULL,
`LONG_LOW_VALUE` bigint(20),
@@ -678,7 +678,7 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` (
`DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`PART_ID` bigint(20) NOT NULL,
`LONG_LOW_VALUE` bigint(20),
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
index 6547cf1..3a2c0e2 100644
--- a/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/hive-schema-2.0.0.mysql.sql
@@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS `CDS` (
CREATE TABLE IF NOT EXISTS `COLUMNS_V2` (
`CD_ID` bigint(20) NOT NULL,
`COMMENT` varchar(256) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TYPE_NAME` varchar(4000) DEFAULT NULL,
`INTEGER_IDX` int(11) NOT NULL,
PRIMARY KEY (`CD_ID`,`COLUMN_NAME`),
@@ -296,7 +296,7 @@ CREATE TABLE IF NOT EXISTS `PARTITION_PARAMS` (
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE IF NOT EXISTS `PART_COL_PRIVS` (
`PART_COLUMN_GRANT_ID` bigint(20) NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`CREATE_TIME` int(11) NOT NULL,
`GRANT_OPTION` smallint(6) NOT NULL,
`GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
@@ -544,7 +544,7 @@ CREATE TABLE IF NOT EXISTS `SKEWED_VALUES` (
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE IF NOT EXISTS `SORT_COLS` (
`SD_ID` bigint(20) NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`ORDER` int(11) NOT NULL,
`INTEGER_IDX` int(11) NOT NULL,
PRIMARY KEY (`SD_ID`,`INTEGER_IDX`),
@@ -604,7 +604,7 @@ CREATE TABLE IF NOT EXISTS `TBLS` (
/*!40101 SET character_set_client = utf8 */;
CREATE TABLE IF NOT EXISTS `TBL_COL_PRIVS` (
`TBL_COLUMN_GRANT_ID` bigint(20) NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
`CREATE_TIME` int(11) NOT NULL,
`GRANT_OPTION` smallint(6) NOT NULL,
`GRANTOR` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL,
@@ -650,7 +650,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` (
`CS_ID` bigint(20) NOT NULL,
`DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TBL_ID` bigint(20) NOT NULL,
`LONG_LOW_VALUE` bigint(20),
@@ -678,7 +678,7 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` (
`DB_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`TABLE_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`PARTITION_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
- `COLUMN_NAME` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+ `COLUMN_NAME` varchar(1000) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`COLUMN_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
`PART_ID` bigint(20) NOT NULL,
`LONG_LOW_VALUE` bigint(20),
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
index 44a9946..1b32d93 100644
--- a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql
@@ -1,4 +1,5 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' ';
SOURCE 021-HIVE-7018.mysql.sql;
+SOURCE 022-HIVE-11970.mysql.sql;
UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' ';
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
index 22d5242..1340f27 100644
--- a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
+++ b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-2.0.0.mysql.sql
@@ -1,4 +1,6 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' ';
+SOURCE 021-HIVE-7018.mysql.sql;
+SOURCE 022-HIVE-11970.mysql.sql;
UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS ' ';
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/oracle/022-HIVE-11970.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/022-HIVE-11970.oracle.sql b/metastore/scripts/upgrade/oracle/022-HIVE-11970.oracle.sql
new file mode 100644
index 0000000..2f11157
--- /dev/null
+++ b/metastore/scripts/upgrade/oracle/022-HIVE-11970.oracle.sql
@@ -0,0 +1,23 @@
+ALTER TABLE COLUMNS_V2 MODIFY (
+ "COLUMN_NAME" VARCHAR2(1000)
+);
+
+ALTER TABLE PART_COL_PRIVS MODIFY (
+ "COLUMN_NAME" VARCHAR2(1000)
+);
+
+ALTER TABLE TBL_COL_PRIVS MODIFY (
+ "COLUMN_NAME" VARCHAR2(1000)
+);
+
+ALTER TABLE SORT_COLS MODIFY (
+ "COLUMN_NAME" VARCHAR2(1000)
+);
+
+ALTER TABLE TAB_COL_STATS MODIFY (
+ "COLUMN_NAME" VARCHAR2(1000)
+);
+
+ALTER TABLE PART_COL_STATS MODIFY (
+ "COLUMN_NAME" VARCHAR2(1000)
+);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql
index 7605bc7..d2e7945 100644
--- a/metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/hive-schema-1.3.0.oracle.sql
@@ -29,7 +29,7 @@ ALTER TABLE NUCLEUS_TABLES ADD CONSTRAINT NUCLEUS_TABLES_PK PRIMARY KEY (CLASS_N
CREATE TABLE PART_COL_PRIVS
(
PART_COLUMN_GRANT_ID NUMBER NOT NULL,
- "COLUMN_NAME" VARCHAR2(128) NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NULL,
CREATE_TIME NUMBER (10) NOT NULL,
GRANT_OPTION NUMBER (5) NOT NULL,
GRANTOR VARCHAR2(128) NULL,
@@ -55,7 +55,7 @@ CREATE TABLE COLUMNS_V2
(
CD_ID NUMBER NOT NULL,
"COMMENT" VARCHAR2(256) NULL,
- "COLUMN_NAME" VARCHAR2(128) NOT NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NOT NULL,
TYPE_NAME VARCHAR2(4000) NOT NULL,
INTEGER_IDX NUMBER(10) NOT NULL
);
@@ -166,7 +166,7 @@ ALTER TABLE INDEX_PARAMS ADD CONSTRAINT INDEX_PARAMS_PK PRIMARY KEY (INDEX_ID,PA
CREATE TABLE TBL_COL_PRIVS
(
TBL_COLUMN_GRANT_ID NUMBER NOT NULL,
- "COLUMN_NAME" VARCHAR2(128) NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NULL,
CREATE_TIME NUMBER (10) NOT NULL,
GRANT_OPTION NUMBER (5) NOT NULL,
GRANTOR VARCHAR2(128) NULL,
@@ -272,7 +272,7 @@ ALTER TABLE TABLE_PARAMS ADD CONSTRAINT TABLE_PARAMS_PK PRIMARY KEY (TBL_ID,PARA
CREATE TABLE SORT_COLS
(
SD_ID NUMBER NOT NULL,
- "COLUMN_NAME" VARCHAR2(128) NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NULL,
"ORDER" NUMBER (10) NOT NULL,
INTEGER_IDX NUMBER(10) NOT NULL
);
@@ -467,7 +467,7 @@ CREATE TABLE TAB_COL_STATS (
CS_ID NUMBER NOT NULL,
DB_NAME VARCHAR2(128) NOT NULL,
TABLE_NAME VARCHAR2(128) NOT NULL,
- COLUMN_NAME VARCHAR2(128) NOT NULL,
+ COLUMN_NAME VARCHAR2(1000) NOT NULL,
COLUMN_TYPE VARCHAR2(128) NOT NULL,
TBL_ID NUMBER NOT NULL,
LONG_LOW_VALUE NUMBER,
@@ -503,7 +503,7 @@ CREATE TABLE PART_COL_STATS (
DB_NAME VARCHAR2(128) NOT NULL,
TABLE_NAME VARCHAR2(128) NOT NULL,
PARTITION_NAME VARCHAR2(767) NOT NULL,
- COLUMN_NAME VARCHAR2(128) NOT NULL,
+ COLUMN_NAME VARCHAR2(1000) NOT NULL,
COLUMN_TYPE VARCHAR2(128) NOT NULL,
PART_ID NUMBER NOT NULL,
LONG_LOW_VALUE NUMBER,
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
index 8d963ce..2dcdd77 100644
--- a/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/hive-schema-2.0.0.oracle.sql
@@ -29,7 +29,7 @@ ALTER TABLE NUCLEUS_TABLES ADD CONSTRAINT NUCLEUS_TABLES_PK PRIMARY KEY (CLASS_N
CREATE TABLE PART_COL_PRIVS
(
PART_COLUMN_GRANT_ID NUMBER NOT NULL,
- "COLUMN_NAME" VARCHAR2(128) NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NULL,
CREATE_TIME NUMBER (10) NOT NULL,
GRANT_OPTION NUMBER (5) NOT NULL,
GRANTOR VARCHAR2(128) NULL,
@@ -55,7 +55,7 @@ CREATE TABLE COLUMNS_V2
(
CD_ID NUMBER NOT NULL,
"COMMENT" VARCHAR2(256) NULL,
- "COLUMN_NAME" VARCHAR2(128) NOT NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NOT NULL,
TYPE_NAME VARCHAR2(4000) NOT NULL,
INTEGER_IDX NUMBER(10) NOT NULL
);
@@ -166,7 +166,7 @@ ALTER TABLE INDEX_PARAMS ADD CONSTRAINT INDEX_PARAMS_PK PRIMARY KEY (INDEX_ID,PA
CREATE TABLE TBL_COL_PRIVS
(
TBL_COLUMN_GRANT_ID NUMBER NOT NULL,
- "COLUMN_NAME" VARCHAR2(128) NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NULL,
CREATE_TIME NUMBER (10) NOT NULL,
GRANT_OPTION NUMBER (5) NOT NULL,
GRANTOR VARCHAR2(128) NULL,
@@ -272,7 +272,7 @@ ALTER TABLE TABLE_PARAMS ADD CONSTRAINT TABLE_PARAMS_PK PRIMARY KEY (TBL_ID,PARA
CREATE TABLE SORT_COLS
(
SD_ID NUMBER NOT NULL,
- "COLUMN_NAME" VARCHAR2(128) NULL,
+ "COLUMN_NAME" VARCHAR2(1000) NULL,
"ORDER" NUMBER (10) NOT NULL,
INTEGER_IDX NUMBER(10) NOT NULL
);
@@ -467,7 +467,7 @@ CREATE TABLE TAB_COL_STATS (
CS_ID NUMBER NOT NULL,
DB_NAME VARCHAR2(128) NOT NULL,
TABLE_NAME VARCHAR2(128) NOT NULL,
- COLUMN_NAME VARCHAR2(128) NOT NULL,
+ COLUMN_NAME VARCHAR2(1000) NOT NULL,
COLUMN_TYPE VARCHAR2(128) NOT NULL,
TBL_ID NUMBER NOT NULL,
LONG_LOW_VALUE NUMBER,
@@ -503,7 +503,7 @@ CREATE TABLE PART_COL_STATS (
DB_NAME VARCHAR2(128) NOT NULL,
TABLE_NAME VARCHAR2(128) NOT NULL,
PARTITION_NAME VARCHAR2(767) NOT NULL,
- COLUMN_NAME VARCHAR2(128) NOT NULL,
+ COLUMN_NAME VARCHAR2(1000) NOT NULL,
COLUMN_TYPE VARCHAR2(128) NOT NULL,
PART_ID NUMBER NOT NULL,
LONG_LOW_VALUE NUMBER,
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql
index f072a1c..bd283d4 100644
--- a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql
@@ -1,4 +1,6 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual;
+@022-HIVE-11970.oracle.sql;
+
UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
index efff2c9..ceb09be 100644
--- a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
+++ b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-2.0.0.oracle.sql
@@ -1,4 +1,6 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0' AS Status from dual;
+@022-HIVE-11970.oracle.sql;
+
UPDATE VERSION SET SCHEMA_VERSION='2.0.0', VERSION_COMMENT='Hive release version 2.0.0' where VER_ID=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0' AS Status from dual;
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/postgres/021-HIVE-11970.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/021-HIVE-11970.postgres.sql b/metastore/scripts/upgrade/postgres/021-HIVE-11970.postgres.sql
new file mode 100644
index 0000000..08cb4a5
--- /dev/null
+++ b/metastore/scripts/upgrade/postgres/021-HIVE-11970.postgres.sql
@@ -0,0 +1,6 @@
+ALTER TABLE "COLUMNS_V2" ALTER "COLUMN_NAME" TYPE character varying(1000);
+ALTER TABLE "PART_COL_PRIVS" ALTER "COLUMN_NAME" TYPE character varying(1000);
+ALTER TABLE "TBL_COL_PRIVS" ALTER "COLUMN_NAME" TYPE character varying(1000);
+ALTER TABLE "SORT_COLS" ALTER "COLUMN_NAME" TYPE character varying(1000);
+ALTER TABLE "TAB_COL_STATS" ALTER "COLUMN_NAME" TYPE character varying(1000);
+ALTER TABLE "PART_COL_STATS" ALTER "COLUMN_NAME" TYPE character varying(1000);
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql
index 3ab5b3e..9bb5765 100644
--- a/metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/hive-schema-1.3.0.postgres.sql
@@ -42,7 +42,7 @@ CREATE TABLE "CDS" (
CREATE TABLE "COLUMNS_V2" (
"CD_ID" bigint NOT NULL,
"COMMENT" character varying(4000),
- "COLUMN_NAME" character varying(128) NOT NULL,
+ "COLUMN_NAME" character varying(1000) NOT NULL,
"TYPE_NAME" character varying(4000),
"INTEGER_IDX" integer NOT NULL
);
@@ -217,7 +217,7 @@ CREATE TABLE "PARTITION_PARAMS" (
CREATE TABLE "PART_COL_PRIVS" (
"PART_COLUMN_GRANT_ID" bigint NOT NULL,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"CREATE_TIME" bigint NOT NULL,
"GRANT_OPTION" smallint NOT NULL,
"GRANTOR" character varying(128) DEFAULT NULL::character varying,
@@ -340,7 +340,7 @@ CREATE TABLE "SERDE_PARAMS" (
CREATE TABLE "SORT_COLS" (
"SD_ID" bigint NOT NULL,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"ORDER" bigint NOT NULL,
"INTEGER_IDX" bigint NOT NULL
);
@@ -382,7 +382,7 @@ CREATE TABLE "TBLS" (
CREATE TABLE "TBL_COL_PRIVS" (
"TBL_COLUMN_GRANT_ID" bigint NOT NULL,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"CREATE_TIME" bigint NOT NULL,
"GRANT_OPTION" smallint NOT NULL,
"GRANTOR" character varying(128) DEFAULT NULL::character varying,
@@ -486,7 +486,7 @@ CREATE TABLE "TAB_COL_STATS" (
"CS_ID" bigint NOT NULL,
"DB_NAME" character varying(128) DEFAULT NULL::character varying,
"TABLE_NAME" character varying(128) DEFAULT NULL::character varying,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying,
"TBL_ID" bigint NOT NULL,
"LONG_LOW_VALUE" bigint,
@@ -522,7 +522,7 @@ CREATE TABLE "PART_COL_STATS" (
"DB_NAME" character varying(128) DEFAULT NULL::character varying,
"TABLE_NAME" character varying(128) DEFAULT NULL::character varying,
"PARTITION_NAME" character varying(767) DEFAULT NULL::character varying,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying,
"PART_ID" bigint NOT NULL,
"LONG_LOW_VALUE" bigint,
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
index 6442eb1..c749a29 100644
--- a/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/hive-schema-2.0.0.postgres.sql
@@ -42,7 +42,7 @@ CREATE TABLE "CDS" (
CREATE TABLE "COLUMNS_V2" (
"CD_ID" bigint NOT NULL,
"COMMENT" character varying(4000),
- "COLUMN_NAME" character varying(128) NOT NULL,
+ "COLUMN_NAME" character varying(1000) NOT NULL,
"TYPE_NAME" character varying(4000),
"INTEGER_IDX" integer NOT NULL
);
@@ -217,7 +217,7 @@ CREATE TABLE "PARTITION_PARAMS" (
CREATE TABLE "PART_COL_PRIVS" (
"PART_COLUMN_GRANT_ID" bigint NOT NULL,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"CREATE_TIME" bigint NOT NULL,
"GRANT_OPTION" smallint NOT NULL,
"GRANTOR" character varying(128) DEFAULT NULL::character varying,
@@ -340,7 +340,7 @@ CREATE TABLE "SERDE_PARAMS" (
CREATE TABLE "SORT_COLS" (
"SD_ID" bigint NOT NULL,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"ORDER" bigint NOT NULL,
"INTEGER_IDX" bigint NOT NULL
);
@@ -382,7 +382,7 @@ CREATE TABLE "TBLS" (
CREATE TABLE "TBL_COL_PRIVS" (
"TBL_COLUMN_GRANT_ID" bigint NOT NULL,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"CREATE_TIME" bigint NOT NULL,
"GRANT_OPTION" smallint NOT NULL,
"GRANTOR" character varying(128) DEFAULT NULL::character varying,
@@ -486,7 +486,7 @@ CREATE TABLE "TAB_COL_STATS" (
"CS_ID" bigint NOT NULL,
"DB_NAME" character varying(128) DEFAULT NULL::character varying,
"TABLE_NAME" character varying(128) DEFAULT NULL::character varying,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying,
"TBL_ID" bigint NOT NULL,
"LONG_LOW_VALUE" bigint,
@@ -522,7 +522,7 @@ CREATE TABLE "PART_COL_STATS" (
"DB_NAME" character varying(128) DEFAULT NULL::character varying,
"TABLE_NAME" character varying(128) DEFAULT NULL::character varying,
"PARTITION_NAME" character varying(767) DEFAULT NULL::character varying,
- "COLUMN_NAME" character varying(128) DEFAULT NULL::character varying,
+ "COLUMN_NAME" character varying(1000) DEFAULT NULL::character varying,
"COLUMN_TYPE" character varying(128) DEFAULT NULL::character varying,
"PART_ID" bigint NOT NULL,
"LONG_LOW_VALUE" bigint,
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql
index cdd3792..b1ec241 100644
--- a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql
@@ -1,5 +1,6 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0';
+\i 021-HIVE-11970.postgres.sql;
UPDATE "VERSION" SET "SCHEMA_VERSION"='1.3.0', "VERSION_COMMENT"='Hive release version 1.3.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0';
http://git-wip-us.apache.org/repos/asf/hive/blob/a953b436/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
index b6b0c35..628444c 100644
--- a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
+++ b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-2.0.0.postgres.sql
@@ -1,5 +1,6 @@
SELECT 'Upgrading MetaStore schema from 1.2.0 to 2.0.0';
+\i 021-HIVE-11970.postgres.sql;
UPDATE "VERSION" SET "SCHEMA_VERSION"='2.0.0', "VERSION_COMMENT"='Hive release version 2.0.0' where "VER_ID"=1;
SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 2.0.0';
[4/4] hive git commit: HIVE-11960 : braces in join conditions are not
supported (Sergey Shelukhin, reviewed by Pengcheng Xiong)
Posted by se...@apache.org.
HIVE-11960 : braces in join conditions are not supported (Sergey Shelukhin, reviewed by Pengcheng Xiong)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/947871a3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/947871a3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/947871a3
Branch: refs/heads/master
Commit: 947871a3b060adbc46cad8d9416117a81f50fd94
Parents: 116c3e3
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 1 14:14:12 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 1 14:14:12 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/FromClauseParser.g | 30 +-
.../apache/hadoop/hive/ql/parse/HiveParser.g | 7 +-
ql/src/test/queries/clientpositive/join_parse.q | 20 +
.../clientnegative/cte_with_in_subquery.q.out | 2 +-
.../results/clientpositive/join_parse.q.out | 516 +++++++++++++++++++
5 files changed, 568 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/947871a3/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
index 038ed99..084c421 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
@@ -94,7 +94,7 @@ joinSource
;
uniqueJoinSource
-@init { gParent.pushMsg("join source", state); }
+@init { gParent.pushMsg("unique join source", state); }
@after { gParent.popMsg(state); }
: KW_PRESERVE? fromSource uniqueJoinExpr
;
@@ -147,6 +147,16 @@ fromSource
@init { gParent.pushMsg("from source", state); }
@after { gParent.popMsg(state); }
:
+ (LPAREN KW_VALUES) => fromSource0
+ | (LPAREN) => LPAREN joinSource RPAREN -> joinSource
+ | fromSource0
+ ;
+
+
+fromSource0
+@init { gParent.pushMsg("from source 0", state); }
+@after { gParent.popMsg(state); }
+ :
((Identifier LPAREN)=> partitionedTableFunction | tableSource | subQuerySource | virtualTableSource) (lateralView^)*
;
@@ -270,11 +280,15 @@ searchCondition
// INSERT INTO <table> (col1,col2,...) VALUES(...),(...),...
// INSERT INTO <table> (col1,col2,...) SELECT * FROM (VALUES(1,2,3),(4,5,6),...) as Foo(a,b,c)
valueRowConstructor
+@init { gParent.pushMsg("value row constructor", state); }
+@after { gParent.popMsg(state); }
:
LPAREN precedenceUnaryPrefixExpression (COMMA precedenceUnaryPrefixExpression)* RPAREN -> ^(TOK_VALUE_ROW precedenceUnaryPrefixExpression+)
;
valuesTableConstructor
+@init { gParent.pushMsg("values table constructor", state); }
+@after { gParent.popMsg(state); }
:
valueRowConstructor (COMMA valueRowConstructor)* -> ^(TOK_VALUES_TABLE valueRowConstructor+)
;
@@ -285,6 +299,8 @@ VALUES(1,2),(3,4) means 2 rows, 2 columns each.
VALUES(1,2,3) means 1 row, 3 columns
*/
valuesClause
+@init { gParent.pushMsg("values clause", state); }
+@after { gParent.popMsg(state); }
:
KW_VALUES valuesTableConstructor -> valuesTableConstructor
;
@@ -294,16 +310,20 @@ This represents a clause like this:
(VALUES(1,2),(2,3)) as VirtTable(col1,col2)
*/
virtualTableSource
- :
- LPAREN valuesClause RPAREN tableNameColList -> ^(TOK_VIRTUAL_TABLE tableNameColList valuesClause)
- ;
+@init { gParent.pushMsg("virtual table source", state); }
+@after { gParent.popMsg(state); }
+ :
+ LPAREN valuesClause RPAREN tableNameColList -> ^(TOK_VIRTUAL_TABLE tableNameColList valuesClause)
+ ;
/*
e.g. as VirtTable(col1,col2)
Note that we only want literals as column names
*/
tableNameColList
+@init { gParent.pushMsg("from source", state); }
+@after { gParent.popMsg(state); }
:
KW_AS? identifier LPAREN identifier (COMMA identifier)* RPAREN -> ^(TOK_VIRTUAL_TABREF ^(TOK_TABNAME identifier) ^(TOK_COL_NAME identifier+))
;
-//-----------------------------------------------------------------------------------
\ No newline at end of file
+//-----------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/947871a3/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 3df67e9..161e549 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -376,6 +376,8 @@ import java.util.Collection;
import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
}
@@ -618,19 +620,22 @@ import org.apache.hadoop.hive.conf.HiveConf;
return msg;
}
+ public static final Log LOG = LogFactory.getLog("HiveParser");
public void pushMsg(String msg, RecognizerSharedState state) {
// ANTLR generated code does not wrap the @init code wit this backtracking check,
// even if the matching @after has it. If we have parser rules with that are doing
// some lookahead with syntactic predicates this can cause the push() and pop() calls
// to become unbalanced, so make sure both push/pop check the backtracking state.
if (state.backtracking == 0) {
+ // LOG.debug("Push " + msg);
msgs.push(msg);
}
}
public void popMsg(RecognizerSharedState state) {
if (state.backtracking == 0) {
- msgs.pop();
+ Object o = msgs.pop();
+ // LOG.debug("Pop " + o);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/947871a3/ql/src/test/queries/clientpositive/join_parse.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join_parse.q b/ql/src/test/queries/clientpositive/join_parse.q
new file mode 100644
index 0000000..5955efd
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/join_parse.q
@@ -0,0 +1,20 @@
+explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key))
+inner join src src1 on src1.value =srcpart.value;
+
+explain
+select srcpart.key, src1.value from
+(srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value;
+
+explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value);
+
+explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value)
+inner join src src2 on src2.key = src1.key;
http://git-wip-us.apache.org/repos/asf/hive/blob/947871a3/ql/src/test/results/clientnegative/cte_with_in_subquery.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/cte_with_in_subquery.q.out b/ql/src/test/results/clientnegative/cte_with_in_subquery.q.out
index fa22b48..1122ca5 100644
--- a/ql/src/test/results/clientnegative/cte_with_in_subquery.q.out
+++ b/ql/src/test/results/clientnegative/cte_with_in_subquery.q.out
@@ -1 +1 @@
-FAILED: ParseException line 1:64 Failed to recognize predicate 'select'. Failed rule: 'queryStatementExpression' in subquery source
+FAILED: ParseException line 1:20 cannot recognize input near 'with' 'q1' 'as' in from source
http://git-wip-us.apache.org/repos/asf/hive/blob/947871a3/ql/src/test/results/clientpositive/join_parse.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_parse.q.out b/ql/src/test/results/clientpositive/join_parse.q.out
new file mode 100644
index 0000000..e1a23a0
--- /dev/null
+++ b/ql/src/test/results/clientpositive/join_parse.q.out
@@ -0,0 +1,516 @@
+PREHOOK: query: explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key))
+inner join src src1 on src1.value =srcpart.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key))
+inner join src src1 on src1.value =srcpart.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is not null and key is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select srcpart.key, src1.value from
+(srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select srcpart.key, src1.value from
+(srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is not null and key is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (value is not null and key is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col2 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col2
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value)
+inner join src src2 on src2.key = src1.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select srcpart.key, src1.value from
+((srcpart inner join src on srcpart.key = src.key)
+inner join src src1 on src1.value =srcpart.value)
+inner join src src2 on src2.key = src1.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1, Stage-4
+ Stage-4 is a root stage
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srcpart
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 550 Data size: 5843 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col1 (type: string)
+ 1 _col1 (type: string)
+ outputColumnNames: _col0, _col4
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col4 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 605 Data size: 6427 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key is not null and value is not null) (type: boolean)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 125 Data size: 1328 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col1
+ Statistics: Num rows: 275 Data size: 2921 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
[3/4] hive git commit: HIVE-11898 : support default partition in
metastoredirectsql (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Posted by se...@apache.org.
HIVE-11898 : support default partition in metastoredirectsql (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/116c3e3b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/116c3e3b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/116c3e3b
Branch: refs/heads/master
Commit: 116c3e3b2d2b2b22fef9abed8f092b04155784cd
Parents: a953b43
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 1 14:11:58 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 1 14:11:58 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/MetaStoreDirectSql.java | 34 +++++++++++++-------
1 file changed, 22 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/116c3e3b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 1f89b7c..95b1ccc 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -109,6 +109,7 @@ class MetaStoreDirectSql {
private final DB dbType;
private final int batchSize;
private final boolean convertMapNullsToEmptyStrings;
+ private final String defaultPartName;
/**
* Whether direct SQL can be used with the current datastore backing {@link #pm}.
@@ -116,6 +117,7 @@ class MetaStoreDirectSql {
private final boolean isCompatibleDatastore;
private final boolean isAggregateStatsCacheEnabled;
private AggregateStatsCache aggrStatsCache;
+
public MetaStoreDirectSql(PersistenceManager pm, Configuration conf) {
this.pm = pm;
this.dbType = determineDbType();
@@ -127,6 +129,7 @@ class MetaStoreDirectSql {
convertMapNullsToEmptyStrings =
HiveConf.getBoolVar(conf, ConfVars.METASTORE_ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS);
+ defaultPartName = HiveConf.getVar(conf, ConfVars.DEFAULTPARTITIONNAME);
String jdoIdFactory = HiveConf.getVar(conf, ConfVars.METASTORE_IDENTIFIER_FACTORY);
if (! ("datanucleus1".equalsIgnoreCase(jdoIdFactory))){
@@ -390,7 +393,7 @@ class MetaStoreDirectSql {
// Derby and Oracle do not interpret filters ANSI-properly in some cases and need a workaround.
boolean dbHasJoinCastBug = (dbType == DB.DERBY || dbType == DB.ORACLE);
String sqlFilter = PartitionFilterGenerator.generateSqlFilter(
- table, tree, params, joins, dbHasJoinCastBug);
+ table, tree, params, joins, dbHasJoinCastBug, defaultPartName);
if (sqlFilter == null) {
return null; // Cannot make SQL filter to push down.
}
@@ -490,8 +493,8 @@ class MetaStoreDirectSql {
}
List<Object> sqlResult = executeWithArray(query, params, queryText);
long queryTime = doTrace ? System.nanoTime() : 0;
+ timingTrace(doTrace, queryText, start, queryTime);
if (sqlResult.isEmpty()) {
- timingTrace(doTrace, queryText, start, queryTime);
return new ArrayList<Partition>(); // no partitions, bail early.
}
@@ -508,7 +511,6 @@ class MetaStoreDirectSql {
result = getPartitionsFromPartitionIds(dbName, tblName, isView, sqlResult);
}
- timingTrace(doTrace, queryText, start, queryTime);
query.closeAll();
return result;
}
@@ -921,14 +923,16 @@ class MetaStoreDirectSql {
private final List<Object> params;
private final List<String> joins;
private final boolean dbHasJoinCastBug;
+ private final String defaultPartName;
- private PartitionFilterGenerator(
- Table table, List<Object> params, List<String> joins, boolean dbHasJoinCastBug) {
+ private PartitionFilterGenerator(Table table, List<Object> params, List<String> joins,
+ boolean dbHasJoinCastBug, String defaultPartName) {
this.table = table;
this.params = params;
this.joins = joins;
this.dbHasJoinCastBug = dbHasJoinCastBug;
this.filterBuffer = new FilterBuilder(false);
+ this.defaultPartName = defaultPartName;
}
/**
@@ -939,13 +943,14 @@ class MetaStoreDirectSql {
* @return the string representation of the expression tree
*/
private static String generateSqlFilter(Table table, ExpressionTree tree,
- List<Object> params, List<String> joins, boolean dbHasJoinCastBug) throws MetaException {
+ List<Object> params, List<String> joins, boolean dbHasJoinCastBug, String defaultPartName)
+ throws MetaException {
assert table != null;
if (tree.getRoot() == null) {
return "";
}
PartitionFilterGenerator visitor = new PartitionFilterGenerator(
- table, params, joins, dbHasJoinCastBug);
+ table, params, joins, dbHasJoinCastBug, defaultPartName);
tree.accept(visitor);
if (visitor.filterBuffer.hasError()) {
LOG.info("Unable to push down SQL filter: " + visitor.filterBuffer.getErrorMessage());
@@ -1071,28 +1076,33 @@ class MetaStoreDirectSql {
// Build the filter and add parameters linearly; we are traversing leaf nodes LTR.
String tableValue = "\"FILTER" + partColIndex + "\".\"PART_KEY_VAL\"";
+
if (node.isReverseOrder) {
params.add(nodeValue);
}
+ String tableColumn = tableValue;
if (colType != FilterType.String) {
// The underlying database field is varchar, we need to compare numbers.
- // Note that this won't work with __HIVE_DEFAULT_PARTITION__. It will fail and fall
- // back to JDO. That is by design; we could add an ugly workaround here but didn't.
if (colType == FilterType.Integral) {
tableValue = "cast(" + tableValue + " as decimal(21,0))";
} else if (colType == FilterType.Date) {
tableValue = "cast(" + tableValue + " as date)";
}
+ // Workaround for HIVE_DEFAULT_PARTITION - ignore it like JDO does, for now.
+ String tableValue0 = tableValue;
+ tableValue = "(case when " + tableColumn + " <> ?";
+ params.add(defaultPartName);
+
if (dbHasJoinCastBug) {
// This is a workaround for DERBY-6358 and Oracle bug; it is pretty horrible.
- tableValue = "(case when \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and "
+ tableValue += (" and \"TBLS\".\"TBL_NAME\" = ? and \"DBS\".\"NAME\" = ? and "
+ "\"FILTER" + partColIndex + "\".\"PART_ID\" = \"PARTITIONS\".\"PART_ID\" and "
- + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex + " then "
- + tableValue + " else null end)";
+ + "\"FILTER" + partColIndex + "\".\"INTEGER_IDX\" = " + partColIndex);
params.add(table.getTableName().toLowerCase());
params.add(table.getDbName().toLowerCase());
}
+ tableValue += " then " + tableValue0 + " else null end)";
}
if (!node.isReverseOrder) {
params.add(nodeValue);