You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/02/12 22:36:04 UTC
svn commit: r1659411 - in /hive/trunk: data/files/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
serde/src/java/org/apache/hadoop/hive/serde2/lazy/
Author: xuefu
Date: Thu Feb 12 21:36:04 2015
New Revision: 1659411
URL: http://svn.apache.org/r1659411
Log:
HIVE-9500: Support nested structs over 24 levels (Aihua via Xuefu)
added new files that were missed in previous commit
Added:
hive/trunk/data/files/nestedcomplex_additional.txt
hive/trunk/ql/src/test/queries/clientpositive/nested_complex_additional.q
hive/trunk/ql/src/test/results/clientpositive/nested_complex_additional.q.out
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
Added: hive/trunk/data/files/nestedcomplex_additional.txt
URL: http://svn.apache.org/viewvc/hive/trunk/data/files/nestedcomplex_additional.txt?rev=1659411&view=auto
==============================================================================
--- hive/trunk/data/files/nestedcomplex_additional.txt (added)
+++ hive/trunk/data/files/nestedcomplex_additional.txt Thu Feb 12 21:36:04 2015
@@ -0,0 +1,2 @@
+11 2 3k2v2k1v1a10100
+11 2 3k2v2k1v1a10100
Added: hive/trunk/ql/src/test/queries/clientpositive/nested_complex_additional.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/nested_complex_additional.q?rev=1659411&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/nested_complex_additional.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/nested_complex_additional.q Thu Feb 12 21:36:04 2015
@@ -0,0 +1,21 @@
+create table nestedcomplex_additional (
+simple_int int,
+max_nested_array array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<int>>>>>>>>>>>>>>>>>>>>>>>>>,
+max_nested_map array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<map<string,string>>>>>>>>>>>>>>>>>>>>>>>>>>,
+max_nested_struct array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<struct<s:string, i:int>>>>>>>>>>>>>>>>>>>>>>>>>>,
+simple_string string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'hive.serialization.extend.additional.nesting.levels'='true',
+ 'line.delim'='\n'
+)
+;
+
+describe nestedcomplex_additional;
+describe extended nestedcomplex_additional;
+
+
+load data local inpath '../../data/files/nestedcomplex_additional.txt' overwrite into table nestedcomplex_additional;
+
+select * from nestedcomplex_additional sort by simple_int;
Added: hive/trunk/ql/src/test/results/clientpositive/nested_complex_additional.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/nested_complex_additional.q.out?rev=1659411&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/nested_complex_additional.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/nested_complex_additional.q.out Thu Feb 12 21:36:04 2015
@@ -0,0 +1,72 @@
+PREHOOK: query: create table nestedcomplex_additional (
+simple_int int,
+max_nested_array array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<int>>>>>>>>>>>>>>>>>>>>>>>>>,
+max_nested_map array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<map<string,string>>>>>>>>>>>>>>>>>>>>>>>>>>,
+max_nested_struct array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<struct<s:string, i:int>>>>>>>>>>>>>>>>>>>>>>>>>>,
+simple_string string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'hive.serialization.extend.additional.nesting.levels'='true',
+ 'line.delim'='\n'
+)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@nestedcomplex_additional
+POSTHOOK: query: create table nestedcomplex_additional (
+simple_int int,
+max_nested_array array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<int>>>>>>>>>>>>>>>>>>>>>>>>>,
+max_nested_map array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<map<string,string>>>>>>>>>>>>>>>>>>>>>>>>>>,
+max_nested_struct array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<struct<s:string, i:int>>>>>>>>>>>>>>>>>>>>>>>>>>,
+simple_string string)
+ROW FORMAT SERDE
+ 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
+WITH SERDEPROPERTIES (
+ 'hive.serialization.extend.additional.nesting.levels'='true',
+ 'line.delim'='\n'
+)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@nestedcomplex_additional
+PREHOOK: query: describe nestedcomplex_additional
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@nestedcomplex_additional
+POSTHOOK: query: describe nestedcomplex_additional
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@nestedcomplex_additional
+simple_int int
+max_nested_array array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<int>>>>>>>>>>>>>>>>>>>>>>>>>
+max_nested_map array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<map<string,string>>>>>>>>>>>>>>>>>>>>>>>>>>
+max_nested_struct array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<struct<s:string,i:int>>>>>>>>>>>>>>>>>>>>>>>>>>
+simple_string string
+PREHOOK: query: describe extended nestedcomplex_additional
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@nestedcomplex_additional
+POSTHOOK: query: describe extended nestedcomplex_additional
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@nestedcomplex_additional
+simple_int int
+max_nested_array array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<int>>>>>>>>>>>>>>>>>>>>>>>>>
+max_nested_map array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<map<string,string>>>>>>>>>>>>>>>>>>>>>>>>>>
+max_nested_struct array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<array<struct<s:string,i:int>>>>>>>>>>>>>>>>>>>>>>>>>>
+simple_string string
+
+#### A masked pattern was here ####
+PREHOOK: query: load data local inpath '../../data/files/nestedcomplex_additional.txt' overwrite into table nestedcomplex_additional
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@nestedcomplex_additional
+POSTHOOK: query: load data local inpath '../../data/files/nestedcomplex_additional.txt' overwrite into table nestedcomplex_additional
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@nestedcomplex_additional
+PREHOOK: query: select * from nestedcomplex_additional sort by simple_int
+PREHOOK: type: QUERY
+PREHOOK: Input: default@nestedcomplex_additional
+#### A masked pattern was here ####
+POSTHOOK: query: select * from nestedcomplex_additional sort by simple_int
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@nestedcomplex_additional
+#### A masked pattern was here ####
+1 [[[[[[[[[[[[[[[[[[[[[[[[[1,2,3]]]]]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[[[[[{"k2":"v2","k1":"v1"}]]]]]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[[[[[{"s":"a","i":10}]]]]]]]]]]]]]]]]]]]]]]]]] 100
+1 [[[[[[[[[[[[[[[[[[[[[[[[[1,2,3]]]]]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[[[[[{"k2":"v2","k1":"v1"}]]]]]]]]]]]]]]]]]]]]]]]]] [[[[[[[[[[[[[[[[[[[[[[[[[{"s":"a","i":10}]]]]]]]]]]]]]]]]]]]]]]]]] 100
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java?rev=1659411&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java Thu Feb 12 21:36:04 2015
@@ -0,0 +1,246 @@
+package org.apache.hadoop.hive.serde2.lazy;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyObjectInspectorParameters;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.Text;
+import org.apache.hive.common.util.HiveStringUtils;
+
+/**
+ * SerDeParameters.
+ *
+ */
+public class LazySerDeParameters implements LazyObjectInspectorParameters {
+ public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};
+ public static final String SERIALIZATION_EXTEND_NESTING_LEVELS
+ = "hive.serialization.extend.nesting.levels";
+ public static final String SERIALIZATION_EXTEND_ADDITIONAL_NESTING_LEVELS
+ = "hive.serialization.extend.additional.nesting.levels";
+
+ private Properties tableProperties;
+ private String serdeName;
+
+ // The list of bytes used for the separators in the column (a nested struct
+ // such as Array<Array<int>> will use multiple separators).
+ // The list of separators + escapeChar are the bytes required to be escaped.
+ private byte[] separators;
+
+ private String nullString;
+ private Text nullSequence;
+ private TypeInfo rowTypeInfo;
+ private boolean lastColumnTakesRest;
+ private List<String> columnNames;
+ private List<TypeInfo> columnTypes;
+
+ private boolean escaped;
+ private byte escapeChar;
+ private boolean[] needsEscape = new boolean[256]; // A flag for each byte to indicate if escape is needed.
+
+ private boolean extendedBooleanLiteral;
+ List<String> timestampFormats;
+
+ public LazySerDeParameters(Configuration job, Properties tbl, String serdeName) throws SerDeException {
+ this.tableProperties = tbl;
+ this.serdeName = serdeName;
+
+ nullString = tbl.getProperty(
+ serdeConstants.SERIALIZATION_NULL_FORMAT, "\\N");
+ nullSequence = new Text(nullString);
+
+ String lastColumnTakesRestString = tbl
+ .getProperty(serdeConstants.SERIALIZATION_LAST_COLUMN_TAKES_REST);
+ lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString
+ .equalsIgnoreCase("true"));
+
+ extractColumnInfo();
+
+ // Create the LazyObject for storing the rows
+ rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
+
+ collectSeparators(tbl);
+
+ // Get the escape information
+ String escapeProperty = tbl.getProperty(serdeConstants.ESCAPE_CHAR);
+ escaped = (escapeProperty != null);
+ if (escaped) {
+ escapeChar = LazyUtils.getByte(escapeProperty, (byte) '\\');
+ needsEscape[escapeChar & 0xFF] = true; // Converts the negative byte into positive index
+ for (byte b : separators) {
+ needsEscape[b & 0xFF] = true; // Converts the negative byte into positive index
+ }
+ }
+
+ extendedBooleanLiteral = (job == null ? false :
+ job.getBoolean(ConfVars.HIVE_LAZYSIMPLE_EXTENDED_BOOLEAN_LITERAL.varname, false));
+
+ String[] timestampFormatsArray =
+ HiveStringUtils.splitAndUnEscape(tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS));
+ if (timestampFormatsArray != null) {
+ timestampFormats = Arrays.asList(timestampFormatsArray);
+ }
+ }
+
+ /**
+ * Extracts and set column names and column types from the table properties
+ * @throws SerDeException
+ */
+ public void extractColumnInfo() throws SerDeException {
+ // Read the configuration parameters
+ String columnNameProperty = tableProperties.getProperty(serdeConstants.LIST_COLUMNS);
+ // NOTE: if "columns.types" is missing, all columns will be of String type
+ String columnTypeProperty = tableProperties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+
+ // Parse the configuration parameters
+
+ if (columnNameProperty != null && columnNameProperty.length() > 0) {
+ columnNames = Arrays.asList(columnNameProperty.split(","));
+ } else {
+ columnNames = new ArrayList<String>();
+ }
+ if (columnTypeProperty == null) {
+ // Default type: all string
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < columnNames.size(); i++) {
+ if (i > 0) {
+ sb.append(":");
+ }
+ sb.append(serdeConstants.STRING_TYPE_NAME);
+ }
+ columnTypeProperty = sb.toString();
+ }
+
+ columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
+
+ if (columnNames.size() != columnTypes.size()) {
+ throw new SerDeException(serdeName + ": columns has " + columnNames.size()
+ + " elements while columns.types has " + columnTypes.size() + " elements!");
+ }
+ }
+
+ public List<TypeInfo> getColumnTypes() {
+ return columnTypes;
+ }
+
+ public List<String> getColumnNames() {
+ return columnNames;
+ }
+
+ public byte[] getSeparators() {
+ return separators;
+ }
+
+ public String getNullString() {
+ return nullString;
+ }
+
+ public Text getNullSequence() {
+ return nullSequence;
+ }
+
+ public TypeInfo getRowTypeInfo() {
+ return rowTypeInfo;
+ }
+
+ public boolean isLastColumnTakesRest() {
+ return lastColumnTakesRest;
+ }
+
+ public boolean isEscaped() {
+ return escaped;
+ }
+
+ public byte getEscapeChar() {
+ return escapeChar;
+ }
+
+ public boolean[] getNeedsEscape() {
+ return needsEscape;
+ }
+
+ public boolean isExtendedBooleanLiteral() {
+ return extendedBooleanLiteral;
+ }
+
+ public List<String> getTimestampFormats() {
+ return timestampFormats;
+ }
+
+ public void setSeparator(int index, byte separator) throws SerDeException {
+ if (index < 0 || index >= separators.length) {
+ throw new SerDeException("Invalid separator array index value: " + index);
+ }
+
+ separators[index] = separator;
+ }
+
+ /**
+ * To be backward-compatible, initialize the first 3 separators to
+ * be the given values from the table properties. The default number of separators is 8; if only
+ * hive.serialization.extend.nesting.levels is set, the number of
+ * separators is extended to 24; if hive.serialization.extend.additional.nesting.levels
+ * is set, the number of separators is extended to 154.
+ * @param tableProperties table properties to extract the user provided separators
+ */
+ private void collectSeparators(Properties tableProperties) {
+ List<Byte> separatorCandidates = new ArrayList<Byte>();
+
+ String extendNestingValue = tableProperties.getProperty(SERIALIZATION_EXTEND_NESTING_LEVELS);
+ String extendAdditionalNestingValue = tableProperties.getProperty(SERIALIZATION_EXTEND_ADDITIONAL_NESTING_LEVELS);
+ boolean extendedNesting = extendNestingValue != null && extendNestingValue.equalsIgnoreCase("true");
+ boolean extendedAdditionalNesting = extendAdditionalNestingValue != null
+ && extendAdditionalNestingValue.equalsIgnoreCase("true");
+
+ separatorCandidates.add(LazyUtils.getByte(tableProperties.getProperty(serdeConstants.FIELD_DELIM,
+ tableProperties.getProperty(serdeConstants.SERIALIZATION_FORMAT)), DefaultSeparators[0]));
+ separatorCandidates.add(LazyUtils.getByte(tableProperties
+ .getProperty(serdeConstants.COLLECTION_DELIM), DefaultSeparators[1]));
+ separatorCandidates.add(LazyUtils.getByte(
+ tableProperties.getProperty(serdeConstants.MAPKEY_DELIM), DefaultSeparators[2]));
+
+ //use only control chars that are very unlikely to be part of the string
+ // the following might/likely to be used in text files for strings
+ // 9 (horizontal tab, HT, \t, ^I)
+ // 10 (line feed, LF, \n, ^J),
+ // 12 (form feed, FF, \f, ^L),
+ // 13 (carriage return, CR, \r, ^M),
+ // 27 (escape, ESC, \e [GCC only], ^[).
+ for (byte b = 4; b <= 8; b++ ) {
+ separatorCandidates.add(b);
+ }
+ separatorCandidates.add((byte)11);
+ for (byte b = 14; b <= 26; b++ ) {
+ separatorCandidates.add(b);
+ }
+ for (byte b = 28; b <= 31; b++ ) {
+ separatorCandidates.add(b);
+ }
+
+ for (byte b = -128; b <= -1; b++ ) {
+ separatorCandidates.add(b);
+ }
+
+ int numSeparators = 8;
+ if(extendedAdditionalNesting) {
+ numSeparators = separatorCandidates.size();
+ } else if (extendedNesting) {
+ numSeparators = 24;
+ }
+
+ separators = new byte[numSeparators];
+ for (int i = 0; i < numSeparators; i++) {
+ separators[i] = separatorCandidates.get(i);
+ }
+ }
+}