You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/09 09:10:22 UTC
svn commit: r1480540 - in /hive/branches/HIVE-4115: ./ data/files/
ql/src/java/org/apache/hadoop/hive/ql/history/
ql/src/java/org/apache/hadoop/hive/ql/io/orc/
ql/src/java/org/apache/hadoop/hive/ql/session/
ql/src/test/queries/clientpositive/ ql/src/te...
Author: amareshwari
Date: Thu May 9 07:10:21 2013
New Revision: 1480540
URL: http://svn.apache.org/r1480540
Log:
Merging r1480181 through r1480539 into HIVE-4115
Added:
hive/branches/HIVE-4115/data/files/orc_create.txt
- copied unchanged from r1480539, hive/trunk/data/files/orc_create.txt
Modified:
hive/branches/HIVE-4115/ (props changed)
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/orc_create.q
hive/branches/HIVE-4115/ql/src/test/results/clientpositive/orc_create.q.out
hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/Operation.java
hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Propchange: hive/branches/HIVE-4115/
------------------------------------------------------------------------------
Merged /hive/branches/branch-0.11:r1480385,1480458
Merged /hive/trunk:r1480181-1480539
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/history/HiveHistory.java Thu May 9 07:10:21 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.hive.ql.QueryPl
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapred.Counters;
import org.apache.hadoop.mapred.Counters.Counter;
import org.apache.hadoop.mapred.Counters.Group;
@@ -532,11 +533,14 @@ public class HiveHistory {
return null;
}
+
+ public void closeStream() {
+ IOUtils.cleanup(LOG, histStream);
+ }
+
@Override
public void finalize() throws Throwable {
- if (histStream !=null){
- histStream.close();
- }
+ closeStream();
super.finalize();
}
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcStruct.java Thu May 9 07:10:21 2013
@@ -21,12 +21,16 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -58,8 +62,18 @@ final class OrcStruct implements Writabl
return fields.length;
}
+ /**
+ * Change the number of fields in the struct. No effect if the number of
+ * fields is the same. The old field values are copied to the new array.
+ * @param numFields the new number of fields
+ */
public void setNumFields(int numFields) {
- fields = new Object[numFields];
+ if (fields.length != numFields) {
+ Object[] oldFields = fields;
+ fields = new Object[numFields];
+ System.arraycopy(oldFields, 0, fields, 0,
+ Math.min(oldFields.length, numFields));
+ }
}
@Override
@@ -148,7 +162,7 @@ final class OrcStruct implements Writabl
}
}
- static class OrcStructInspector extends StructObjectInspector {
+ static class OrcStructInspector extends SettableStructObjectInspector {
private final List<StructField> fields;
OrcStructInspector(StructTypeInfo info) {
@@ -223,9 +237,52 @@ final class OrcStruct implements Writabl
public Category getCategory() {
return Category.STRUCT;
}
+
+ @Override
+ public Object create() {
+ return new OrcStruct(0);
+ }
+
+ @Override
+ public Object setStructFieldData(Object struct, StructField field,
+ Object fieldValue) {
+ OrcStruct orcStruct = (OrcStruct) struct;
+ int offset = ((Field) field).offset;
+ // if the offset is bigger than our current number of fields, grow it
+ if (orcStruct.getNumFields() <= offset) {
+ orcStruct.setNumFields(offset+1);
+ }
+ orcStruct.setFieldValue(offset, fieldValue);
+ return struct;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || o.getClass() != getClass()) {
+ return false;
+ } else if (o == this) {
+ return true;
+ } else {
+ List<StructField> other = ((OrcStructInspector) o).fields;
+ if (other.size() != fields.size()) {
+ return false;
+ }
+ for(int i = 0; i < fields.size(); ++i) {
+ StructField left = other.get(i);
+ StructField right = fields.get(i);
+ if (!(left.getFieldName().equals(right.getFieldName()) &&
+ left.getFieldObjectInspector().equals
+ (right.getFieldObjectInspector()))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
}
- static class OrcMapObjectInspector implements MapObjectInspector {
+ static class OrcMapObjectInspector
+ implements MapObjectInspector, SettableMapObjectInspector {
private final ObjectInspector key;
private final ObjectInspector value;
@@ -275,9 +332,45 @@ final class OrcStruct implements Writabl
public Category getCategory() {
return Category.MAP;
}
+
+ @Override
+ public Object create() {
+ return new HashMap<Object,Object>();
+ }
+
+ @Override
+ public Object put(Object map, Object key, Object value) {
+ ((Map) map).put(key, value);
+ return map;
+ }
+
+ @Override
+ public Object remove(Object map, Object key) {
+ ((Map) map).remove(key);
+ return map;
+ }
+
+ @Override
+ public Object clear(Object map) {
+ ((Map) map).clear();
+ return map;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || o.getClass() != getClass()) {
+ return false;
+ } else if (o == this) {
+ return true;
+ } else {
+ OrcMapObjectInspector other = (OrcMapObjectInspector) o;
+ return other.key.equals(key) && other.value.equals(value);
+ }
+ }
}
- static class OrcListObjectInspector implements ListObjectInspector {
+ static class OrcListObjectInspector
+ implements ListObjectInspector, SettableListObjectInspector {
private final ObjectInspector child;
OrcListObjectInspector(ListTypeInfo info) {
@@ -319,6 +412,43 @@ final class OrcStruct implements Writabl
public Category getCategory() {
return Category.LIST;
}
+
+ @Override
+ public Object create(int size) {
+ ArrayList<Object> result = new ArrayList<Object>(size);
+ for(int i = 0; i < size; ++i) {
+ result.add(null);
+ }
+ return result;
+ }
+
+ @Override
+ public Object set(Object list, int index, Object element) {
+ List l = (List) list;
+ for(int i=l.size(); i < index+1; ++i) {
+ l.add(null);
+ }
+ l.set(index, element);
+ return list;
+ }
+
+ @Override
+ public Object resize(Object list, int newSize) {
+ ((ArrayList) list).ensureCapacity(newSize);
+ return list;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || o.getClass() != getClass()) {
+ return false;
+ } else if (o == this) {
+ return true;
+ } else {
+ ObjectInspector other = ((OrcListObjectInspector) o).child;
+ return other.equals(child);
+ }
+ }
}
static ObjectInspector createObjectInspector(TypeInfo info) {
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUnion.java Thu May 9 07:10:21 2013
@@ -134,5 +134,25 @@ final class OrcUnion implements UnionObj
public Category getCategory() {
return Category.UNION;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || o.getClass() != getClass()) {
+ return false;
+ } else if (o == this) {
+ return true;
+ } else {
+ List<ObjectInspector> other = ((OrcUnionObjectInspector) o).children;
+ if (other.size() != children.size()) {
+ return false;
+ }
+ for(int i = 0; i < children.size(); ++i) {
+ if (!other.get(i).equals(children.get(i))) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
}
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Thu May 9 07:10:21 2013
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
-import java.net.URL;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.GregorianCalendar;
@@ -41,7 +40,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.MapRedStats;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
Modified: hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/orc_create.q?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/orc_create.q Thu May 9 07:10:21 2013
@@ -1,4 +1,18 @@
DROP TABLE orc_create;
+DROP TABLE orc_create_complex;
+DROP TABLE orc_create_staging;
+
+CREATE TABLE orc_create_staging (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+ COLLECTION ITEMS TERMINATED BY ','
+ MAP KEYS TERMINATED BY ':';
+
+DESCRIBE FORMATTED orc_create_staging;
CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string)
@@ -26,4 +40,27 @@ CREATE TABLE orc_create (key INT, value
DESCRIBE FORMATTED orc_create;
+CREATE TABLE orc_create_complex (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC;
+
+DESCRIBE FORMATTED orc_create_complex;
+
+LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging;
+
+SELECT * from orc_create_staging;
+
+INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging;
+
+SELECT * from orc_create_complex;
+SELECT str from orc_create_complex;
+SELECT mp from orc_create_complex;
+SELECT lst from orc_create_complex;
+SELECT strct from orc_create_complex;
+
DROP TABLE orc_create;
+DROP TABLE orc_create_complex;
+DROP TABLE orc_create_staging;
Modified: hive/branches/HIVE-4115/ql/src/test/results/clientpositive/orc_create.q.out
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/test/results/clientpositive/orc_create.q.out?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/test/results/clientpositive/orc_create.q.out (original)
+++ hive/branches/HIVE-4115/ql/src/test/results/clientpositive/orc_create.q.out Thu May 9 07:10:21 2013
@@ -2,6 +2,69 @@ PREHOOK: query: DROP TABLE orc_create
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE orc_create
POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_complex
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_complex
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_staging
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_staging
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE orc_create_staging (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+ COLLECTION ITEMS TERMINATED BY ','
+ MAP KEYS TERMINATED BY ':'
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_staging (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) ROW FORMAT DELIMITED
+ FIELDS TERMINATED BY '|'
+ COLLECTION ITEMS TERMINATED BY ','
+ MAP KEYS TERMINATED BY ':'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_staging
+PREHOOK: query: DESCRIBE FORMATTED orc_create_staging
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED orc_create_staging
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+str string None
+mp map<string,string> None
+lst array<string> None
+strct struct<A:string,B:string> None
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+InputFormat: org.apache.hadoop.mapred.TextInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ colelction.delim ,
+ field.delim |
+ mapkey.delim :
+ serialization.format |
PREHOOK: query: CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string)
STORED AS ORC
@@ -185,6 +248,156 @@ Bucket Columns: []
Sort Columns: []
Storage Desc Params:
serialization.format 1
+PREHOOK: query: CREATE TABLE orc_create_complex (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_complex (
+ str STRING,
+ mp MAP<STRING,STRING>,
+ lst ARRAY<STRING>,
+ strct STRUCT<A:STRING,B:STRING>
+) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_complex
+PREHOOK: query: DESCRIBE FORMATTED orc_create_complex
+PREHOOK: type: DESCTABLE
+POSTHOOK: query: DESCRIBE FORMATTED orc_create_complex
+POSTHOOK: type: DESCTABLE
+# col_name data_type comment
+
+str string from deserializer
+mp map<string,string> from deserializer
+lst array<string> from deserializer
+strct struct<A:string,B:string> from deserializer
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Protect Mode: None
+Retention: 0
+#### A masked pattern was here ####
+Table Type: MANAGED_TABLE
+Table Parameters:
+#### A masked pattern was here ####
+
+# Storage Information
+SerDe Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+Compressed: No
+Num Buckets: -1
+Bucket Columns: []
+Sort Columns: []
+Storage Desc Params:
+ serialization.format 1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging
+PREHOOK: type: LOAD
+PREHOOK: Output: default@orc_create_staging
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create.txt' OVERWRITE INTO TABLE orc_create_staging
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@orc_create_staging
+PREHOOK: query: SELECT * from orc_create_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_staging
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_staging
+#### A masked pattern was here ####
+line1 {"key11":"value11","key12":"value12","key13":"value13"} ["a","b","c"] {"a":"one","b":"two"}
+line2 {"key21":"value21","key22":"value22","key23":"value23"} ["d","e","f"] {"a":"three","b":"four"}
+line3 {"key31":"value31","key32":"value32","key33":"value33"} ["g","h","i"] {"a":"five","b":"six"}
+PREHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_staging
+PREHOOK: Output: default@orc_create_complex
+POSTHOOK: query: INSERT OVERWRITE TABLE orc_create_complex SELECT * FROM orc_create_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_staging
+POSTHOOK: Output: default@orc_create_complex
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: SELECT * from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+line1 {"key12":"value12","key11":"value11","key13":"value13"} ["a","b","c"] {"A":"one","B":"two"}
+line2 {"key21":"value21","key23":"value23","key22":"value22"} ["d","e","f"] {"A":"three","B":"four"}
+line3 {"key33":"value33","key31":"value31","key32":"value32"} ["g","h","i"] {"A":"five","B":"six"}
+PREHOOK: query: SELECT str from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT str from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+line1
+line2
+line3
+PREHOOK: query: SELECT mp from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT mp from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+{"key12":"value12","key11":"value11","key13":"value13"}
+{"key21":"value21","key23":"value23","key22":"value22"}
+{"key33":"value33","key31":"value31","key32":"value32"}
+PREHOOK: query: SELECT lst from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT lst from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+["a","b","c"]
+["d","e","f"]
+["g","h","i"]
+PREHOOK: query: SELECT strct from orc_create_complex
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT strct from orc_create_complex
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_complex
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+{"a":"one","b":"two"}
+{"a":"three","b":"four"}
+{"a":"five","b":"six"}
PREHOOK: query: DROP TABLE orc_create
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@orc_create
@@ -193,3 +406,31 @@ POSTHOOK: query: DROP TABLE orc_create
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@orc_create
POSTHOOK: Output: default@orc_create
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_complex
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_complex
+PREHOOK: Output: default@orc_create_complex
+POSTHOOK: query: DROP TABLE orc_create_complex
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_complex
+POSTHOOK: Output: default@orc_create_complex
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_staging
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_staging
+PREHOOK: Output: default@orc_create_staging
+POSTHOOK: query: DROP TABLE orc_create_staging
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_staging
+POSTHOOK: Output: default@orc_create_staging
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
Modified: hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java (original)
+++ hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java Thu May 9 07:10:21 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.ql.processors.CommandProcessor;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hive.service.cli.FetchOrientation;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.OperationState;
@@ -85,6 +86,12 @@ public abstract class HiveCommandOperati
}
}
+
+ private void tearDownSessionIO() {
+ IOUtils.cleanup(LOG, parentSession.getSessionState().out);
+ IOUtils.cleanup(LOG, parentSession.getSessionState().err);
+ }
+
/* (non-Javadoc)
* @see org.apache.hive.service.cli.operation.Operation#run()
*/
@@ -121,6 +128,7 @@ public abstract class HiveCommandOperati
@Override
public void close() throws HiveSQLException {
setState(OperationState.CLOSED);
+ tearDownSessionIO();
cleanTmpFile();
}
@@ -187,6 +195,7 @@ public abstract class HiveCommandOperati
if (resultReader != null) {
SessionState sessionState = getParentSession().getSessionState();
File tmp = sessionState.getTmpOutputFile();
+ IOUtils.cleanup(LOG, resultReader);
tmp.delete();
resultReader = null;
}
Modified: hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/Operation.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/Operation.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/Operation.java (original)
+++ hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/operation/Operation.java Thu May 9 07:10:21 2013
@@ -32,7 +32,7 @@ import org.apache.hive.service.cli.sessi
public abstract class Operation {
- private final HiveSession parentSession;
+ protected final HiveSession parentSession;
private OperationState state = OperationState.INITIALIZED;
private final OperationHandle opHandle;
private HiveConf configuration;
Modified: hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1480540&r1=1480539&r2=1480540&view=diff
==============================================================================
--- hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Thu May 9 07:10:21 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.ql.history.HiveHistory;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hive.common.util.HiveVersionInfo;
import org.apache.hive.service.cli.FetchOrientation;
@@ -295,6 +296,10 @@ public class HiveSessionImpl implements
operationManager.closeOperation(opHandle);
}
opHandleSet.clear();
+ HiveHistory hiveHist = sessionState.getHiveHistory();
+ if (null != hiveHist) {
+ hiveHist.closeStream();
+ }
} finally {
release();
}
@@ -367,4 +372,4 @@ public class HiveSessionImpl implements
protected HiveSession getSession() {
return this;
}
-}
\ No newline at end of file
+}