You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sd...@apache.org on 2012/05/22 20:37:38 UTC
svn commit: r1341589 - in /hive/trunk: bin/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/scripts/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/metadata/
ql/src/test/queries/clientpositive/ ql/src/test/res...
Author: sdong
Date: Tue May 22 18:37:37 2012
New Revision: 1341589
URL: http://svn.apache.org/viewvc?rev=1341589&view=rev
Log:
HIVE-3030 escape more chars for script operator (Namit Jain via Siying Dong)
Added:
hive/trunk/data/scripts/doubleescapedtab.py
hive/trunk/data/scripts/escapedcarriagereturn.py
hive/trunk/data/scripts/escapednewline.py
hive/trunk/data/scripts/escapedtab.py
Modified:
hive/trunk/bin/hive
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/data/scripts/newline.py
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java
hive/trunk/ql/src/test/queries/clientpositive/newline.q
hive/trunk/ql/src/test/results/clientpositive/newline.q.out
Modified: hive/trunk/bin/hive
URL: http://svn.apache.org/viewvc/hive/trunk/bin/hive?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/bin/hive (original)
+++ hive/trunk/bin/hive Tue May 22 18:37:37 2012
@@ -216,6 +216,7 @@ if [ "$DEBUG" ]; then
else
get_debug_params "$DEBUG"
export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS $HIVE_MAIN_CLIENT_DEBUG_OPTS"
+ export HADOOP_OPTS="$HADOOP_OPTS $HADOOP_CLIENT_OPTS"
fi
fi
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue May 22 18:37:37 2012
@@ -389,7 +389,7 @@ public class HiveConf extends Configurat
"org.apache.hadoop.hive.ql.exec.TextRecordReader"),
HIVESCRIPTRECORDWRITER("hive.script.recordwriter",
"org.apache.hadoop.hive.ql.exec.TextRecordWriter"),
- HIVESCRIPTESCAPENEWLINES("hive.script.escape.newlines", false),
+ HIVESCRIPTESCAPE("hive.transform.escape.input", false),
// HWI
HIVEHWILISTENHOST("hive.hwi.listen.host", "0.0.0.0"),
Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Tue May 22 18:37:37 2012
@@ -1273,12 +1273,12 @@
</property>
<property>
- <name>hive.script.escape.newlines</name>
+ <name>hive.transform.escape.input</name>
<value>false</value>
<description>
- This adds an option to escape the newlines when they are passed to the
- user script. This is useful is the hive tables can contain data that
- can contain newlines.
+ This adds an option to escape special chars (newlines, carriage returns and
+ tabs) when they are passed to the user script. This is useful if the hive tables
+ can contain data that contains special characters.
</description>
</property>
Added: hive/trunk/data/scripts/doubleescapedtab.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/doubleescapedtab.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/doubleescapedtab.py (added)
+++ hive/trunk/data/scripts/doubleescapedtab.py Tue May 22 18:37:37 2012
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+ print "1\\\\\\t2"
+ print "1\\\\\\\\t2"
+
Added: hive/trunk/data/scripts/escapedcarriagereturn.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/escapedcarriagereturn.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/escapedcarriagereturn.py (added)
+++ hive/trunk/data/scripts/escapedcarriagereturn.py Tue May 22 18:37:37 2012
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+ print "1\\\\r2"
+
Added: hive/trunk/data/scripts/escapednewline.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/escapednewline.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/escapednewline.py (added)
+++ hive/trunk/data/scripts/escapednewline.py Tue May 22 18:37:37 2012
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+ print "1\\\\n2"
+
Added: hive/trunk/data/scripts/escapedtab.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/escapedtab.py?rev=1341589&view=auto
==============================================================================
--- hive/trunk/data/scripts/escapedtab.py (added)
+++ hive/trunk/data/scripts/escapedtab.py Tue May 22 18:37:37 2012
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import sys
+
+for line in sys.stdin:
+ print "1\\\\t2"
+
Modified: hive/trunk/data/scripts/newline.py
URL: http://svn.apache.org/viewvc/hive/trunk/data/scripts/newline.py?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/data/scripts/newline.py (original)
+++ hive/trunk/data/scripts/newline.py Tue May 22 18:37:37 2012
@@ -20,3 +20,5 @@ import sys
for line in sys.stdin:
print "1\\n2"
+ print "1\\r2"
+ print "1\\t2"
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java Tue May 22 18:37:37 2012
@@ -60,8 +60,8 @@ public class TextRecordReader implements
int bytesConsumed = lineReader.readLine((Text) row);
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPENEWLINES)) {
- return HiveUtils.unescapeNewLine((Text) row);
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPE)) {
+ return HiveUtils.unescapeText((Text) row);
}
return bytesConsumed;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordWriter.java Tue May 22 18:37:37 2012
@@ -46,8 +46,8 @@ public class TextRecordWriter implements
Text text = (Text) row;
Text escapeText = text;
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPENEWLINES)) {
- escapeText = HiveUtils.escapeNewLine(text);
+ if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVESCRIPTESCAPE)) {
+ escapeText = HiveUtils.escapeText(text);
}
out.write(escapeText.getBytes(), 0, escapeText.getLength());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveUtils.java Tue May 22 18:37:37 2012
@@ -97,10 +97,16 @@ public final class HiveUtils {
return (escape.toString());
}
+ static final byte[] escapeEscapeBytes = "\\\\".getBytes();;
+ static final byte[] escapeUnescapeBytes = "\\".getBytes();
static final byte[] newLineEscapeBytes = "\\n".getBytes();;
static final byte[] newLineUnescapeBytes = "\n".getBytes();
+ static final byte[] carriageReturnEscapeBytes = "\\r".getBytes();;
+ static final byte[] carriageReturnUnescapeBytes = "\r".getBytes();
+ static final byte[] tabEscapeBytes = "\\t".getBytes();;
+ static final byte[] tabUnescapeBytes = "\t".getBytes();
- public static Text escapeNewLine(Text text) {
+ public static Text escapeText(Text text) {
int length = text.getLength();
byte[] textBytes = text.getBytes();
@@ -109,20 +115,50 @@ public final class HiveUtils {
for (int i = 0; i < length; ++i) {
int c = text.charAt(i);
+ byte[] escaped;
+ int start;
+ int len;
+
switch (c) {
+
+ case '\\':
+ escaped = escapeEscapeBytes;
+ start = 0;
+ len = escaped.length;
+ break;
+
case '\n':
- byte[] escaped = newLineEscapeBytes;
- escape.append(escaped, 0, escaped.length);
+ escaped = newLineEscapeBytes;
+ start = 0;
+ len = escaped.length;
break;
+
+ case '\r':
+ escaped = carriageReturnEscapeBytes;
+ start = 0;
+ len = escaped.length;
+ break;
+
+ case '\t':
+ escaped = tabEscapeBytes;
+ start = 0;
+ len = escaped.length;
+ break;
+
default:
- escape.append(textBytes, i, 1);
+ escaped = textBytes;
+ start = i;
+ len = 1;
break;
}
+
+ escape.append(escaped, start, len);
+
}
return escape;
}
- public static int unescapeNewLine(Text text) {
+ public static int unescapeText(Text text) {
Text escape = new Text(text);
text.clear();
@@ -136,8 +172,11 @@ public final class HiveUtils {
case '\\':
if (hadSlash) {
text.append(textBytes, i, 1);
+ hadSlash = false;
+ }
+ else {
+ hadSlash = true;
}
- hadSlash = true;
break;
case 'n':
if (hadSlash) {
@@ -149,6 +188,28 @@ public final class HiveUtils {
}
hadSlash = false;
break;
+ case 'r':
+ if (hadSlash) {
+ byte[] carriageReturn = carriageReturnUnescapeBytes;
+ text.append(carriageReturn, 0, carriageReturn.length);
+ }
+ else {
+ text.append(textBytes, i, 1);
+ }
+ hadSlash = false;
+ break;
+
+ case 't':
+ if (hadSlash) {
+ byte[] tab = tabUnescapeBytes;
+ text.append(tab, 0, tab.length);
+ }
+ else {
+ text.append(textBytes, i, 1);
+ }
+ hadSlash = false;
+ break;
+
default:
if (hadSlash) {
text.append(textBytes, i-1, 1);
Modified: hive/trunk/ql/src/test/queries/clientpositive/newline.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/newline.q?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/newline.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/newline.q Tue May 22 18:37:37 2012
@@ -1,9 +1,53 @@
add file ../data/scripts/newline.py;
-set hive.script.escape.newlines=true;
+set hive.transform.escape.input=true;
create table tmp_tmp(key string, value string) stored as rcfile;
insert overwrite table tmp_tmp
SELECT TRANSFORM(key, value) USING
-'python newline.py' AS key, value FROM src limit 5;
+'python newline.py' AS key, value FROM src limit 6;
-select * from tmp_tmp;
\ No newline at end of file
+select * from tmp_tmp;
+
+drop table tmp_tmp;
+
+add file ../data/scripts/escapednewline.py;
+add file ../data/scripts/escapedtab.py;
+add file ../data/scripts/doubleescapedtab.py;
+add file ../data/scripts/escapedcarriagereturn.py;
+
+create table tmp_tmp(key string, value string) stored as rcfile;
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapednewline.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
+
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedcarriagereturn.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
+
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedtab.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
+
+insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python doubleescapedtab.py' AS key, value FROM src limit 5;
+
+select * from tmp_tmp;
+
+SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp;
Modified: hive/trunk/ql/src/test/results/clientpositive/newline.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/newline.q.out?rev=1341589&r1=1341588&r2=1341589&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/newline.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/newline.q.out Tue May 22 18:37:37 2012
@@ -5,13 +5,13 @@ POSTHOOK: type: CREATETABLE
POSTHOOK: Output: default@tmp_tmp
PREHOOK: query: insert overwrite table tmp_tmp
SELECT TRANSFORM(key, value) USING
-'python newline.py' AS key, value FROM src limit 5
+'python newline.py' AS key, value FROM src limit 6
PREHOOK: type: QUERY
PREHOOK: Input: default@src
PREHOOK: Output: default@tmp_tmp
POSTHOOK: query: insert overwrite table tmp_tmp
SELECT TRANSFORM(key, value) USING
-'python newline.py' AS key, value FROM src limit 5
+'python newline.py' AS key, value FROM src limit 6
POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@tmp_tmp
@@ -31,9 +31,270 @@ POSTHOOK: Lineage: tmp_tmp.value SCRIPT
2 NULL
1
2 NULL
+1 2
1
2 NULL
1
2 NULL
-1
-2 NULL
+1 2
+PREHOOK: query: drop table tmp_tmp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@tmp_tmp
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: drop table tmp_tmp
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@tmp_tmp
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: create table tmp_tmp(key string, value string) stored as rcfile
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table tmp_tmp(key string, value string) stored as rcfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapednewline.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapednewline.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\n2 NULL
+1\n2 NULL
+1\n2 NULL
+1\n2 NULL
+1\n2 NULL
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\n2 NULL
+1\n2 NULL
+1\n2 NULL
+1\n2 NULL
+1\n2 NULL
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedcarriagereturn.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedcarriagereturn.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\r2 NULL
+1\r2 NULL
+1\r2 NULL
+1\r2 NULL
+1\r2 NULL
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\r2 NULL
+1\r2 NULL
+1\r2 NULL
+1\r2 NULL
+1\r2 NULL
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedtab.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python escapedtab.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\t2 NULL
+1\t2 NULL
+1\t2 NULL
+1\t2 NULL
+1\t2 NULL
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\t2 NULL
+1\t2 NULL
+1\t2 NULL
+1\t2 NULL
+1\t2 NULL
+PREHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python doubleescapedtab.py' AS key, value FROM src limit 5
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@tmp_tmp
+POSTHOOK: query: insert overwrite table tmp_tmp
+SELECT TRANSFORM(key, value) USING
+'python doubleescapedtab.py' AS key, value FROM src limit 5
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@tmp_tmp
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: select * from tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\ 2
+1\\t2 NULL
+1\ 2
+1\\t2 NULL
+1\ 2
+PREHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT TRANSFORM(key, value) USING
+'/bin/cat' AS (key, value) FROM tmp_tmp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tmp_tmp
+#### A masked pattern was here ####
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.key SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: tmp_tmp.value SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
+1\ 2
+1\\t2 NULL
+1\ 2
+1\\t2 NULL
+1\ 2