You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2010/01/01 05:19:35 UTC
svn commit: r894971 - in /hadoop/hive/trunk: ./
contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/
contrib/src/test/queries/clientpositive/
contrib/src/test/results/clientpositive/
ql/src/java/org/apache/hadoop/hive/ql/exec/
Author: heyongqiang
Date: Fri Jan 1 04:19:34 2010
New Revision: 894971
URL: http://svn.apache.org/viewvc?rev=894971&view=rev
Log:
HIVE-1023 typedbytes: datatypes should be derived from data
Added:
hadoop/hive/trunk/contrib/src/test/queries/clientpositive/serde_typedbytes3.q
hadoop/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RecordReader.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=894971&r1=894970&r2=894971&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Fri Jan 1 04:19:34 2010
@@ -155,6 +155,9 @@
HIVE-963. No out of memory errors for skew join
(Ning Zhang via namit)
+ HIVE-1023. typedbytes: datatypes should be derived from data
+ (namit via He Yongqiang)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java?rev=894971&r1=894970&r2=894971&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java (original)
+++ hadoop/hive/trunk/contrib/src/java/org/apache/hadoop/hive/contrib/util/typedbytes/TypedBytesRecordReader.java Fri Jan 1 04:19:34 2010
@@ -22,6 +22,11 @@
import java.io.InputStream;
import java.io.DataInputStream;
import java.util.ArrayList;
+import java.util.List;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.Map;
+import java.util.HashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
@@ -36,6 +41,13 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.hive.ql.io.NonSyncDataOutputBuffer;
import org.apache.hadoop.hive.ql.exec.RecordReader;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveTypeEntry;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public class TypedBytesRecordReader implements RecordReader {
@@ -43,13 +55,38 @@
private TypedBytesWritableInput tbIn;
NonSyncDataOutputBuffer barrStr = new NonSyncDataOutputBuffer();
- TypedBytesWritableOutput tbOut = new TypedBytesWritableOutput(barrStr);
+ TypedBytesWritableOutput tbOut;
ArrayList<Writable> row = new ArrayList<Writable>(0);
+ ArrayList<String> rowTypeName = new ArrayList<String>(0);
+ List<String> columnTypes;
- public void initialize(InputStream in, Configuration conf) throws IOException {
+ ArrayList<ObjectInspector> srcOIns = new ArrayList<ObjectInspector>();
+ ArrayList<ObjectInspector> dstOIns = new ArrayList<ObjectInspector>();
+ ArrayList<Converter> converters = new ArrayList<Converter>();
+
+ static private Map<Type, String> typedBytesToTypeName = new HashMap<Type, String>();
+ static {
+ typedBytesToTypeName.put(getType(1), Constants.TINYINT_TYPE_NAME);
+ typedBytesToTypeName.put(getType(2), Constants.BOOLEAN_TYPE_NAME);
+ typedBytesToTypeName.put(getType(3), Constants.INT_TYPE_NAME);
+ typedBytesToTypeName.put(getType(4), Constants.BIGINT_TYPE_NAME);
+ typedBytesToTypeName.put(getType(5), Constants.FLOAT_TYPE_NAME);
+ typedBytesToTypeName.put(getType(6), Constants.DOUBLE_TYPE_NAME);
+ typedBytesToTypeName.put(getType(7), Constants.STRING_TYPE_NAME);
+ typedBytesToTypeName.put(getType(11), Constants.SMALLINT_TYPE_NAME);
+ }
+
+ public void initialize(InputStream in, Configuration conf, Properties tbl) throws IOException {
din = new DataInputStream(in);
tbIn = new TypedBytesWritableInput(din);
+ tbOut = new TypedBytesWritableOutput(barrStr);
+ String columnTypeProperty = tbl.getProperty(Constants.LIST_COLUMN_TYPES);
+ columnTypes = Arrays.asList(columnTypeProperty.split(","));
+ for (String columnType:columnTypes) {
+ PrimitiveTypeEntry dstTypeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(columnType);
+ dstOIns.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(dstTypeEntry.primitiveCategory));
+ }
}
public Writable createRow() throws IOException {
@@ -80,90 +117,121 @@
}
return null;
}
-
+
public int next(Writable data) throws IOException {
int pos = 0;
barrStr.reset();
while (true) {
Type type = tbIn.readTypeCode();
-
+
// it was a empty stream
if (type == null)
return -1;
-
+
if (type == Type.ENDOFRECORD) {
tbOut.writeEndOfRecord();
if (barrStr.getLength() > 0)
((BytesWritable)data).set(barrStr.getData(), 0, barrStr.getLength());
return barrStr.getLength();
}
-
+
if (pos >= row.size()) {
Writable wrt = allocateWritable(type);
assert pos == row.size();
+ assert pos == rowTypeName.size();
row.add(wrt);
+ rowTypeName.add(type.name());
+ String typeName = typedBytesToTypeName.get(type);
+ PrimitiveTypeEntry srcTypeEntry = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName);
+ srcOIns.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(srcTypeEntry.primitiveCategory));
+ converters.add(ObjectInspectorConverters.getConverter(srcOIns.get(pos), dstOIns.get(pos)));
+ }
+ else {
+ if (!rowTypeName.get(pos).equals(type.name()))
+ throw new RuntimeException("datatype of row changed from " +
+ rowTypeName.get(pos) + " to " + type.name());
}
-
+
+ Writable w = row.get(pos);
switch (type) {
case BYTE: {
- ByteWritable bw = (ByteWritable)row.get(pos);
- tbIn.readByte(bw);
- tbOut.writeByte(bw);
+ tbIn.readByte((ByteWritable)w);
break;
}
case BOOL: {
- BooleanWritable bw = (BooleanWritable)row.get(pos);
- tbIn.readBoolean(bw);
- tbOut.writeBoolean(bw);
+ tbIn.readBoolean((BooleanWritable)w);
break;
}
case INT: {
- IntWritable iw = (IntWritable)row.get(pos);
- tbIn.readInt(iw);
- tbOut.writeInt(iw);
+ tbIn.readInt((IntWritable)w);
break;
}
case SHORT: {
- ShortWritable sw = (ShortWritable)row.get(pos);
- tbIn.readShort(sw);
- tbOut.writeShort(sw);
+ tbIn.readShort((ShortWritable)w);
break;
- }
+ }
case LONG: {
- LongWritable lw = (LongWritable)row.get(pos);
- tbIn.readLong(lw);
- tbOut.writeLong(lw);
+ tbIn.readLong((LongWritable)w);
break;
}
case FLOAT: {
- FloatWritable fw = (FloatWritable)row.get(pos);
- tbIn.readFloat(fw);
- tbOut.writeFloat(fw);
+ tbIn.readFloat((FloatWritable)w);
break;
}
case DOUBLE: {
- DoubleWritable dw = (DoubleWritable)row.get(pos);
- tbIn.readDouble(dw);
- tbOut.writeDouble(dw);
+ tbIn.readDouble((DoubleWritable)w);
break;
}
case STRING: {
- Text txt = (Text)row.get(pos);
- tbIn.readText(txt);
- tbOut.writeText(txt);
+ tbIn.readText((Text)w);
break;
}
default:
assert false; // should never come here
}
-
+
+ write(pos, w);
pos++;
}
}
-
+
+ private void write(int pos, Writable inpw) throws IOException {
+ String typ = columnTypes.get(pos);
+
+ Writable w = (Writable)converters.get(pos).convert(inpw);
+
+ if (typ.equalsIgnoreCase(Constants.BOOLEAN_TYPE_NAME))
+ tbOut.writeBoolean((BooleanWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.TINYINT_TYPE_NAME))
+ tbOut.writeByte((ByteWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.SMALLINT_TYPE_NAME))
+ tbOut.writeShort((ShortWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.INT_TYPE_NAME))
+ tbOut.writeInt((IntWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.BIGINT_TYPE_NAME))
+ tbOut.writeLong((LongWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.FLOAT_TYPE_NAME))
+ tbOut.writeFloat((FloatWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.DOUBLE_TYPE_NAME))
+ tbOut.writeDouble((DoubleWritable)w);
+ else if (typ.equalsIgnoreCase(Constants.STRING_TYPE_NAME))
+ tbOut.writeText((Text)w);
+ else
+ assert false;
+ }
+
public void close() throws IOException {
if (din != null)
din.close();
}
+
+ static public Type getType(int code) {
+ for (Type type : Type.values()) {
+ if (type.code == code) {
+ return type;
+ }
+ }
+ return null;
+ }
}
Added: hadoop/hive/trunk/contrib/src/test/queries/clientpositive/serde_typedbytes3.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/src/test/queries/clientpositive/serde_typedbytes3.q?rev=894971&view=auto
==============================================================================
--- hadoop/hive/trunk/contrib/src/test/queries/clientpositive/serde_typedbytes3.q (added)
+++ hadoop/hive/trunk/contrib/src/test/queries/clientpositive/serde_typedbytes3.q Fri Jan 1 04:19:34 2010
@@ -0,0 +1,29 @@
+add jar ../build/contrib/hive_contrib.jar;
+
+drop table dest1;
+CREATE TABLE dest1(key STRING, value STRING) STORED AS TEXTFILE;
+
+EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+FROM (
+ FROM src
+ SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue;
+
+SELECT dest1.* FROM dest1;
+
+drop table dest1;
Added: hadoop/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out?rev=894971&view=auto
==============================================================================
--- hadoop/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out (added)
+++ hadoop/hive/trunk/contrib/src/test/results/clientpositive/serde_typedbytes3.q.out Fri Jan 1 04:19:34 2010
@@ -0,0 +1,654 @@
+PREHOOK: query: drop table dest1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table dest1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE dest1(key STRING, value STRING) STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE dest1(key STRING, value STRING) STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@dest1
+PREHOOK: query: EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+FROM (
+ FROM src
+ SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TRANSFORM (TOK_EXPLIST (TOK_FUNCTION TOK_SMALLINT (. (TOK_TABLE_OR_COL src) key)) (. (TOK_TABLE_OR_COL src) value)) (TOK_SERDE (TOK_SERDENAME 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe')) (TOK_RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter') '/bin/cat' (TOK_SERDE (TOK_SERDENAME 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe')) (TOK_RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader') (TOK_ALIASLIST tkey tvalue)))))) tmap)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB dest1)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL tkey)) (TOK_SELEXPR (TOK_TABLE_OR_COL tvalue)))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-4 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ tmap:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: UDFToShort(key)
+ type: smallint
+ expr: value
+ type: string
+ outputColumnNames: _col0, _col1
+ Transform Operator
+ command: /bin/cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-4
+ Conditional Operator
+ list of dependent Tasks:
+ Move Operator
+ files:
+ hdfs directory: true
+ destination: file:/data/users/njain/hive1/hive1/build/ql/tmp/1224635/10000
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/data/users/njain/hive1/hive1/build/ql/tmp/4847210/10002
+ Reduce Output Operator
+ sort order:
+ Map-reduce partition columns:
+ expr: rand()
+ type: double
+ tag: -1
+ value expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ Reduce Operator Tree:
+ Extract
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: dest1
+
+
+PREHOOK: query: FROM (
+ FROM src
+ SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM (
+ FROM src
+ SELECT TRANSFORM(cast(src.key as smallint), src.value) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDWRITER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordWriter'
+ USING '/bin/cat'
+ AS (tkey, tvalue) ROW FORMAT SERDE 'org.apache.hadoop.hive.contrib.serde2.TypedBytesSerDe'
+ RECORDREADER 'org.apache.hadoop.hive.contrib.util.typedbytes.TypedBytesRecordReader'
+) tmap
+INSERT OVERWRITE TABLE dest1 SELECT tkey, tvalue
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@dest1
+PREHOOK: query: SELECT dest1.* FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+PREHOOK: Output: file:/data/users/njain/hive1/hive1/build/ql/tmp/2086086825/10000
+POSTHOOK: query: SELECT dest1.* FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+POSTHOOK: Output: file:/data/users/njain/hive1/hive1/build/ql/tmp/2086086825/10000
+238 val_238
+86 val_86
+311 val_311
+27 val_27
+165 val_165
+409 val_409
+255 val_255
+278 val_278
+98 val_98
+484 val_484
+265 val_265
+193 val_193
+401 val_401
+150 val_150
+273 val_273
+224 val_224
+369 val_369
+66 val_66
+128 val_128
+213 val_213
+146 val_146
+406 val_406
+429 val_429
+374 val_374
+152 val_152
+469 val_469
+145 val_145
+495 val_495
+37 val_37
+327 val_327
+281 val_281
+277 val_277
+209 val_209
+15 val_15
+82 val_82
+403 val_403
+166 val_166
+417 val_417
+430 val_430
+252 val_252
+292 val_292
+219 val_219
+287 val_287
+153 val_153
+193 val_193
+338 val_338
+446 val_446
+459 val_459
+394 val_394
+237 val_237
+482 val_482
+174 val_174
+413 val_413
+494 val_494
+207 val_207
+199 val_199
+466 val_466
+208 val_208
+174 val_174
+399 val_399
+396 val_396
+247 val_247
+417 val_417
+489 val_489
+162 val_162
+377 val_377
+397 val_397
+309 val_309
+365 val_365
+266 val_266
+439 val_439
+342 val_342
+367 val_367
+325 val_325
+167 val_167
+195 val_195
+475 val_475
+17 val_17
+113 val_113
+155 val_155
+203 val_203
+339 val_339
+0 val_0
+455 val_455
+128 val_128
+311 val_311
+316 val_316
+57 val_57
+302 val_302
+205 val_205
+149 val_149
+438 val_438
+345 val_345
+129 val_129
+170 val_170
+20 val_20
+489 val_489
+157 val_157
+378 val_378
+221 val_221
+92 val_92
+111 val_111
+47 val_47
+72 val_72
+4 val_4
+280 val_280
+35 val_35
+427 val_427
+277 val_277
+208 val_208
+356 val_356
+399 val_399
+169 val_169
+382 val_382
+498 val_498
+125 val_125
+386 val_386
+437 val_437
+469 val_469
+192 val_192
+286 val_286
+187 val_187
+176 val_176
+54 val_54
+459 val_459
+51 val_51
+138 val_138
+103 val_103
+239 val_239
+213 val_213
+216 val_216
+430 val_430
+278 val_278
+176 val_176
+289 val_289
+221 val_221
+65 val_65
+318 val_318
+332 val_332
+311 val_311
+275 val_275
+137 val_137
+241 val_241
+83 val_83
+333 val_333
+180 val_180
+284 val_284
+12 val_12
+230 val_230
+181 val_181
+67 val_67
+260 val_260
+404 val_404
+384 val_384
+489 val_489
+353 val_353
+373 val_373
+272 val_272
+138 val_138
+217 val_217
+84 val_84
+348 val_348
+466 val_466
+58 val_58
+8 val_8
+411 val_411
+230 val_230
+208 val_208
+348 val_348
+24 val_24
+463 val_463
+431 val_431
+179 val_179
+172 val_172
+42 val_42
+129 val_129
+158 val_158
+119 val_119
+496 val_496
+0 val_0
+322 val_322
+197 val_197
+468 val_468
+393 val_393
+454 val_454
+100 val_100
+298 val_298
+199 val_199
+191 val_191
+418 val_418
+96 val_96
+26 val_26
+165 val_165
+327 val_327
+230 val_230
+205 val_205
+120 val_120
+131 val_131
+51 val_51
+404 val_404
+43 val_43
+436 val_436
+156 val_156
+469 val_469
+468 val_468
+308 val_308
+95 val_95
+196 val_196
+288 val_288
+481 val_481
+457 val_457
+98 val_98
+282 val_282
+197 val_197
+187 val_187
+318 val_318
+318 val_318
+409 val_409
+470 val_470
+137 val_137
+369 val_369
+316 val_316
+169 val_169
+413 val_413
+85 val_85
+77 val_77
+0 val_0
+490 val_490
+87 val_87
+364 val_364
+179 val_179
+118 val_118
+134 val_134
+395 val_395
+282 val_282
+138 val_138
+238 val_238
+419 val_419
+15 val_15
+118 val_118
+72 val_72
+90 val_90
+307 val_307
+19 val_19
+435 val_435
+10 val_10
+277 val_277
+273 val_273
+306 val_306
+224 val_224
+309 val_309
+389 val_389
+327 val_327
+242 val_242
+369 val_369
+392 val_392
+272 val_272
+331 val_331
+401 val_401
+242 val_242
+452 val_452
+177 val_177
+226 val_226
+5 val_5
+497 val_497
+402 val_402
+396 val_396
+317 val_317
+395 val_395
+58 val_58
+35 val_35
+336 val_336
+95 val_95
+11 val_11
+168 val_168
+34 val_34
+229 val_229
+233 val_233
+143 val_143
+472 val_472
+322 val_322
+498 val_498
+160 val_160
+195 val_195
+42 val_42
+321 val_321
+430 val_430
+119 val_119
+489 val_489
+458 val_458
+78 val_78
+76 val_76
+41 val_41
+223 val_223
+492 val_492
+149 val_149
+449 val_449
+218 val_218
+228 val_228
+138 val_138
+453 val_453
+30 val_30
+209 val_209
+64 val_64
+468 val_468
+76 val_76
+74 val_74
+342 val_342
+69 val_69
+230 val_230
+33 val_33
+368 val_368
+103 val_103
+296 val_296
+113 val_113
+216 val_216
+367 val_367
+344 val_344
+167 val_167
+274 val_274
+219 val_219
+239 val_239
+485 val_485
+116 val_116
+223 val_223
+256 val_256
+263 val_263
+70 val_70
+487 val_487
+480 val_480
+401 val_401
+288 val_288
+191 val_191
+5 val_5
+244 val_244
+438 val_438
+128 val_128
+467 val_467
+432 val_432
+202 val_202
+316 val_316
+229 val_229
+469 val_469
+463 val_463
+280 val_280
+2 val_2
+35 val_35
+283 val_283
+331 val_331
+235 val_235
+80 val_80
+44 val_44
+193 val_193
+321 val_321
+335 val_335
+104 val_104
+466 val_466
+366 val_366
+175 val_175
+403 val_403
+483 val_483
+53 val_53
+105 val_105
+257 val_257
+406 val_406
+409 val_409
+190 val_190
+406 val_406
+401 val_401
+114 val_114
+258 val_258
+90 val_90
+203 val_203
+262 val_262
+348 val_348
+424 val_424
+12 val_12
+396 val_396
+201 val_201
+217 val_217
+164 val_164
+431 val_431
+454 val_454
+478 val_478
+298 val_298
+125 val_125
+431 val_431
+164 val_164
+424 val_424
+187 val_187
+382 val_382
+5 val_5
+70 val_70
+397 val_397
+480 val_480
+291 val_291
+24 val_24
+351 val_351
+255 val_255
+104 val_104
+70 val_70
+163 val_163
+438 val_438
+119 val_119
+414 val_414
+200 val_200
+491 val_491
+237 val_237
+439 val_439
+360 val_360
+248 val_248
+479 val_479
+305 val_305
+417 val_417
+199 val_199
+444 val_444
+120 val_120
+429 val_429
+169 val_169
+443 val_443
+323 val_323
+325 val_325
+277 val_277
+230 val_230
+478 val_478
+178 val_178
+468 val_468
+310 val_310
+317 val_317
+333 val_333
+493 val_493
+460 val_460
+207 val_207
+249 val_249
+265 val_265
+480 val_480
+83 val_83
+136 val_136
+353 val_353
+172 val_172
+214 val_214
+462 val_462
+233 val_233
+406 val_406
+133 val_133
+175 val_175
+189 val_189
+454 val_454
+375 val_375
+401 val_401
+421 val_421
+407 val_407
+384 val_384
+256 val_256
+26 val_26
+134 val_134
+67 val_67
+384 val_384
+379 val_379
+18 val_18
+462 val_462
+492 val_492
+100 val_100
+298 val_298
+9 val_9
+341 val_341
+498 val_498
+146 val_146
+458 val_458
+362 val_362
+186 val_186
+285 val_285
+348 val_348
+167 val_167
+18 val_18
+273 val_273
+183 val_183
+281 val_281
+344 val_344
+97 val_97
+469 val_469
+315 val_315
+84 val_84
+28 val_28
+37 val_37
+448 val_448
+152 val_152
+348 val_348
+307 val_307
+194 val_194
+414 val_414
+477 val_477
+222 val_222
+126 val_126
+90 val_90
+169 val_169
+403 val_403
+400 val_400
+200 val_200
+97 val_97
+PREHOOK: query: drop table dest1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table dest1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@dest1
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RecordReader.java?rev=894971&r1=894970&r2=894971&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RecordReader.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/RecordReader.java Fri Jan 1 04:19:34 2010
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
@@ -27,11 +28,11 @@
public interface RecordReader {
- public void initialize(InputStream in, Configuration conf) throws IOException;
+ public void initialize(InputStream in, Configuration conf, Properties tbl) throws IOException;
public Writable createRow() throws IOException;
public int next(Writable row) throws IOException;
-
+
public void close() throws IOException;
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java?rev=894971&r1=894970&r2=894971&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ScriptOperator.java Fri Jan 1 04:19:34 2010
@@ -250,12 +250,12 @@
Map<String, String> env = pb.environment();
addJobConfToEnvironment(hconf, env);
env.put(safeEnvVarName(HiveConf.ConfVars.HIVEALIAS.varname), String.valueOf(alias));
-
+
// Create an environment variable that uniquely identifies this script operator
String idEnvVarName = HiveConf.getVar(hconf, HiveConf.ConfVars.HIVESCRIPTIDENVVAR);
String idEnvVarVal = this.getOperatorId();
env.put(safeEnvVarName(idEnvVarName), idEnvVarVal);
-
+
scriptPid = pb.start(); // Runtime.getRuntime().exec(wrappedCmdArgs);
DataOutputStream scriptOut = new DataOutputStream(new BufferedOutputStream(scriptPid.getOutputStream()));
@@ -266,13 +266,13 @@
scriptOutWriter.initialize(scriptOut, hconf);
RecordReader scriptOutputReader = conf.getOutRecordReaderClass().newInstance();
- scriptOutputReader.initialize(scriptIn, hconf);
+ scriptOutputReader.initialize(scriptIn, hconf, conf.getScriptOutputInfo().getProperties());
outThread = new StreamThread(scriptOutputReader, new OutputStreamProcessor(
scriptOutputDeserializer.getObjectInspector()), "OutputProcessor");
RecordReader scriptErrReader = conf.getOutRecordReaderClass().newInstance();
- scriptErrReader.initialize(scriptErr, hconf);
+ scriptErrReader.initialize(scriptErr, hconf, conf.getScriptOutputInfo().getProperties());
errThread = new StreamThread(scriptErrReader,
new ErrorStreamProcessor
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java?rev=894971&r1=894970&r2=894971&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TextRecordReader.java Fri Jan 1 04:19:34 2010
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.InputStream;
+import java.util.Properties;
import org.apache.hadoop.mapred.LineRecordReader.LineReader;
@@ -34,7 +35,7 @@
private InputStream in;
private Text row;
- public void initialize(InputStream in, Configuration conf) throws IOException {
+ public void initialize(InputStream in, Configuration conf, Properties tbl) throws IOException {
lineReader = new LineReader(in, conf);
this.in = in;
}
@@ -50,7 +51,7 @@
return lineReader.readLine((Text)row);
}
-
+
public void close() throws IOException {
if (in != null)
in.close();