You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2010/09/29 07:22:53 UTC
svn commit: r1002472 - in /hadoop/hive/branches/branch-0.6: ./
metastore/src/test/org/apache/hadoop/hive/metastore/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: nzhang
Date: Wed Sep 29 05:22:52 2010
New Revision: 1002472
URL: http://svn.apache.org/viewvc?rev=1002472&view=rev
Log:
HIVE-1524. parallel execution failed if mapred.job.name is set (Yuanjun Li via Ning Zhang)
Added:
hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/parallel.q
hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/parallel.q.out
Removed:
hadoop/hive/branches/branch-0.6/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreRemote.java
Modified:
hadoop/hive/branches/branch-0.6/CHANGES.txt
hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Modified: hadoop/hive/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/CHANGES.txt?rev=1002472&r1=1002471&r2=1002472&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.6/CHANGES.txt Wed Sep 29 05:22:52 2010
@@ -607,6 +607,9 @@ Release 0.6.0 - Unreleased
metastore methods
(Edward Capriolo via jvs)
+ HIVE-1524. parallel execution failed if mapred.job.name is set
+ (Yuanjun Li via Ning Zhang)
+
Release 0.5.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1002472&r1=1002471&r2=1002472&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Wed Sep 29 05:22:52 2010
@@ -52,6 +52,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
+import java.util.UUID;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -61,6 +62,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
+import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -139,22 +141,26 @@ public final class Utilities {
} catch (Exception e) {
} finally {
// where a single process works with multiple plans - we must clear
- // the cache before working with the next plan.
- gWorkMap.remove(getJobName(job));
+ // the cache before working with the next plan.
+ String jobID = getHiveJobID(job);
+ if (jobID != null) {
+ gWorkMap.remove(jobID);
+ }
}
}
public static MapredWork getMapRedWork(Configuration job) {
MapredWork gWork = null;
try {
- gWork = gWorkMap.get(getJobName(job));
+ String jobID = getHiveJobID(job);
+ assert jobID != null;
+ gWork = gWorkMap.get(jobID);
if (gWork == null) {
- InputStream in = new FileInputStream("HIVE_PLAN"
- + sanitizedJobId(job));
+ InputStream in = new FileInputStream("HIVE_PLAN" + jobID);
MapredWork ret = deserializeMapRedWork(in, job);
gWork = ret;
gWork.initialize();
- gWorkMap.put(getJobName(job), gWork);
+ gWorkMap.put(jobID, gWork);
}
return (gWork);
} catch (Exception e) {
@@ -272,6 +278,10 @@ public final class Utilities {
public static void setMapRedWork(Configuration job, MapredWork w, String hiveScratchDir) {
try {
+ // this is the unique job ID, which is kept in JobConf as part of the plan file name
+ String jobID = UUID.randomUUID().toString();
+ Path planPath = new Path(hiveScratchDir, jobID);
+ HiveConf.setVar(job, HiveConf.ConfVars.PLAN, planPath.toUri().toString());
// Serialize the plan to the default hdfs instance
// Except for hadoop local mode execution where we should be
@@ -280,45 +290,33 @@ public final class Utilities {
if(!HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJT).equals("local")) {
// use the default file system of the job
FileSystem fs = FileSystem.get(job);
- Path planPath = new Path(hiveScratchDir, "plan." + randGen.nextInt());
+
FSDataOutputStream out = fs.create(planPath);
serializeMapRedWork(w, out);
- HiveConf.setVar(job, HiveConf.ConfVars.PLAN, planPath.toString());
+
// Set up distributed cache
DistributedCache.createSymlink(job);
- String uriWithLink = planPath.toUri().toString() + "#HIVE_PLAN"
- + sanitizedJobId(job);
+ String uriWithLink = planPath.toUri().toString() + "#HIVE_PLAN" + jobID;
DistributedCache.addCacheFile(new URI(uriWithLink), job);
}
// Cache the plan in this process
w.initialize();
- gWorkMap.put(getJobName(job), w);
+ gWorkMap.put(jobID, w);
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
- public static String getJobName(Configuration job) {
- String s = HiveConf.getVar(job, HiveConf.ConfVars.HADOOPJOBNAME);
- // This is just a backup case. We would like Hive to always have jobnames.
- if (s == null) {
- // There is no job name => we set one
- s = "JOB" + randGen.nextInt();
- HiveConf.setVar(job, HiveConf.ConfVars.HADOOPJOBNAME, s);
- }
- return s;
- }
-
- /**
- * Returns a unique ID for the job.
- */
-
- public static int sanitizedJobId(Configuration job) {
- String s = getJobName(job);
- return s.hashCode();
- }
+ private static String getHiveJobID(Configuration job) {
+ String planPath= HiveConf.getVar(job, HiveConf.ConfVars.PLAN);
+ if (planPath != null) {
+ return (new Path(planPath)).getName();
+ }
+ return null;
+ }
+
/**
* Serialize a single Task.
Added: hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/parallel.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/parallel.q?rev=1002472&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/parallel.q (added)
+++ hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/parallel.q Wed Sep 29 05:22:52 2010
@@ -0,0 +1,28 @@
+set mapred.job.name='test_parallel';
+set hive.exec.parallel=true;
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+
+create table if not exists src_a like src;
+create table if not exists src_b like src;
+
+explain
+from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value;
+
+from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value;
+
+select * from src_a order by key, value;
+select * from src_b order by key, value;
+
+
+set hive.input.format=org.apache.hadoop.hive.ql.io.CombineHiveInputFormat;
+
+from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value;
+
+select * from src_a order by key, value;
+select * from src_b order by key, value;
Added: hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/parallel.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/parallel.q.out?rev=1002472&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/parallel.q.out (added)
+++ hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/parallel.q.out Wed Sep 29 05:22:52 2010
@@ -0,0 +1,1567 @@
+PREHOOK: query: create table if not exists src_a like src
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table if not exists src_a like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@src_a
+PREHOOK: query: create table if not exists src_b like src
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table if not exists src_b like src
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@src_b
+PREHOOK: query: explain
+from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_TABREF src)) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_TABLE_OR_COL key)) (TOK_SELEXPR (TOK_TABLE_OR_COL value))) (TOK_GROUPBY (TOK_TABLE_OR_COL key) (TOK_TABLE_OR_COL value)))) s)) (TOK_INSERT (TOK_DESTINATION (TOK_TAB src_a)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL s) value))) (TOK_GROUPBY (. (TOK_TABLE_OR_COL s) key) (. (TOK_TABLE_OR_COL s) value))) (TOK_INSERT (TOK_DESTINATION (TOK_TAB src_b)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL s) key)) (TOK_SELEXPR (. (TOK_TABLE_OR_COL s) value))) (TOK_GROUPBY (. (TOK_TABLE_OR_COL s) key) (. (TOK_TABLE_OR_COL s) value))))
+
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-3 depends on stages: Stage-2
+ Stage-0 depends on stages: Stage-3
+ Stage-4 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+ s:src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ outputColumnNames: key, value
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ tag: -1
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ expr: KEY._col1
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ mode: hash
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-3
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/nzhang/hive_2010-08-09_22-25-23_894_8240793898629070545/-mr-10004
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ tag: -1
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ expr: KEY._col1
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 1
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: src_a
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: src_a
+
+ Stage: Stage-4
+ Map Reduce
+ Alias -> Map Operator Tree:
+ file:/tmp/nzhang/hive_2010-08-09_22-25-23_894_8240793898629070545/-mr-10005
+ Reduce Output Operator
+ key expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ sort order: ++
+ Map-reduce partition columns:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ tag: -1
+ Reduce Operator Tree:
+ Group By Operator
+ bucketGroup: false
+ keys:
+ expr: KEY._col0
+ type: string
+ expr: KEY._col1
+ type: string
+ mode: mergepartial
+ outputColumnNames: _col0, _col1
+ Select Operator
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
+ GlobalTableId: 2
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: src_b
+
+ Stage: Stage-1
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: src_b
+
+
+PREHOOK: query: from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_a
+PREHOOK: Output: default@src_b
+POSTHOOK: query: from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_a
+POSTHOOK: Output: default@src_b
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from src_a order by key, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_a
+PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-25-37_362_5325705374349052168/-mr-10000
+POSTHOOK: query: select * from src_a order by key, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_a
+POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-25-37_362_5325705374349052168/-mr-10000
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0
+10 val_10
+100 val_100
+103 val_103
+104 val_104
+105 val_105
+11 val_11
+111 val_111
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+119 val_119
+12 val_12
+120 val_120
+125 val_125
+126 val_126
+128 val_128
+129 val_129
+131 val_131
+133 val_133
+134 val_134
+136 val_136
+137 val_137
+138 val_138
+143 val_143
+145 val_145
+146 val_146
+149 val_149
+15 val_15
+150 val_150
+152 val_152
+153 val_153
+155 val_155
+156 val_156
+157 val_157
+158 val_158
+160 val_160
+162 val_162
+163 val_163
+164 val_164
+165 val_165
+166 val_166
+167 val_167
+168 val_168
+169 val_169
+17 val_17
+170 val_170
+172 val_172
+174 val_174
+175 val_175
+176 val_176
+177 val_177
+178 val_178
+179 val_179
+18 val_18
+180 val_180
+181 val_181
+183 val_183
+186 val_186
+187 val_187
+189 val_189
+19 val_19
+190 val_190
+191 val_191
+192 val_192
+193 val_193
+194 val_194
+195 val_195
+196 val_196
+197 val_197
+199 val_199
+2 val_2
+20 val_20
+200 val_200
+201 val_201
+202 val_202
+203 val_203
+205 val_205
+207 val_207
+208 val_208
+209 val_209
+213 val_213
+214 val_214
+216 val_216
+217 val_217
+218 val_218
+219 val_219
+221 val_221
+222 val_222
+223 val_223
+224 val_224
+226 val_226
+228 val_228
+229 val_229
+230 val_230
+233 val_233
+235 val_235
+237 val_237
+238 val_238
+239 val_239
+24 val_24
+241 val_241
+242 val_242
+244 val_244
+247 val_247
+248 val_248
+249 val_249
+252 val_252
+255 val_255
+256 val_256
+257 val_257
+258 val_258
+26 val_26
+260 val_260
+262 val_262
+263 val_263
+265 val_265
+266 val_266
+27 val_27
+272 val_272
+273 val_273
+274 val_274
+275 val_275
+277 val_277
+278 val_278
+28 val_28
+280 val_280
+281 val_281
+282 val_282
+283 val_283
+284 val_284
+285 val_285
+286 val_286
+287 val_287
+288 val_288
+289 val_289
+291 val_291
+292 val_292
+296 val_296
+298 val_298
+30 val_30
+302 val_302
+305 val_305
+306 val_306
+307 val_307
+308 val_308
+309 val_309
+310 val_310
+311 val_311
+315 val_315
+316 val_316
+317 val_317
+318 val_318
+321 val_321
+322 val_322
+323 val_323
+325 val_325
+327 val_327
+33 val_33
+331 val_331
+332 val_332
+333 val_333
+335 val_335
+336 val_336
+338 val_338
+339 val_339
+34 val_34
+341 val_341
+342 val_342
+344 val_344
+345 val_345
+348 val_348
+35 val_35
+351 val_351
+353 val_353
+356 val_356
+360 val_360
+362 val_362
+364 val_364
+365 val_365
+366 val_366
+367 val_367
+368 val_368
+369 val_369
+37 val_37
+373 val_373
+374 val_374
+375 val_375
+377 val_377
+378 val_378
+379 val_379
+382 val_382
+384 val_384
+386 val_386
+389 val_389
+392 val_392
+393 val_393
+394 val_394
+395 val_395
+396 val_396
+397 val_397
+399 val_399
+4 val_4
+400 val_400
+401 val_401
+402 val_402
+403 val_403
+404 val_404
+406 val_406
+407 val_407
+409 val_409
+41 val_41
+411 val_411
+413 val_413
+414 val_414
+417 val_417
+418 val_418
+419 val_419
+42 val_42
+421 val_421
+424 val_424
+427 val_427
+429 val_429
+43 val_43
+430 val_430
+431 val_431
+432 val_432
+435 val_435
+436 val_436
+437 val_437
+438 val_438
+439 val_439
+44 val_44
+443 val_443
+444 val_444
+446 val_446
+448 val_448
+449 val_449
+452 val_452
+453 val_453
+454 val_454
+455 val_455
+457 val_457
+458 val_458
+459 val_459
+460 val_460
+462 val_462
+463 val_463
+466 val_466
+467 val_467
+468 val_468
+469 val_469
+47 val_47
+470 val_470
+472 val_472
+475 val_475
+477 val_477
+478 val_478
+479 val_479
+480 val_480
+481 val_481
+482 val_482
+483 val_483
+484 val_484
+485 val_485
+487 val_487
+489 val_489
+490 val_490
+491 val_491
+492 val_492
+493 val_493
+494 val_494
+495 val_495
+496 val_496
+497 val_497
+498 val_498
+5 val_5
+51 val_51
+53 val_53
+54 val_54
+57 val_57
+58 val_58
+64 val_64
+65 val_65
+66 val_66
+67 val_67
+69 val_69
+70 val_70
+72 val_72
+74 val_74
+76 val_76
+77 val_77
+78 val_78
+8 val_8
+80 val_80
+82 val_82
+83 val_83
+84 val_84
+85 val_85
+86 val_86
+87 val_87
+9 val_9
+90 val_90
+92 val_92
+95 val_95
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: select * from src_b order by key, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_b
+PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-25-43_561_4685920856018764066/-mr-10000
+POSTHOOK: query: select * from src_b order by key, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_b
+POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-25-43_561_4685920856018764066/-mr-10000
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0
+10 val_10
+100 val_100
+103 val_103
+104 val_104
+105 val_105
+11 val_11
+111 val_111
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+119 val_119
+12 val_12
+120 val_120
+125 val_125
+126 val_126
+128 val_128
+129 val_129
+131 val_131
+133 val_133
+134 val_134
+136 val_136
+137 val_137
+138 val_138
+143 val_143
+145 val_145
+146 val_146
+149 val_149
+15 val_15
+150 val_150
+152 val_152
+153 val_153
+155 val_155
+156 val_156
+157 val_157
+158 val_158
+160 val_160
+162 val_162
+163 val_163
+164 val_164
+165 val_165
+166 val_166
+167 val_167
+168 val_168
+169 val_169
+17 val_17
+170 val_170
+172 val_172
+174 val_174
+175 val_175
+176 val_176
+177 val_177
+178 val_178
+179 val_179
+18 val_18
+180 val_180
+181 val_181
+183 val_183
+186 val_186
+187 val_187
+189 val_189
+19 val_19
+190 val_190
+191 val_191
+192 val_192
+193 val_193
+194 val_194
+195 val_195
+196 val_196
+197 val_197
+199 val_199
+2 val_2
+20 val_20
+200 val_200
+201 val_201
+202 val_202
+203 val_203
+205 val_205
+207 val_207
+208 val_208
+209 val_209
+213 val_213
+214 val_214
+216 val_216
+217 val_217
+218 val_218
+219 val_219
+221 val_221
+222 val_222
+223 val_223
+224 val_224
+226 val_226
+228 val_228
+229 val_229
+230 val_230
+233 val_233
+235 val_235
+237 val_237
+238 val_238
+239 val_239
+24 val_24
+241 val_241
+242 val_242
+244 val_244
+247 val_247
+248 val_248
+249 val_249
+252 val_252
+255 val_255
+256 val_256
+257 val_257
+258 val_258
+26 val_26
+260 val_260
+262 val_262
+263 val_263
+265 val_265
+266 val_266
+27 val_27
+272 val_272
+273 val_273
+274 val_274
+275 val_275
+277 val_277
+278 val_278
+28 val_28
+280 val_280
+281 val_281
+282 val_282
+283 val_283
+284 val_284
+285 val_285
+286 val_286
+287 val_287
+288 val_288
+289 val_289
+291 val_291
+292 val_292
+296 val_296
+298 val_298
+30 val_30
+302 val_302
+305 val_305
+306 val_306
+307 val_307
+308 val_308
+309 val_309
+310 val_310
+311 val_311
+315 val_315
+316 val_316
+317 val_317
+318 val_318
+321 val_321
+322 val_322
+323 val_323
+325 val_325
+327 val_327
+33 val_33
+331 val_331
+332 val_332
+333 val_333
+335 val_335
+336 val_336
+338 val_338
+339 val_339
+34 val_34
+341 val_341
+342 val_342
+344 val_344
+345 val_345
+348 val_348
+35 val_35
+351 val_351
+353 val_353
+356 val_356
+360 val_360
+362 val_362
+364 val_364
+365 val_365
+366 val_366
+367 val_367
+368 val_368
+369 val_369
+37 val_37
+373 val_373
+374 val_374
+375 val_375
+377 val_377
+378 val_378
+379 val_379
+382 val_382
+384 val_384
+386 val_386
+389 val_389
+392 val_392
+393 val_393
+394 val_394
+395 val_395
+396 val_396
+397 val_397
+399 val_399
+4 val_4
+400 val_400
+401 val_401
+402 val_402
+403 val_403
+404 val_404
+406 val_406
+407 val_407
+409 val_409
+41 val_41
+411 val_411
+413 val_413
+414 val_414
+417 val_417
+418 val_418
+419 val_419
+42 val_42
+421 val_421
+424 val_424
+427 val_427
+429 val_429
+43 val_43
+430 val_430
+431 val_431
+432 val_432
+435 val_435
+436 val_436
+437 val_437
+438 val_438
+439 val_439
+44 val_44
+443 val_443
+444 val_444
+446 val_446
+448 val_448
+449 val_449
+452 val_452
+453 val_453
+454 val_454
+455 val_455
+457 val_457
+458 val_458
+459 val_459
+460 val_460
+462 val_462
+463 val_463
+466 val_466
+467 val_467
+468 val_468
+469 val_469
+47 val_47
+470 val_470
+472 val_472
+475 val_475
+477 val_477
+478 val_478
+479 val_479
+480 val_480
+481 val_481
+482 val_482
+483 val_483
+484 val_484
+485 val_485
+487 val_487
+489 val_489
+490 val_490
+491 val_491
+492 val_492
+493 val_493
+494 val_494
+495 val_495
+496 val_496
+497 val_497
+498 val_498
+5 val_5
+51 val_51
+53 val_53
+54 val_54
+57 val_57
+58 val_58
+64 val_64
+65 val_65
+66 val_66
+67 val_67
+69 val_69
+70 val_70
+72 val_72
+74 val_74
+76 val_76
+77 val_77
+78 val_78
+8 val_8
+80 val_80
+82 val_82
+83 val_83
+84 val_84
+85 val_85
+86 val_86
+87 val_87
+9 val_9
+90 val_90
+92 val_92
+95 val_95
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@src_a
+PREHOOK: Output: default@src_b
+POSTHOOK: query: from (select key, value from src group by key, value) s
+insert overwrite table src_a select s.key, s.value group by s.key, s.value
+insert overwrite table src_b select s.key, s.value group by s.key, s.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@src_a
+POSTHOOK: Output: default@src_b
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select * from src_a order by key, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_a
+PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-26-02_417_889900537103370052/-mr-10000
+POSTHOOK: query: select * from src_a order by key, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_a
+POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-26-02_417_889900537103370052/-mr-10000
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0
+10 val_10
+100 val_100
+103 val_103
+104 val_104
+105 val_105
+11 val_11
+111 val_111
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+119 val_119
+12 val_12
+120 val_120
+125 val_125
+126 val_126
+128 val_128
+129 val_129
+131 val_131
+133 val_133
+134 val_134
+136 val_136
+137 val_137
+138 val_138
+143 val_143
+145 val_145
+146 val_146
+149 val_149
+15 val_15
+150 val_150
+152 val_152
+153 val_153
+155 val_155
+156 val_156
+157 val_157
+158 val_158
+160 val_160
+162 val_162
+163 val_163
+164 val_164
+165 val_165
+166 val_166
+167 val_167
+168 val_168
+169 val_169
+17 val_17
+170 val_170
+172 val_172
+174 val_174
+175 val_175
+176 val_176
+177 val_177
+178 val_178
+179 val_179
+18 val_18
+180 val_180
+181 val_181
+183 val_183
+186 val_186
+187 val_187
+189 val_189
+19 val_19
+190 val_190
+191 val_191
+192 val_192
+193 val_193
+194 val_194
+195 val_195
+196 val_196
+197 val_197
+199 val_199
+2 val_2
+20 val_20
+200 val_200
+201 val_201
+202 val_202
+203 val_203
+205 val_205
+207 val_207
+208 val_208
+209 val_209
+213 val_213
+214 val_214
+216 val_216
+217 val_217
+218 val_218
+219 val_219
+221 val_221
+222 val_222
+223 val_223
+224 val_224
+226 val_226
+228 val_228
+229 val_229
+230 val_230
+233 val_233
+235 val_235
+237 val_237
+238 val_238
+239 val_239
+24 val_24
+241 val_241
+242 val_242
+244 val_244
+247 val_247
+248 val_248
+249 val_249
+252 val_252
+255 val_255
+256 val_256
+257 val_257
+258 val_258
+26 val_26
+260 val_260
+262 val_262
+263 val_263
+265 val_265
+266 val_266
+27 val_27
+272 val_272
+273 val_273
+274 val_274
+275 val_275
+277 val_277
+278 val_278
+28 val_28
+280 val_280
+281 val_281
+282 val_282
+283 val_283
+284 val_284
+285 val_285
+286 val_286
+287 val_287
+288 val_288
+289 val_289
+291 val_291
+292 val_292
+296 val_296
+298 val_298
+30 val_30
+302 val_302
+305 val_305
+306 val_306
+307 val_307
+308 val_308
+309 val_309
+310 val_310
+311 val_311
+315 val_315
+316 val_316
+317 val_317
+318 val_318
+321 val_321
+322 val_322
+323 val_323
+325 val_325
+327 val_327
+33 val_33
+331 val_331
+332 val_332
+333 val_333
+335 val_335
+336 val_336
+338 val_338
+339 val_339
+34 val_34
+341 val_341
+342 val_342
+344 val_344
+345 val_345
+348 val_348
+35 val_35
+351 val_351
+353 val_353
+356 val_356
+360 val_360
+362 val_362
+364 val_364
+365 val_365
+366 val_366
+367 val_367
+368 val_368
+369 val_369
+37 val_37
+373 val_373
+374 val_374
+375 val_375
+377 val_377
+378 val_378
+379 val_379
+382 val_382
+384 val_384
+386 val_386
+389 val_389
+392 val_392
+393 val_393
+394 val_394
+395 val_395
+396 val_396
+397 val_397
+399 val_399
+4 val_4
+400 val_400
+401 val_401
+402 val_402
+403 val_403
+404 val_404
+406 val_406
+407 val_407
+409 val_409
+41 val_41
+411 val_411
+413 val_413
+414 val_414
+417 val_417
+418 val_418
+419 val_419
+42 val_42
+421 val_421
+424 val_424
+427 val_427
+429 val_429
+43 val_43
+430 val_430
+431 val_431
+432 val_432
+435 val_435
+436 val_436
+437 val_437
+438 val_438
+439 val_439
+44 val_44
+443 val_443
+444 val_444
+446 val_446
+448 val_448
+449 val_449
+452 val_452
+453 val_453
+454 val_454
+455 val_455
+457 val_457
+458 val_458
+459 val_459
+460 val_460
+462 val_462
+463 val_463
+466 val_466
+467 val_467
+468 val_468
+469 val_469
+47 val_47
+470 val_470
+472 val_472
+475 val_475
+477 val_477
+478 val_478
+479 val_479
+480 val_480
+481 val_481
+482 val_482
+483 val_483
+484 val_484
+485 val_485
+487 val_487
+489 val_489
+490 val_490
+491 val_491
+492 val_492
+493 val_493
+494 val_494
+495 val_495
+496 val_496
+497 val_497
+498 val_498
+5 val_5
+51 val_51
+53 val_53
+54 val_54
+57 val_57
+58 val_58
+64 val_64
+65 val_65
+66 val_66
+67 val_67
+69 val_69
+70 val_70
+72 val_72
+74 val_74
+76 val_76
+77 val_77
+78 val_78
+8 val_8
+80 val_80
+82 val_82
+83 val_83
+84 val_84
+85 val_85
+86 val_86
+87 val_87
+9 val_9
+90 val_90
+92 val_92
+95 val_95
+96 val_96
+97 val_97
+98 val_98
+PREHOOK: query: select * from src_b order by key, value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src_b
+PREHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-26-08_562_2161651706050856117/-mr-10000
+POSTHOOK: query: select * from src_b order by key, value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src_b
+POSTHOOK: Output: file:/tmp/nzhang/hive_2010-08-09_22-26-08_562_2161651706050856117/-mr-10000
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_a.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: src_b.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+0 val_0
+10 val_10
+100 val_100
+103 val_103
+104 val_104
+105 val_105
+11 val_11
+111 val_111
+113 val_113
+114 val_114
+116 val_116
+118 val_118
+119 val_119
+12 val_12
+120 val_120
+125 val_125
+126 val_126
+128 val_128
+129 val_129
+131 val_131
+133 val_133
+134 val_134
+136 val_136
+137 val_137
+138 val_138
+143 val_143
+145 val_145
+146 val_146
+149 val_149
+15 val_15
+150 val_150
+152 val_152
+153 val_153
+155 val_155
+156 val_156
+157 val_157
+158 val_158
+160 val_160
+162 val_162
+163 val_163
+164 val_164
+165 val_165
+166 val_166
+167 val_167
+168 val_168
+169 val_169
+17 val_17
+170 val_170
+172 val_172
+174 val_174
+175 val_175
+176 val_176
+177 val_177
+178 val_178
+179 val_179
+18 val_18
+180 val_180
+181 val_181
+183 val_183
+186 val_186
+187 val_187
+189 val_189
+19 val_19
+190 val_190
+191 val_191
+192 val_192
+193 val_193
+194 val_194
+195 val_195
+196 val_196
+197 val_197
+199 val_199
+2 val_2
+20 val_20
+200 val_200
+201 val_201
+202 val_202
+203 val_203
+205 val_205
+207 val_207
+208 val_208
+209 val_209
+213 val_213
+214 val_214
+216 val_216
+217 val_217
+218 val_218
+219 val_219
+221 val_221
+222 val_222
+223 val_223
+224 val_224
+226 val_226
+228 val_228
+229 val_229
+230 val_230
+233 val_233
+235 val_235
+237 val_237
+238 val_238
+239 val_239
+24 val_24
+241 val_241
+242 val_242
+244 val_244
+247 val_247
+248 val_248
+249 val_249
+252 val_252
+255 val_255
+256 val_256
+257 val_257
+258 val_258
+26 val_26
+260 val_260
+262 val_262
+263 val_263
+265 val_265
+266 val_266
+27 val_27
+272 val_272
+273 val_273
+274 val_274
+275 val_275
+277 val_277
+278 val_278
+28 val_28
+280 val_280
+281 val_281
+282 val_282
+283 val_283
+284 val_284
+285 val_285
+286 val_286
+287 val_287
+288 val_288
+289 val_289
+291 val_291
+292 val_292
+296 val_296
+298 val_298
+30 val_30
+302 val_302
+305 val_305
+306 val_306
+307 val_307
+308 val_308
+309 val_309
+310 val_310
+311 val_311
+315 val_315
+316 val_316
+317 val_317
+318 val_318
+321 val_321
+322 val_322
+323 val_323
+325 val_325
+327 val_327
+33 val_33
+331 val_331
+332 val_332
+333 val_333
+335 val_335
+336 val_336
+338 val_338
+339 val_339
+34 val_34
+341 val_341
+342 val_342
+344 val_344
+345 val_345
+348 val_348
+35 val_35
+351 val_351
+353 val_353
+356 val_356
+360 val_360
+362 val_362
+364 val_364
+365 val_365
+366 val_366
+367 val_367
+368 val_368
+369 val_369
+37 val_37
+373 val_373
+374 val_374
+375 val_375
+377 val_377
+378 val_378
+379 val_379
+382 val_382
+384 val_384
+386 val_386
+389 val_389
+392 val_392
+393 val_393
+394 val_394
+395 val_395
+396 val_396
+397 val_397
+399 val_399
+4 val_4
+400 val_400
+401 val_401
+402 val_402
+403 val_403
+404 val_404
+406 val_406
+407 val_407
+409 val_409
+41 val_41
+411 val_411
+413 val_413
+414 val_414
+417 val_417
+418 val_418
+419 val_419
+42 val_42
+421 val_421
+424 val_424
+427 val_427
+429 val_429
+43 val_43
+430 val_430
+431 val_431
+432 val_432
+435 val_435
+436 val_436
+437 val_437
+438 val_438
+439 val_439
+44 val_44
+443 val_443
+444 val_444
+446 val_446
+448 val_448
+449 val_449
+452 val_452
+453 val_453
+454 val_454
+455 val_455
+457 val_457
+458 val_458
+459 val_459
+460 val_460
+462 val_462
+463 val_463
+466 val_466
+467 val_467
+468 val_468
+469 val_469
+47 val_47
+470 val_470
+472 val_472
+475 val_475
+477 val_477
+478 val_478
+479 val_479
+480 val_480
+481 val_481
+482 val_482
+483 val_483
+484 val_484
+485 val_485
+487 val_487
+489 val_489
+490 val_490
+491 val_491
+492 val_492
+493 val_493
+494 val_494
+495 val_495
+496 val_496
+497 val_497
+498 val_498
+5 val_5
+51 val_51
+53 val_53
+54 val_54
+57 val_57
+58 val_58
+64 val_64
+65 val_65
+66 val_66
+67 val_67
+69 val_69
+70 val_70
+72 val_72
+74 val_74
+76 val_76
+77 val_77
+78 val_78
+8 val_8
+80 val_80
+82 val_82
+83 val_83
+84 val_84
+85 val_85
+86 val_86
+87 val_87
+9 val_9
+90 val_90
+92 val_92
+95 val_95
+96 val_96
+97 val_97
+98 val_98