You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2010/06/22 02:21:52 UTC
svn commit: r956749 - in /hadoop/hive/branches/branch-0.6: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/

Author: heyongqiang
Date: Tue Jun 22 00:21:52 2010
New Revision: 956749

URL: http://svn.apache.org/viewvc?rev=956749&view=rev
Log:
HIVE-1421: problem with sequence and rcfiles are mixed for null partitions (namit via He Yongqiang)

Added:
    hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/fileformat_mix.q
    hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/fileformat_mix.q.out
Modified:
    hadoop/hive/branches/branch-0.6/CHANGES.txt
    hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
    hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java

Modified: hadoop/hive/branches/branch-0.6/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/CHANGES.txt?rev=956749&r1=956748&r2=956749&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/CHANGES.txt (original)
+++ hadoop/hive/branches/branch-0.6/CHANGES.txt Tue Jun 22 00:21:52 2010
@@ -529,6 +529,9 @@ Release 0.6.0 -  Unreleased
     HIVE-1418. Bug in RCfiles with Lateral Views
     (He Yongqiang via namit)
 
+    HIVE-1421. problem with sequence and rcfiles are mixed for null partitions
+    (namit via He Yongqiang)
+
 Release 0.5.0 -  Unreleased
 
   INCOMPATIBLE CHANGES

Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=956749&r1=956748&r2=956749&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Tue Jun 22 00:21:52 2010
@@ -1211,16 +1211,18 @@ public class ExecDriver extends Task<Map
 
     // The input file does not exist, replace it by a empty file
     Class<? extends HiveOutputFormat> outFileFormat = null;
-
-    TableDesc tableDesc;
+    boolean nonNative = true;
     if (isEmptyPath) {
-      tableDesc = work.getPathToPartitionInfo().get(path).getTableDesc();
+      PartitionDesc partDesc = work.getPathToPartitionInfo().get(path);
+      outFileFormat = partDesc.getOutputFileFormatClass();
+      nonNative = partDesc.getTableDesc().isNonNative();
     } else {
-      tableDesc = work.getAliasToPartnInfo().get(alias).getTableDesc();
+      TableDesc tableDesc =  work.getAliasToPartnInfo().get(alias).getTableDesc();
+      outFileFormat = tableDesc.getOutputFileFormatClass();
+      nonNative = tableDesc.isNonNative();
     }
-    outFileFormat = tableDesc.getOutputFileFormatClass();
 
-    if (tableDesc.isNonNative()) {
+    if (nonNative) {
       FileInputFormat.addInputPaths(job, path);
       LOG.info("Add a non-native table " + path);
       return numEmptyPaths;

Modified: hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=956749&r1=956748&r2=956749&view=diff
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hadoop/hive/branches/branch-0.6/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Tue Jun 22 00:21:52 2010
@@ -304,6 +304,7 @@ public abstract class Operator<T extends
       return;
     }
 
+    this.out = null;
     if (!areAllParentsInitialized()) {
       return;
     }

Added: hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/fileformat_mix.q
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/fileformat_mix.q?rev=956749&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/fileformat_mix.q (added)
+++ hadoop/hive/branches/branch-0.6/ql/src/test/queries/clientpositive/fileformat_mix.q Tue Jun 22 00:21:52 2010
@@ -0,0 +1,17 @@
+drop table fileformat_mix_test;
+
+create table fileformat_mix_test (src int, value string) partitioned by (ds string);
+alter table fileformat_mix_test set fileformat Sequencefile;
+
+insert overwrite table fileformat_mix_test partition (ds='1')
+select key, value from src;
+
+alter table fileformat_mix_test add partition (ds='2');
+
+alter table fileformat_mix_test set fileformat rcfile;
+
+select count(1) from fileformat_mix_test;
+
+select src from fileformat_mix_test;
+
+drop table fileformat_mix_test;
\ No newline at end of file

Added: hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/fileformat_mix.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/fileformat_mix.q.out?rev=956749&view=auto
==============================================================================
--- hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/fileformat_mix.q.out (added)
+++ hadoop/hive/branches/branch-0.6/ql/src/test/results/clientpositive/fileformat_mix.q.out Tue Jun 22 00:21:52 2010
@@ -0,0 +1,574 @@
+PREHOOK: query: drop table fileformat_mix_test
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table fileformat_mix_test
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table fileformat_mix_test (src int, value string) partitioned by (ds string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table fileformat_mix_test (src int, value string) partitioned by (ds string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@fileformat_mix_test
+PREHOOK: query: alter table fileformat_mix_test set fileformat Sequencefile
+PREHOOK: type: null
+POSTHOOK: query: alter table fileformat_mix_test set fileformat Sequencefile
+POSTHOOK: type: null
+POSTHOOK: Input: default@fileformat_mix_test
+POSTHOOK: Output: default@fileformat_mix_test
+PREHOOK: query: insert overwrite table fileformat_mix_test partition (ds='1')
+select key, value from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@fileformat_mix_test@ds=1
+POSTHOOK: query: insert overwrite table fileformat_mix_test partition (ds='1')
+select key, value from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@fileformat_mix_test@ds=1
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).src EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: alter table fileformat_mix_test add partition (ds='2')
+PREHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: query: alter table fileformat_mix_test add partition (ds='2')
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+POSTHOOK: Output: default@fileformat_mix_test@ds=2
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).src EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: alter table fileformat_mix_test set fileformat rcfile
+PREHOOK: type: null
+POSTHOOK: query: alter table fileformat_mix_test set fileformat rcfile
+POSTHOOK: type: null
+POSTHOOK: Input: default@fileformat_mix_test
+POSTHOOK: Output: default@fileformat_mix_test
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).src EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: select count(1) from fileformat_mix_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@fileformat_mix_test@ds=1
+PREHOOK: Input: default@fileformat_mix_test@ds=2
+PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-20_21-21-12_115_4377005987442515283/10000
+POSTHOOK: query: select count(1) from fileformat_mix_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@fileformat_mix_test@ds=1
+POSTHOOK: Input: default@fileformat_mix_test@ds=2
+POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-20_21-21-12_115_4377005987442515283/10000
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).src EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+500
+PREHOOK: query: select src from fileformat_mix_test
+PREHOOK: type: QUERY
+PREHOOK: Input: default@fileformat_mix_test@ds=1
+PREHOOK: Input: default@fileformat_mix_test@ds=2
+PREHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-20_21-21-17_269_3478340175683868925/10000
+POSTHOOK: query: select src from fileformat_mix_test
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@fileformat_mix_test@ds=1
+POSTHOOK: Input: default@fileformat_mix_test@ds=2
+POSTHOOK: Output: file:/var/folders/6g/6grtCwPMEf4sqHUPpy6xQG9ByHg/-Tmp-/heyongqiang/hive_2010-06-20_21-21-17_269_3478340175683868925/10000
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).src EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+238
+86
+311
+27
+165
+409
+255
+278
+98
+484
+265
+193
+401
+150
+273
+224
+369
+66
+128
+213
+146
+406
+429
+374
+152
+469
+145
+495
+37
+327
+281
+277
+209
+15
+82
+403
+166
+417
+430
+252
+292
+219
+287
+153
+193
+338
+446
+459
+394
+237
+482
+174
+413
+494
+207
+199
+466
+208
+174
+399
+396
+247
+417
+489
+162
+377
+397
+309
+365
+266
+439
+342
+367
+325
+167
+195
+475
+17
+113
+155
+203
+339
+0
+455
+128
+311
+316
+57
+302
+205
+149
+438
+345
+129
+170
+20
+489
+157
+378
+221
+92
+111
+47
+72
+4
+280
+35
+427
+277
+208
+356
+399
+169
+382
+498
+125
+386
+437
+469
+192
+286
+187
+176
+54
+459
+51
+138
+103
+239
+213
+216
+430
+278
+176
+289
+221
+65
+318
+332
+311
+275
+137
+241
+83
+333
+180
+284
+12
+230
+181
+67
+260
+404
+384
+489
+353
+373
+272
+138
+217
+84
+348
+466
+58
+8
+411
+230
+208
+348
+24
+463
+431
+179
+172
+42
+129
+158
+119
+496
+0
+322
+197
+468
+393
+454
+100
+298
+199
+191
+418
+96
+26
+165
+327
+230
+205
+120
+131
+51
+404
+43
+436
+156
+469
+468
+308
+95
+196
+288
+481
+457
+98
+282
+197
+187
+318
+318
+409
+470
+137
+369
+316
+169
+413
+85
+77
+0
+490
+87
+364
+179
+118
+134
+395
+282
+138
+238
+419
+15
+118
+72
+90
+307
+19
+435
+10
+277
+273
+306
+224
+309
+389
+327
+242
+369
+392
+272
+331
+401
+242
+452
+177
+226
+5
+497
+402
+396
+317
+395
+58
+35
+336
+95
+11
+168
+34
+229
+233
+143
+472
+322
+498
+160
+195
+42
+321
+430
+119
+489
+458
+78
+76
+41
+223
+492
+149
+449
+218
+228
+138
+453
+30
+209
+64
+468
+76
+74
+342
+69
+230
+33
+368
+103
+296
+113
+216
+367
+344
+167
+274
+219
+239
+485
+116
+223
+256
+263
+70
+487
+480
+401
+288
+191
+5
+244
+438
+128
+467
+432
+202
+316
+229
+469
+463
+280
+2
+35
+283
+331
+235
+80
+44
+193
+321
+335
+104
+466
+366
+175
+403
+483
+53
+105
+257
+406
+409
+190
+406
+401
+114
+258
+90
+203
+262
+348
+424
+12
+396
+201
+217
+164
+431
+454
+478
+298
+125
+431
+164
+424
+187
+382
+5
+70
+397
+480
+291
+24
+351
+255
+104
+70
+163
+438
+119
+414
+200
+491
+237
+439
+360
+248
+479
+305
+417
+199
+444
+120
+429
+169
+443
+323
+325
+277
+230
+478
+178
+468
+310
+317
+333
+493
+460
+207
+249
+265
+480
+83
+136
+353
+172
+214
+462
+233
+406
+133
+175
+189
+454
+375
+401
+421
+407
+384
+256
+26
+134
+67
+384
+379
+18
+462
+492
+100
+298
+9
+341
+498
+146
+458
+362
+186
+285
+348
+167
+18
+273
+183
+281
+344
+97
+469
+315
+84
+28
+37
+448
+152
+348
+307
+194
+414
+477
+222
+126
+90
+169
+403
+400
+200
+97
+PREHOOK: query: drop table fileformat_mix_test
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table fileformat_mix_test
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: default@fileformat_mix_test
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).src EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: fileformat_mix_test PARTITION(ds=1).value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]