You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2011/03/31 02:59:04 UTC
svn commit: r1087154 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/JoinUtil.java exec/persistence/RowContainer.java
plan/ConditionalResolverSkewJoin.java
Author: namit
Date: Thu Mar 31 00:59:03 2011
New Revision: 1087154
URL: http://svn.apache.org/viewvc?rev=1087154&view=rev
Log:
HIVE-2083 Bug: RowContainer was set to 1 in JoinUtils.
(He Yongqiang via namit)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1087154&r1=1087153&r2=1087154&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Thu Mar 31 00:59:03 2011
@@ -291,7 +291,7 @@ public class JoinUtil {
public static SerDe getSpillSerDe(byte alias,
Map<Byte, TableDesc> spillTableDesc,JoinDesc conf,
boolean noOuterJoin) {
- TableDesc desc = getSpillTableDesc(alias,spillTableDesc,conf, noOuterJoin);
+ TableDesc desc = getSpillTableDesc(alias, spillTableDesc, conf, noOuterJoin);
if (desc == null) {
return null;
}
@@ -355,10 +355,11 @@ public class JoinUtil {
JoinDesc conf,boolean noOuterJoin) throws HiveException {
TableDesc tblDesc = JoinUtil.getSpillTableDesc(alias,spillTableDesc,conf, noOuterJoin);
- SerDe serde = JoinUtil.getSpillSerDe(alias,spillTableDesc,conf,noOuterJoin);
+ SerDe serde = JoinUtil.getSpillSerDe(alias, spillTableDesc, conf,
+ noOuterJoin);
if (serde == null) {
- containerSize = 1;
+ containerSize = -1;
}
RowContainer rc = new RowContainer(containerSize, hconf);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java?rev=1087154&r1=1087153&r2=1087154&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/RowContainer.java Thu Mar 31 00:59:03 2011
@@ -116,17 +116,13 @@ public class RowContainer<Row extends Li
JobConf jobCloneUsingLocalFs = null;
private LocalFileSystem localFs;
- public RowContainer() {
-
- }
-
public RowContainer(Configuration jc) throws HiveException {
this(BLOCKSIZE, jc);
}
- public RowContainer(int blockSize, Configuration jc) throws HiveException {
+ public RowContainer(int bs, Configuration jc) throws HiveException {
// no 0-sized block
- this.blockSize = blockSize == 0 ? BLOCKSIZE : blockSize;
+ this.blockSize = bs <= 0 ? BLOCKSIZE : bs;
this.size = 0;
this.itrCursor = 0;
this.addCursor = 0;
@@ -139,7 +135,7 @@ public class RowContainer<Row extends Li
this.standardOI = null;
this.jc = jc;
}
-
+
private JobConf getLocalFSJobConfClone(Configuration jc) {
if (this.jobCloneUsingLocalFs == null) {
this.jobCloneUsingLocalFs = new JobConf(jc);
@@ -149,12 +145,6 @@ public class RowContainer<Row extends Li
}
- public RowContainer(int blockSize, SerDe sd, ObjectInspector oi, Configuration jc)
- throws HiveException {
- this(blockSize, jc);
- setSerDe(sd, oi);
- }
-
public void setSerDe(SerDe sd, ObjectInspector oi) {
this.serde = sd;
this.standardOI = oi;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java?rev=1087154&r1=1087153&r2=1087154&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java Thu Mar 31 00:59:03 2011
@@ -94,7 +94,7 @@ public class ConditionalResolverSkewJoin
Path dirPath = new Path(path);
FileSystem inpFs = dirPath.getFileSystem(conf);
FileStatus[] fstatus = inpFs.listStatus(dirPath);
- if (fstatus.length > 0) {
+ if (fstatus != null && fstatus.length > 0) {
Task <? extends Serializable> task = entry.getValue();
List<Task <? extends Serializable>> parentOps = task.getParentTasks();
if(parentOps!=null){