You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/10/28 19:43:05 UTC
svn commit: r1536478 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql:
exec/ exec/mr/ io/
Author: hashutosh
Date: Mon Oct 28 18:43:05 2013
New Revision: 1536478
URL: http://svn.apache.org/r1536478
Log:
HIVE-5666 : use Path instead of String for IOContext.inputPath (Thejas Nair via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Mon Oct 28 18:43:05 2013
@@ -120,7 +120,7 @@ public class MapJoinOperator extends Abs
}
String baseDir = null;
- String currentInputFile = getExecContext().getCurrentInputFile();
+ String currentInputFile = getExecContext().getCurrentInputPath().toString();
LOG.info("******* Load from HashTable File: input : " + currentInputFile);
String fileName = getExecContext().getLocalWork().getBucketFileName(currentInputFile);
try {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java Mon Oct 28 18:43:05 2013
@@ -441,7 +441,7 @@ public class MapOperator extends Operato
// Find context for current input file
@Override
public void cleanUpInputFileChangedOp() throws HiveException {
- Path fpath = normalizePath(getExecContext().getCurrentInputFile());
+ Path fpath = getExecContext().getCurrentInputPath();
for (String onefile : conf.getPathToAliases().keySet()) {
Path onepath = normalizePath(onefile);
@@ -537,7 +537,7 @@ public class MapOperator extends Operato
VirtualColumn vc = vcs.get(i);
if (vc.equals(VirtualColumn.FILENAME)) {
if (ctx.inputFileChanged()) {
- vcValues[i] = new Text(ctx.getCurrentInputFile());
+ vcValues[i] = new Text(ctx.getCurrentInputPath().toString());
}
} else if (vc.equals(VirtualColumn.BLOCKOFFSET)) {
long current = ctx.getIoCxt().getCurrentBlockStart();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Mon Oct 28 18:43:05 2013
@@ -515,19 +515,19 @@ public class SMBMapJoinOperator extends
private void setUpFetchContexts(String alias, MergeQueue mergeQueue) throws HiveException {
mergeQueue.clearFetchContext();
- String currentInputFile = getExecContext().getCurrentInputFile();
+ Path currentInputPath = getExecContext().getCurrentInputPath();
BucketMapJoinContext bucketMatcherCxt = localWork.getBucketMapjoinContext();
Class<? extends BucketMatcher> bucketMatcherCls = bucketMatcherCxt.getBucketMatcherClass();
BucketMatcher bucketMatcher = ReflectionUtils.newInstance(bucketMatcherCls, null);
- getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputFile));
+ getExecContext().setFileId(bucketMatcherCxt.createFileId(currentInputPath.toString()));
LOG.info("set task id: " + getExecContext().getFileId());
bucketMatcher.setAliasBucketFileNameMapping(bucketMatcherCxt
.getAliasBucketFileNameMapping());
- List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputFile,
+ List<Path> aliasFiles = bucketMatcher.getAliasBucketFiles(currentInputPath.toString(),
bucketMatcherCxt.getMapJoinBigTableAlias(), alias);
mergeQueue.setupContext(aliasFiles);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java Mon Oct 28 18:43:05 2013
@@ -105,9 +105,9 @@ public class TableScanOperator extends O
Map<String, Integer> bucketNameMapping =
(conf != null) ? conf.getBucketFileNameMapping() : null;
if ((bucketNameMapping != null) && (!bucketNameMapping.isEmpty())) {
- String currentInputFile = getExecContext().getCurrentInputFile();
+ Path currentInputPath = getExecContext().getCurrentInputPath();
getExecContext().setFileId(Integer.toString(bucketNameMapping.get(
- Utilities.getFileNameFromDirName(currentInputFile))));
+ currentInputPath.getName())));
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecMapperContext.java Mon Oct 28 18:43:05 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.m
import java.util.Map;
import org.apache.commons.logging.Log;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.exec.FetchOperator;
import org.apache.hadoop.hive.ql.io.IOContext;
import org.apache.hadoop.hive.ql.plan.MapredLocalWork;
@@ -29,14 +30,14 @@ public class ExecMapperContext {
public static final Log l4j = ExecMapper.l4j;
- // lastInputFile should be changed by the root of the operator tree ExecMapper.map()
+ // lastInputPath should be changed by the root of the operator tree ExecMapper.map()
// but kept unchanged throughout the operator tree for one row
- private String lastInputFile = null;
+ private Path lastInputPath = null;
// currentInputFile will be updated only by inputFileChanged(). If inputFileChanged()
- // is not called throughout the opertor tree, currentInputFile won't be used anyways
+ // is not called throughout the operator tree, currentInputPath won't be used anyways
// so it won't be updated.
- private String currentInputFile = null;
+ private Path currentInputPath = null;
private boolean inputFileChecked = false;
// for SMB join, replaced with number part of task-id , making output file name
@@ -80,10 +81,10 @@ public class ExecMapperContext {
*/
public boolean inputFileChanged() {
if (!inputFileChecked) {
- currentInputFile = this.ioCxt.getInputFile();
+ currentInputPath = this.ioCxt.getInputPath();
inputFileChecked = true;
}
- return lastInputFile == null || !lastInputFile.equals(currentInputFile);
+ return lastInputPath == null || !lastInputPath.equals(currentInputPath);
}
/**
@@ -93,25 +94,25 @@ public class ExecMapperContext {
*/
public void resetRow() {
// Update the lastInputFile with the currentInputFile.
- lastInputFile = currentInputFile;
+ lastInputPath = currentInputPath;
inputFileChecked = false;
}
- public String getLastInputFile() {
- return lastInputFile;
+ public Path getLastInputPath() {
+ return lastInputPath;
}
- public void setLastInputFile(String lastInputFile) {
- this.lastInputFile = lastInputFile;
+ public void setLastInputPath(Path lastInputPath) {
+ this.lastInputPath = lastInputPath;
}
- public String getCurrentInputFile() {
- currentInputFile = this.ioCxt.getInputFile();
- return currentInputFile;
+ public Path getCurrentInputPath() {
+ currentInputPath = this.ioCxt.getInputPath();
+ return currentInputPath;
}
- public void setCurrentInputFile(String currentInputFile) {
- this.currentInputFile = currentInputFile;
+ public void setCurrentInputPath(Path currentInputPath) {
+ this.currentInputPath = currentInputPath;
}
public JobConf getJc() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java Mon Oct 28 18:43:05 2013
@@ -146,12 +146,12 @@ public abstract class HiveContextAwareRe
return IOContext.get();
}
- public void initIOContext(long startPos, boolean isBlockPointer, String inputFile) {
+ public void initIOContext(long startPos, boolean isBlockPointer, Path inputPath) {
ioCxtRef = this.getIOContext();
ioCxtRef.currentBlockStart = startPos;
ioCxtRef.isBlockPointer = isBlockPointer;
- ioCxtRef.inputFile = inputFile;
- LOG.info("Processing file " + inputFile);
+ ioCxtRef.inputPath = inputPath;
+ LOG.info("Processing file " + inputPath);
initDone = true;
}
@@ -184,7 +184,7 @@ public abstract class HiveContextAwareRe
blockStart = in.getPosition();
in.close();
}
- this.initIOContext(blockStart, blockPointer, path.makeQualified(fs).toString());
+ this.initIOContext(blockStart, blockPointer, path.makeQualified(fs));
this.initIOContextSortedProps(split, recordReader, job);
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java?rev=1536478&r1=1536477&r2=1536478&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java Mon Oct 28 18:43:05 2013
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.ql.io;
+import org.apache.hadoop.fs.Path;
+
/**
* IOContext basically contains the position information of the current
@@ -67,7 +69,7 @@ public class IOContext {
UNKNOWN
}
- String inputFile;
+ Path inputPath;
public IOContext() {
this.currentBlockStart = 0;
@@ -109,12 +111,12 @@ public class IOContext {
this.isBlockPointer = isBlockPointer;
}
- public String getInputFile() {
- return inputFile;
+ public Path getInputPath() {
+ return inputPath;
}
- public void setInputFile(String inputFile) {
- this.inputFile = inputFile;
+ public void setInputPath(Path inputPath) {
+ this.inputPath = inputPath;
}
public void setIOExceptions(boolean ioe) {