You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2013/07/23 01:47:04 UTC
svn commit: r1505854 - in /hive/trunk/hcatalog:
core/src/main/java/org/apache/hcatalog/shims/
shims/src/20/java/org/apache/hadoop/mapred/
shims/src/20/java/org/apache/hcatalog/shims/
shims/src/23/java/org/apache/hadoop/mapred/ shims/src/23/java/org/apa...
Author: gates
Date: Mon Jul 22 23:47:03 2013
New Revision: 1505854
URL: http://svn.apache.org/r1505854
Log:
HIVE-4677 WebHCat e2e tests fail on Hadoop 2 (daijy via gates)
Modified:
hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java
hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
Modified: hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java (original)
+++ hive/trunk/hcatalog/core/src/main/java/org/apache/hcatalog/shims/HCatHadoopShims.java Mon Jul 22 23:47:03 2013
@@ -101,4 +101,6 @@ public interface HCatHadoopShims {
*/
public boolean isFileInHDFS(FileSystem fs, Path path) throws IOException;
+ public InetSocketAddress getAddress(Configuration conf);
+
}
Modified: hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java (original)
+++ hive/trunk/hcatalog/shims/src/20/java/org/apache/hadoop/mapred/TempletonJobTracker.java Mon Jul 22 23:47:03 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.shims.HCatHadoopShims;
/*
* Communicate with the JobTracker as a specific user.
@@ -35,14 +36,13 @@ public class TempletonJobTracker {
/**
* Create a connection to the Job Tracker.
*/
- public TempletonJobTracker(InetSocketAddress addr,
- Configuration conf)
+ public TempletonJobTracker(Configuration conf)
throws IOException {
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
cnx = (JobSubmissionProtocol)
RPC.getProxy(JobSubmissionProtocol.class,
JobSubmissionProtocol.versionID,
- addr,
+ HCatHadoopShims.Instance.get().getAddress(conf),
ugi,
conf,
NetUtils.getSocketFactory(conf,
Modified: hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java (original)
+++ hive/trunk/hcatalog/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java Mon Jul 22 23:47:03 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.Outpu
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.Progressable;
public class HCatHadoopShims20S implements HCatHadoopShims {
@@ -149,4 +150,11 @@ public class HCatHadoopShims20S implemen
// In hadoop 1.x.x the file system URI is sufficient to determine the uri of the file
return "hdfs".equals(fs.getUri().getScheme());
}
+
+ @Override
+ public InetSocketAddress getAddress(Configuration conf) {
+ String jobTrackerStr =
+ conf.get("mapred.job.tracker", "localhost:8012");
+ return NetUtils.createSocketAddr(jobTrackerStr);
+ }
}
Modified: hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java (original)
+++ hive/trunk/hcatalog/shims/src/23/java/org/apache/hadoop/mapred/TempletonJobTracker.java Mon Jul 22 23:47:03 2013
@@ -32,8 +32,7 @@ public class TempletonJobTracker {
/**
* Create a connection to the Job Tracker.
*/
- public TempletonJobTracker(InetSocketAddress addr,
- Configuration conf)
+ public TempletonJobTracker(Configuration conf)
throws IOException {
jc = new JobClient(conf);
Modified: hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java (original)
+++ hive/trunk/hcatalog/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java Mon Jul 22 23:47:03 2013
@@ -127,4 +127,9 @@ public class HCatHadoopShims23 implement
// resolvePath is a sure shot way of knowing which file system the file is.
return "hdfs".equals(fs.resolvePath(path).toUri().getScheme());
}
+
+ @Override
+ public InetSocketAddress getAddress(Configuration conf) {
+ return null;
+ }
}
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/DeleteDelegator.java Mon Jul 22 23:47:03 2013
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.TempletonJobTracker;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.shims.HCatHadoopShims;
import org.apache.hcatalog.templeton.tool.JobState;
/**
@@ -39,8 +40,7 @@ public class DeleteDelegator extends Tem
TempletonJobTracker tracker = null;
JobState state = null;
try {
- tracker = new TempletonJobTracker(getAddress(appConf),
- appConf);
+ tracker = new TempletonJobTracker(appConf);
JobID jobid = StatusDelegator.StringToJobID(id);
if (jobid == null)
throw new BadParam("Invalid jobid: " + id);
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/ListDelegator.java Mon Jul 22 23:47:03 2013
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import org.apache.hadoop.mapred.JobStatus;
import org.apache.hadoop.mapred.TempletonJobTracker;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hcatalog.shims.HCatHadoopShims;
import org.apache.hcatalog.templeton.tool.JobState;
/**
@@ -41,8 +42,7 @@ public class ListDelegator extends Templ
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
TempletonJobTracker tracker = null;
try {
- tracker = new TempletonJobTracker(getAddress(appConf),
- appConf);
+ tracker = new TempletonJobTracker(appConf);
ArrayList<String> ids = new ArrayList<String>();
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StatusDelegator.java Mon Jul 22 23:47:03 2013
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.JobProfile;
import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hcatalog.shims.HCatHadoopShims;
import org.apache.hadoop.mapred.TempletonJobTracker;
import org.apache.hcatalog.templeton.tool.JobState;
@@ -44,8 +45,7 @@ public class StatusDelegator extends Tem
TempletonJobTracker tracker = null;
JobState state = null;
try {
- tracker = new TempletonJobTracker(getAddress(appConf),
- appConf);
+ tracker = new TempletonJobTracker(appConf);
JobID jobid = StatusDelegator.StringToJobID(id);
if (jobid == null)
throw new BadParam("Invalid jobid: " + id);
Modified: hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java?rev=1505854&r1=1505853&r2=1505854&view=diff
==============================================================================
--- hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java (original)
+++ hive/trunk/hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/TempletonDelegator.java Mon Jul 22 23:47:03 2013
@@ -21,7 +21,6 @@ package org.apache.hcatalog.templeton;
import java.net.InetSocketAddress;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
/**
* The helper class for all the Templeton delegator classes. A
@@ -34,10 +33,4 @@ public class TempletonDelegator {
public TempletonDelegator(AppConfig appConf) {
this.appConf = appConf;
}
-
- public static InetSocketAddress getAddress(Configuration conf) {
- String jobTrackerStr =
- conf.get("mapred.job.tracker", "localhost:8012");
- return NetUtils.createSocketAddr(jobTrackerStr);
- }
}