You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2014/01/17 04:19:32 UTC
svn commit: r1559002 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/conf/
ql/src/test/results/clientpositive/
shims/0.20/src/main/java/org/apache/hadoop/hive/shims/
shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/
shims/0.23/src/main/j...
Author: thejas
Date: Fri Jan 17 03:19:30 2014
New Revision: 1559002
URL: http://svn.apache.org/r1559002
Log:
HIVE-6159 : Hive uses deprecated hadoop configuration in Hadoop 2.0 (shanyu zhao via Thejas Nair)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out
hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Jan 17 03:19:30 2014
@@ -146,7 +146,6 @@ public class HiveConf extends Configurat
*/
public static final HiveConf.ConfVars[] dbVars = {
HiveConf.ConfVars.HADOOPBIN,
- HiveConf.ConfVars.HADOOPJT,
HiveConf.ConfVars.METASTOREWAREHOUSE,
HiveConf.ConfVars.SCRATCHDIR
};
@@ -231,22 +230,23 @@ public class HiveConf extends Configurat
// a symbolic name to reference in the Hive source code. Properties with non-null
// values will override any values set in the underlying Hadoop configuration.
HADOOPBIN("hadoop.bin.path", findHadoopBinary()),
- HADOOPFS("fs.default.name", null),
HIVE_FS_HAR_IMPL("fs.har.impl", "org.apache.hadoop.hive.shims.HiveHarFileSystem"),
- HADOOPMAPFILENAME("map.input.file", null),
- HADOOPMAPREDINPUTDIR("mapred.input.dir", null),
- HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", false),
- HADOOPJT("mapred.job.tracker", null),
- MAPREDMAXSPLITSIZE("mapred.max.split.size", 256000000L),
- MAPREDMINSPLITSIZE("mapred.min.split.size", 1L),
- MAPREDMINSPLITSIZEPERNODE("mapred.min.split.size.per.rack", 1L),
- MAPREDMINSPLITSIZEPERRACK("mapred.min.split.size.per.node", 1L),
+ HADOOPFS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPFS"), null),
+ HADOOPMAPFILENAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPFILENAME"), null),
+ HADOOPMAPREDINPUTDIR(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIR"), null),
+ HADOOPMAPREDINPUTDIRRECURSIVE(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPMAPREDINPUTDIRRECURSIVE"), false),
+ MAPREDMAXSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 256000000L),
+ MAPREDMINSPLITSIZE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 1L),
+ MAPREDMINSPLITSIZEPERNODE(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 1L),
+ MAPREDMINSPLITSIZEPERRACK(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 1L),
// The number of reduce tasks per job. Hadoop sets this value to 1 by default
// By setting this property to -1, Hive will automatically determine the correct
// number of reducers.
- HADOOPNUMREDUCERS("mapred.reduce.tasks", -1),
- HADOOPJOBNAME("mapred.job.name", null),
- HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", true),
+ HADOOPNUMREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPNUMREDUCERS"), -1),
+ HADOOPJOBNAME(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPJOBNAME"), null),
+ HADOOPSPECULATIVEEXECREDUCERS(ShimLoader.getHadoopShims().getHadoopConfNames().get("HADOOPSPECULATIVEEXECREDUCERS"), true),
+ MAPREDSETUPCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false),
+ MAPREDTASKCLEANUPNEEDED(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false),
// Metastore stuff. Be sure to update HiveConf.metaVars when you add
// something here!
Modified: hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out Fri Jan 17 03:19:30 2014
@@ -2,7 +2,6 @@ PREHOOK: query: select count(*) from src
PREHOOK: type: QUERY
PREHOOK: Input: default@src
#### A masked pattern was here ####
-Key: mapred.job.tracker, Value: local
Key: hive.exec.post.hooks, Value: org.apache.hadoop.hive.ql.hooks.VerifyOverriddenConfigsHook
Key: hive.config.doesnt.exit, Value: abc
500
Modified: hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Fri Jan 17 03:19:30 2014
@@ -34,6 +34,8 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.HashMap;
+import java.util.Map;
import javax.security.auth.Subject;
import javax.security.auth.login.LoginException;
@@ -753,4 +755,22 @@ public class Hadoop20Shims implements Ha
public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
return new ProxyFileSystem(fs, uri);
}
+ @Override
+ public Map<String, String> getHadoopConfNames() {
+ Map<String, String> ret = new HashMap<String, String>();
+ ret.put("HADOOPFS", "fs.default.name");
+ ret.put("HADOOPMAPFILENAME", "map.input.file");
+ ret.put("HADOOPMAPREDINPUTDIR", "mapred.input.dir");
+ ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapred.input.dir.recursive");
+ ret.put("MAPREDMAXSPLITSIZE", "mapred.max.split.size");
+ ret.put("MAPREDMINSPLITSIZE", "mapred.min.split.size");
+ ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.rack");
+ ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.node");
+ ret.put("HADOOPNUMREDUCERS", "mapred.reduce.tasks");
+ ret.put("HADOOPJOBNAME", "mapred.job.name");
+ ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution");
+ ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed");
+ ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed");
+ return ret;
+ }
}
Modified: hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Fri Jan 17 03:19:30 2014
@@ -24,6 +24,8 @@ import java.net.URL;
import java.util.Comparator;
import java.util.Iterator;
import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -390,4 +392,22 @@ public class Hadoop20SShims extends Hado
public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
return new ProxyFileSystem(fs, uri);
}
+ @Override
+ public Map<String, String> getHadoopConfNames() {
+ Map<String, String> ret = new HashMap<String, String>();
+ ret.put("HADOOPFS", "fs.default.name");
+ ret.put("HADOOPMAPFILENAME", "map.input.file");
+ ret.put("HADOOPMAPREDINPUTDIR", "mapred.input.dir");
+ ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapred.input.dir.recursive");
+ ret.put("MAPREDMAXSPLITSIZE", "mapred.max.split.size");
+ ret.put("MAPREDMINSPLITSIZE", "mapred.min.split.size");
+ ret.put("MAPREDMINSPLITSIZEPERNODE", "mapred.min.split.size.per.rack");
+ ret.put("MAPREDMINSPLITSIZEPERRACK", "mapred.min.split.size.per.node");
+ ret.put("HADOOPNUMREDUCERS", "mapred.reduce.tasks");
+ ret.put("HADOOPJOBNAME", "mapred.job.name");
+ ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapred.reduce.tasks.speculative.execution");
+ ret.put("MAPREDSETUPCLEANUPNEEDED", "mapred.committer.job.setup.cleanup.needed");
+ ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed");
+ return ret;
+ }
}
Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Fri Jan 17 03:19:30 2014
@@ -25,6 +25,7 @@ import java.net.URL;
import java.util.Comparator;
import java.util.Iterator;
import java.util.Map;
+import java.util.HashMap;
import java.net.URI;
import java.io.FileNotFoundException;
@@ -537,4 +538,23 @@ public class Hadoop23Shims extends Hadoo
public FileSystem createProxyFileSystem(FileSystem fs, URI uri) {
return new ProxyFileSystem23(fs, uri);
}
+
+ @Override
+ public Map<String, String> getHadoopConfNames() {
+ Map<String, String> ret = new HashMap<String, String>();
+ ret.put("HADOOPFS", "fs.defaultFS");
+ ret.put("HADOOPMAPFILENAME", "mapreduce.map.input.file");
+ ret.put("HADOOPMAPREDINPUTDIR", "mapreduce.input.fileinputformat.inputdir");
+ ret.put("HADOOPMAPREDINPUTDIRRECURSIVE", "mapreduce.input.fileinputformat.input.dir.recursive");
+ ret.put("MAPREDMAXSPLITSIZE", "mapreduce.input.fileinputformat.split.maxsize");
+ ret.put("MAPREDMINSPLITSIZE", "mapreduce.input.fileinputformat.split.minsize");
+ ret.put("MAPREDMINSPLITSIZEPERNODE", "mapreduce.input.fileinputformat.split.minsize.per.rack");
+ ret.put("MAPREDMINSPLITSIZEPERRACK", "mapreduce.input.fileinputformat.split.minsize.per.node");
+ ret.put("HADOOPNUMREDUCERS", "mapreduce.job.reduces");
+ ret.put("HADOOPJOBNAME", "mapreduce.job.name");
+ ret.put("HADOOPSPECULATIVEEXECREDUCERS", "mapreduce.reduce.speculative");
+ ret.put("MAPREDSETUPCLEANUPNEEDED", "mapreduce.job.committer.setup.cleanup.needed");
+ ret.put("MAPREDTASKCLEANUPNEEDED", "mapreduce.job.committer.task.cleanup.needed");
+ return ret;
+ }
}
Modified: hive/trunk/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/trunk/shims/common-secure/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Fri Jan 17 03:19:30 2014
@@ -39,6 +39,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
import org.apache.hadoop.http.HtmlQuoting;
@@ -324,18 +325,18 @@ public abstract class HadoopShimsSecure
@Override
public InputSplitShim[] getSplits(JobConf job, int numSplits) throws IOException {
- long minSize = job.getLong("mapred.min.split.size", 0);
+ long minSize = job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZE"), 0);
// For backward compatibility, let the above parameter be used
- if (job.getLong("mapred.min.split.size.per.node", 0) == 0) {
+ if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERNODE"), 0) == 0) {
super.setMinSplitSizeNode(minSize);
}
- if (job.getLong("mapred.min.split.size.per.rack", 0) == 0) {
+ if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMINSPLITSIZEPERRACK"), 0) == 0) {
super.setMinSplitSizeRack(minSize);
}
- if (job.getLong("mapred.max.split.size", 0) == 0) {
+ if (job.getLong(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDMAXSPLITSIZE"), 0) == 0) {
super.setMaxSplitSize(minSize);
}
@@ -426,11 +427,11 @@ public abstract class HadoopShimsSecure
// option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
- conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
+ conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDSETUPCLEANUPNEEDED"), false);
// option to bypass task cleanup task was introduced in hadoop-23 (MAPREDUCE-2206)
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
- conf.setBoolean("mapreduce.job.committer.task.cleanup.needed", false);
+ conf.setBoolean(ShimLoader.getHadoopShims().getHadoopConfNames().get("MAPREDTASKCLEANUPNEEDED"), false);
}
@Override
Modified: hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1559002&r1=1559001&r2=1559002&view=diff
==============================================================================
--- hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Fri Jan 17 03:19:30 2014
@@ -28,6 +28,7 @@ import java.security.PrivilegedException
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import javax.security.auth.login.LoginException;
@@ -517,4 +518,6 @@ public interface HadoopShims {
* other file system.
*/
public FileSystem createProxyFileSystem(FileSystem fs, URI uri);
+
+ public Map<String, String> getHadoopConfNames();
}