You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2011/12/07 02:29:07 UTC
svn commit: r1211261 - in /hive/trunk: ./ common/
common/src/java/org/apache/hadoop/hive/conf/ common/src/test/
common/src/test/org/ common/src/test/org/apache/
common/src/test/org/apache/hadoop/ common/src/test/org/apache/hadoop/hive/
common/src/test/...
Author: jvs
Date: Wed Dec 7 01:29:06 2011
New Revision: 1211261
URL: http://svn.apache.org/viewvc?rev=1211261&view=rev
Log:
HIVE-2362. HiveConf properties not appearing in the output of 'set' or 'set -v'
(Carl Steinbach via jvs)
Added:
hive/trunk/common/src/test/
hive/trunk/common/src/test/org/
hive/trunk/common/src/test/org/apache/
hive/trunk/common/src/test/org/apache/hadoop/
hive/trunk/common/src/test/org/apache/hadoop/hive/
hive/trunk/common/src/test/org/apache/hadoop/hive/conf/
hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
hive/trunk/common/src/test/resources/
hive/trunk/common/src/test/resources/hive-site.xml
Modified:
hive/trunk/build-common.xml
hive/trunk/common/build.xml
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/eclipse-templates/.classpath
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1211261&r1=1211260&r2=1211261&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Wed Dec 7 01:29:06 2011
@@ -45,11 +45,13 @@
<!-- configuration needed for tests -->
<property name="test.src.dir" value="${basedir}/src/test"/>
<property name="test.src.data.dir" value="${hive.root}/data"/>
+ <property name="test.resources.dir" value="${basedir}/src/test/resources"/>
<property name="test.build.dir" value="${build.dir}/test"/>
<property name="test.log.dir" value="${test.build.dir}/logs"/>
<property name="test.data.dir" value="${test.build.dir}/data"/>
<property name="test.build.src" value="${test.build.dir}/src"/>
<property name="test.build.classes" value="${test.build.dir}/classes"/>
+ <property name="test.build.resources" value="${test.build.dir}/resources"/>
<property name="test.include" value="Test*"/>
<property name="test.classpath.id" value="test.classpath"/>
<property name="test.output" value="true"/>
@@ -64,6 +66,7 @@
<path id="test.classpath">
<pathelement location="${test.build.classes}" />
+ <pathelement location="${test.build.resources}" />
<pathelement location="" />
<pathelement location="${test.src.data.dir}/conf"/>
<pathelement location="${hive.conf.dir}"/>
@@ -189,6 +192,10 @@
<mkdir dir="${test.build.dir}"/>
<mkdir dir="${test.build.src}"/>
<mkdir dir="${test.build.classes}"/>
+ <mkdir dir="${test.build.resources}"/>
+ <copy todir="${test.build.resources}" failonerror="false">
+ <fileset dir="${test.resources.dir}"/>
+ </copy>
</target>
<target name="init" depends="create-dirs">
@@ -369,6 +376,7 @@
<sysproperty key="test.tmp.dir" value="${build.dir}/tmp"/>
<sysproperty key="test.src.data.dir" value="${test.src.data.dir}"/>
<sysproperty key="test.warehouse.dir" value="${test.warehouse.dir}"/>
+ <sysproperty key="test.build.resources" value="${test.build.resources}"/>
<sysproperty key="mapred.job.tracker" value="${mapred.job.tracker}"/>
<sysproperty key="fs.default.name" value="${fs.default.name}"/>
<sysproperty key="build.dir" value="${build.dir}"/>
Modified: hive/trunk/common/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/build.xml?rev=1211261&r1=1211260&r2=1211261&view=diff
==============================================================================
--- hive/trunk/common/build.xml (original)
+++ hive/trunk/common/build.xml Wed Dec 7 01:29:06 2011
@@ -44,10 +44,5 @@ to call at top-level: ant deploy-contrib
<fileset dir="${src.dir}/conf"/>
</copy>
</target>
-
- <target name="test">
- <echo message="Project: ${ant.project.name}"/>
- <echo message="Nothing to do!"/>
- </target>
-
+
</project>
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1211261&r1=1211260&r2=1211261&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Dec 7 01:29:06 2011
@@ -18,6 +18,8 @@
package org.apache.hadoop.hive.conf;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URL;
@@ -44,6 +46,30 @@ public class HiveConf extends Configurat
protected Properties origProp;
protected String auxJars;
private static final Log l4j = LogFactory.getLog(HiveConf.class);
+ private static URL hiveSiteURL = null;
+ private static URL confVarURL = null;
+
+ static {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = HiveConf.class.getClassLoader();
+ }
+
+ // Log a warning if hive-default.xml is found on the classpath
+ URL hiveDefaultURL = classLoader.getResource("hive-default.xml");
+ if (hiveDefaultURL != null) {
+ l4j.warn("DEPRECATED: Ignoring hive-default.xml found on the CLASSPATH at " +
+ hiveDefaultURL.getPath());
+ }
+
+ // Look for hive-site.xml on the CLASSPATH and log its location if found.
+ hiveSiteURL = classLoader.getResource("hive-site.xml");
+ if (hiveSiteURL == null) {
+ l4j.warn("hive-site.xml not found on CLASSPATH");
+ } else {
+ l4j.debug("Using hive-site.xml found on CLASSPATH at " + hiveSiteURL.getPath());
+ }
+ }
/**
* Metastore related options that the db is initialized against. When a conf
@@ -115,8 +141,8 @@ public class HiveConf extends Configurat
*/
public static enum ConfVars {
// QL execution stuff
- SCRIPTWRAPPER("hive.exec.script.wrapper", null),
- PLAN("hive.exec.plan", null),
+ SCRIPTWRAPPER("hive.exec.script.wrapper", ""),
+ PLAN("hive.exec.plan", ""),
SCRATCHDIR("hive.exec.scratchdir", "/tmp/hive-" + System.getProperty("user.name")),
SUBMITVIACHILD("hive.exec.submitviachild", false),
SCRIPTERRORLIMIT("hive.exec.script.maxerrsize", 100000),
@@ -147,7 +173,7 @@ public class HiveConf extends Configurat
SHOW_JOB_FAIL_DEBUG_INFO("hive.exec.show.job.failure.debug.info", true),
JOB_DEBUG_TIMEOUT("hive.exec.job.debug.timeout", 30000),
TASKLOG_DEBUG_TIMEOUT("hive.exec.tasklog.debug.timeout", 20000),
- OUTPUT_FILE_EXTENSION("hive.output.file.extension", null),
+ OUTPUT_FILE_EXTENSION("hive.output.file.extension", ""),
// should hive determine whether to run in local mode automatically ?
LOCALMODEAUTO("hive.exec.mode.local.auto", false),
@@ -165,12 +191,12 @@ public class HiveConf extends Configurat
HADOOPBIN("hadoop.bin.path", System.getenv("HADOOP_HOME") + "/bin/hadoop"),
HADOOPCONF("hadoop.config.dir", System.getenv("HADOOP_HOME") + "/conf"),
HADOOPFS("fs.default.name", "file:///"),
- HADOOPMAPFILENAME("map.input.file", null),
- HADOOPMAPREDINPUTDIR("mapred.input.dir", null),
+ HADOOPMAPFILENAME("map.input.file", ""),
+ HADOOPMAPREDINPUTDIR("mapred.input.dir", ""),
HADOOPMAPREDINPUTDIRRECURSIVE("mapred.input.dir.recursive", false),
HADOOPJT("mapred.job.tracker", "local"),
HADOOPNUMREDUCERS("mapred.reduce.tasks", -1),
- HADOOPJOBNAME("mapred.job.name", null),
+ HADOOPJOBNAME("mapred.job.name", ""),
HADOOPSPECULATIVEEXECREDUCERS("mapred.reduce.tasks.speculative.execution", false),
// Metastore stuff. Be sure to update HiveConf.metaVars when you add
@@ -325,7 +351,9 @@ public class HiveConf extends Configurat
// HWI
HIVEHWILISTENHOST("hive.hwi.listen.host", "0.0.0.0"),
HIVEHWILISTENPORT("hive.hwi.listen.port", "9999"),
- HIVEHWIWARFILE("hive.hwi.war.file", System.getenv("HWI_WAR_FILE")),
+ HIVEHWIWARFILE("hive.hwi.war.file",
+ (System.getenv("HWI_WAR_FILE") != null) ?
+ System.getenv("HWI_WAR_FILE") : ""),
// mapper/reducer memory in local mode
HIVEHADOOPMAXMEM("hive.mapred.local.mem", 0),
@@ -468,7 +496,7 @@ public class HiveConf extends Configurat
// Hive Variables
HIVEVARIABLESUBSTITUTE("hive.variable.substitute", true),
- SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook",null),
+ SEMANTIC_ANALYZER_HOOK("hive.semantic.analyzer.hook", ""),
HIVE_AUTHORIZATION_ENABLED("hive.security.authorization.enabled", false),
HIVE_AUTHORIZATION_MANAGER("hive.security.authorization.manager",
@@ -576,6 +604,33 @@ public class HiveConf extends Configurat
}
}
+ /**
+ * Writes the default ConfVars out to a temporary File and returns
+ * a URL pointing to the temporary file.
+ * We need this in order to initialize the ConfVar properties
+ * in the underling Configuration object using the addResource()
+ * method.
+ */
+ private static synchronized URL getConfVarURL() {
+ if (confVarURL == null) {
+ try {
+ File confVarFile = File.createTempFile("hive-default-", ".xml");
+ Configuration conf = new Configuration(false);
+
+ applyDefaultConfVars(conf);
+
+ FileOutputStream fout = new FileOutputStream(confVarFile);
+ conf.writeXml(fout);
+ fout.close();
+ confVarURL = confVarFile.toURI().toURL();
+ } catch (Exception e) {
+ // We're pretty screwed if we can't load the default conf vars
+ throw new RuntimeException(e);
+ }
+ }
+ return confVarURL;
+ }
+
public static int getIntVar(Configuration conf, ConfVars var) {
assert (var.valClass == Integer.class);
return conf.getInt(var.varname, var.defaultIntVal);
@@ -674,6 +729,7 @@ public class HiveConf extends Configurat
public HiveConf() {
super();
+ initialize(this.getClass());
}
public HiveConf(Class<?> cls) {
@@ -696,8 +752,12 @@ public class HiveConf extends Configurat
origProp = (Properties)other.origProp.clone();
}
- private Properties getUnderlyingProps() {
- Iterator<Map.Entry<String, String>> iter = this.iterator();
+ public Properties getAllProperties() {
+ return getProperties(this);
+ }
+
+ private static Properties getProperties(Configuration conf) {
+ Iterator<Map.Entry<String, String>> iter = conf.iterator();
Properties p = new Properties();
while (iter.hasNext()) {
Map.Entry<String, String> e = iter.next();
@@ -710,33 +770,25 @@ public class HiveConf extends Configurat
hiveJar = (new JobConf(cls)).getJar();
// preserve the original configuration
- origProp = getUnderlyingProps();
+ origProp = getAllProperties();
- // let's add the hive configuration
- URL hconfurl = getClassLoader().getResource("hive-default.xml");
- if (hconfurl == null) {
- l4j.debug("hive-default.xml not found.");
- } else {
- addResource(hconfurl);
- }
- URL hsiteurl = getClassLoader().getResource("hive-site.xml");
- if (hsiteurl == null) {
- l4j.debug("hive-site.xml not found.");
- } else {
- addResource(hsiteurl);
+ // Overlay the default ConfVars
+ addResource(getConfVarURL());
+
+ // Overlay hive-site.xml if it exists
+ if (hiveSiteURL != null) {
+ addResource(hiveSiteURL);
}
// if hadoop configuration files are already in our path - then define
// the containing directory as the configuration directory
- URL hadoopconfurl = getClassLoader().getResource("hadoop-default.xml");
- if (hadoopconfurl == null) {
- hadoopconfurl = getClassLoader().getResource("hadoop-site.xml");
- }
+ URL hadoopconfurl = getClassLoader().getResource("core-site.xml");
if (hadoopconfurl != null) {
String conffile = hadoopconfurl.getPath();
this.setVar(ConfVars.HADOOPCONF, conffile.substring(0, conffile.lastIndexOf('/')));
}
+ // Overlay system properties
applySystemProperties();
// if the running class was loaded directly (through eclipse) rather than through a
@@ -748,10 +800,9 @@ public class HiveConf extends Configurat
if (auxJars == null) {
auxJars = this.get(ConfVars.HIVEAUXJARS.varname);
}
-
}
- public void applySystemProperties() {
+ private void applySystemProperties() {
for (ConfVars oneVar : ConfVars.values()) {
if (System.getProperty(oneVar.varname) != null) {
if (System.getProperty(oneVar.varname).length() > 0) {
@@ -761,9 +812,27 @@ public class HiveConf extends Configurat
}
}
+ private static void applyDefaultConfVars(Configuration conf) {
+ for (ConfVars var : ConfVars.values()) {
+ if (String.class.equals(var.valClass)) {
+ conf.set(var.varname, var.defaultVal);
+ } else if (Integer.class.equals(var.valClass)) {
+ conf.setInt(var.varname, var.defaultIntVal);
+ } else if (Long.class.equals(var.valClass)) {
+ conf.setLong(var.varname, var.defaultLongVal);
+ } else if (Float.class.equals(var.valClass)) {
+ conf.setFloat(var.varname, var.defaultFloatVal);
+ } else if (Boolean.class.equals(var.valClass)) {
+ conf.setBoolean(var.varname, var.defaultBoolVal);
+ } else {
+ l4j.warn("Unable to set default configuration value for " + var.varname);
+ }
+ }
+ }
+
public Properties getChangedProperties() {
Properties ret = new Properties();
- Properties newProp = getUnderlyingProps();
+ Properties newProp = getAllProperties();
for (Object one : newProp.keySet()) {
String oneProp = (String) one;
@@ -775,8 +844,8 @@ public class HiveConf extends Configurat
return (ret);
}
- public Properties getAllProperties() {
- return getUnderlyingProps();
+ public String getHiveSitePath() {
+ return hiveSiteURL.getPath();
}
public String getJar() {
@@ -824,5 +893,4 @@ public class HiveConf extends Configurat
return -1;
}
}
-
}
Added: hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java?rev=1211261&view=auto
==============================================================================
--- hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java (added)
+++ hive/trunk/common/src/test/org/apache/hadoop/hive/conf/TestHiveConf.java Wed Dec 7 01:29:06 2011
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.conf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+
+
+/**
+ * TestHiveConf
+ *
+ */
+public class TestHiveConf extends TestCase {
+
+ public void testHiveSitePath() throws Exception {
+ String expectedPath = System.getProperty("test.build.resources") + "/hive-site.xml";
+ assertEquals(expectedPath, new HiveConf().getHiveSitePath());
+ }
+
+ public void testHiveSiteProperties() throws Exception {
+ HiveConf conf = new HiveConf();
+
+ // ConfVar only defined in HiveConf
+ assertEquals(ConfVars.HIVESKEWJOINKEY.defaultIntVal, conf.getIntVar(ConfVars.HIVESKEWJOINKEY));
+
+ // ConfVar overridden in local hive-site.xml
+ assertEquals("hive-site.xml", conf.get("javax.jdo.option.ConnectionDriverName"));
+
+ // Hadoop property overridden in ConfVars and hive-site.xml
+ assertEquals("hive-site.xml", conf.get("mapred.reduce.tasks"));
+
+ // Test property defined in hive-site.xml only
+ assertEquals("hive-site.xml", conf.get("test.property1"));
+
+ // Test Hive conf property variable substitution in hive-site.xml
+ assertEquals(conf.get("hive.exec.default.partition.name"), conf.get("test.var.hiveconf.property"));
+ }
+}
Added: hive/trunk/common/src/test/resources/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/test/resources/hive-site.xml?rev=1211261&view=auto
==============================================================================
--- hive/trunk/common/src/test/resources/hive-site.xml (added)
+++ hive/trunk/common/src/test/resources/hive-site.xml Wed Dec 7 01:29:06 2011
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+<property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>hive-site.xml</value>
+ <description>Override ConfVar defined in HiveConf</description>
+</property>
+
+<property>
+ <name>mapred.reduce.tasks</name>
+ <value>hive-site.xml</value>
+ <description>Override mapred ConfVar defined in HiveConf</description>
+</property>
+
+<property>
+ <name>test.property1</name>
+ <value>hive-site.xml</value>
+ <description>Test property defined in hive-site.xml only</description>
+</property>
+
+<property>
+ <name>test.var.hiveconf.property</name>
+ <value>${hive.exec.default.partition.name}</value>
+ <description>Test hiveconf property substitution</description>
+</property>
+
+</configuration>
Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1211261&r1=1211260&r2=1211261&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Wed Dec 7 01:29:06 2011
@@ -19,12 +19,11 @@
<configuration>
-<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
-<!-- that are implied by Hadoop setup variables. -->
-<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
-<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
-<!-- resource). -->
+<!-- WARNING!!! This file is provided for documentation purposes ONLY! -->
+<!-- WARNING!!! Any changes you make to this file will be ignored by Hive. -->
+<!-- WARNING!!! You must make your changes in hive-site.xml instead. -->
+
<!-- Hive Execution Parameters -->
<property>
<name>mapred.reduce.tasks</name>
Modified: hive/trunk/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/trunk/eclipse-templates/.classpath?rev=1211261&r1=1211260&r2=1211261&view=diff
==============================================================================
--- hive/trunk/eclipse-templates/.classpath (original)
+++ hive/trunk/eclipse-templates/.classpath Wed Dec 7 01:29:06 2011
@@ -43,6 +43,7 @@
<classpathentry kind="src" path="build/ql/gen/antlr/gen-java"/>
<classpathentry kind="src" path="cli/src/java"/>
<classpathentry kind="src" path="common/src/java"/>
+ <classpathentry kind="src" path="common/src/test"/>
<classpathentry kind="src" path="contrib/src/java"/>
<classpathentry kind="src" path="contrib/src/test"/>
<classpathentry kind="src" path="metastore/src/gen/thrift/gen-javabean"/>