You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2011/07/21 20:50:29 UTC
svn commit: r1149311 - in /hive/trunk: bin/ext/
cli/src/java/org/apache/hadoop/hive/cli/ common/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/common/cli/
hwi/src/java/org/apache/hadoop/hive/hwi/ metastore/ metas...
Author: cws
Date: Thu Jul 21 18:50:23 2011
New Revision: 1149311
URL: http://svn.apache.org/viewvc?rev=1149311&view=rev
Log:
HIVE-2139. Enable HiveServer to accept -hiveconf option (Patrick Hunt via cws)
Added:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/LogUtils.java (with props)
hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/
hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/CommonCliOptions.java (with props)
Modified:
hive/trunk/bin/ext/hiveserver.sh
hive/trunk/bin/ext/metastore.sh
hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
hive/trunk/common/build.xml
hive/trunk/common/ivy.xml
hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java
hive/trunk/metastore/ivy.xml
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java
Modified: hive/trunk/bin/ext/hiveserver.sh
URL: http://svn.apache.org/viewvc/hive/trunk/bin/ext/hiveserver.sh?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/bin/ext/hiveserver.sh (original)
+++ hive/trunk/bin/ext/hiveserver.sh Thu Jul 21 18:50:23 2011
@@ -25,11 +25,11 @@ hiveserver() {
JAR=${HIVE_LIB}/hive-service-*.jar
# hadoop 20 or newer - skip the aux_jars option and hiveconf
- exec $HADOOP jar $JAR $CLASS $HIVE_PORT "$@"
+
+ exec $HADOOP jar $JAR $CLASS "$@"
}
hiveserver_help() {
- echo "usage HIVE_PORT=xxxx ./hive --service hiveserver"
- echo " HIVE_PORT : Specify the server port"
+ hiveserver -h
}
Modified: hive/trunk/bin/ext/metastore.sh
URL: http://svn.apache.org/viewvc/hive/trunk/bin/ext/metastore.sh?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/bin/ext/metastore.sh (original)
+++ hive/trunk/bin/ext/metastore.sh Thu Jul 21 18:50:23 2011
@@ -25,11 +25,11 @@ metastore() {
JAR=${HIVE_LIB}/hive-service-*.jar
# hadoop 20 or newer - skip the aux_jars option and hiveconf
- exec $HADOOP jar $JAR $CLASS $METASTORE_PORT "$@"
+
+ exec $HADOOP jar $JAR $CLASS "$@"
}
metastore_help() {
- echo "usage METASTORE_PORT=xxxx ./hive --service metastore"
- echo " METASTORE_PORT : Specify the metastore server port"
+ metastore -h
}
Modified: hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Thu Jul 21 18:50:23 2011
@@ -32,18 +32,20 @@ import java.util.Map;
import java.util.Set;
import jline.ArgumentCompletor;
+import jline.ArgumentCompletor.AbstractArgumentDelimiter;
+import jline.ArgumentCompletor.ArgumentDelimiter;
import jline.Completor;
import jline.ConsoleReader;
import jline.History;
import jline.SimpleCompletor;
-import jline.ArgumentCompletor.AbstractArgumentDelimiter;
-import jline.ArgumentCompletor.ArgumentDelimiter;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.HiveInterruptUtils;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Schema;
@@ -512,7 +514,14 @@ public class CliDriver {
// NOTE: It is critical to do this here so that log4j is reinitialized
// before any of the other core hive classes are loaded
- SessionState.initHiveLog4j();
+ boolean logInitFailed = false;
+ String logInitDetailMessage;
+ try {
+ logInitDetailMessage = LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+ logInitFailed = true;
+ logInitDetailMessage = e.getMessage();
+ }
CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
ss.in = System.in;
@@ -526,7 +535,14 @@ public class CliDriver {
if (!oproc.process_stage2(ss)) {
System.exit(2);
}
- ss.printInitInfo();
+
+ if (!ss.getIsSilent()) {
+ if (logInitFailed) {
+ System.err.println(logInitDetailMessage);
+ } else {
+ SessionState.getConsole().printInfo(logInitDetailMessage);
+ }
+ }
// set all properties specified via command line
HiveConf conf = ss.getConf();
Modified: hive/trunk/common/build.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/build.xml?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/common/build.xml (original)
+++ hive/trunk/common/build.xml Thu Jul 21 18:50:23 2011
@@ -27,9 +27,25 @@ to call at top-level: ant deploy-contrib
<property name="src.dir" location="${basedir}/src/java"/>
<import file="../build-common.xml"/>
+ <target name="compile" depends="init, setup, ivy-retrieve">
+ <echo message="Compiling: ${ant.project.name}"/>
+ <javac
+ encoding="${build.encoding}"
+ srcdir="${src.dir}"
+ includes="**/*.java"
+ destdir="${build.classes}"
+ debug="${javac.debug}"
+ deprecation="${javac.deprecation}">
+ <compilerarg line="${javac.args} ${javac.args.warnings}" />
+ <classpath refid="classpath"/>
+ </javac>
+ <copy todir="${build.classes}" failonerror="false">
+ <fileset dir="${src.dir}/conf"/>
+ </copy>
+ </target>
+
<target name="test">
<echo message="Nothing to do!"/>
</target>
-
</project>
Modified: hive/trunk/common/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/common/ivy.xml?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/common/ivy.xml (original)
+++ hive/trunk/common/ivy.xml Thu Jul 21 18:50:23 2011
@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
@@ -26,5 +27,6 @@
<dependency org="hadoop" name="core" rev="${hadoop.version.ant-internal}">
<artifact name="hadoop" type="source" ext="tar.gz"/>
</dependency>
+ <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
</dependencies>
</ivy-module>
Added: hive/trunk/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/LogUtils.java?rev=1149311&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/LogUtils.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/LogUtils.java Thu Jul 21 18:50:23 2011
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.net.URL;
+
+import org.apache.log4j.LogManager;
+import org.apache.log4j.PropertyConfigurator;
+
+/**
+ * Utilities common to logging operations.
+ */
+public class LogUtils {
+
+ public static final String HIVE_L4J = "hive-log4j.properties";
+ public static final String HIVE_EXEC_L4J = "hive-exec-log4j.properties";
+
+ @SuppressWarnings("serial")
+ public static class LogInitializationException extends Exception {
+ public LogInitializationException(String msg) {
+ super(msg);
+ }
+ }
+
+ /**
+ * Initialize log4j based on hive-log4j.properties.
+ *
+ * @return an message suitable for display to the user
+ * @throws LogInitializationException if log4j fails to initialize correctly
+ */
+ public static String initHiveLog4j() throws LogInitializationException {
+ // allow hive log4j to override any normal initialized one
+ URL hive_l4j = LogUtils.class.getClassLoader().getResource(HIVE_L4J);
+ if (hive_l4j != null) {
+ LogManager.resetConfiguration();
+ PropertyConfigurator.configure(hive_l4j);
+ return "Logging initialized using configuration in " + hive_l4j;
+ } else {
+ throw new LogInitializationException("Unable to initialize logging using "
+ + LogUtils.HIVE_L4J + ", not found on CLASSPATH!");
+ }
+ }
+}
Propchange: hive/trunk/common/src/java/org/apache/hadoop/hive/common/LogUtils.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/CommonCliOptions.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/CommonCliOptions.java?rev=1149311&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/CommonCliOptions.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/CommonCliOptions.java Thu Jul 21 18:50:23 2011
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common.cli;
+
+import java.util.Properties;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+/**
+ * Reusable code for Hive Cli's.
+ * <p>
+ * Basic usage is: create an instance (usually a subclass if you want to
+ * all your own options or processing instructions), parse, and then use
+ * the resulting information.
+ * <p>
+ * See {@link HiveServer} or {@link HiveMetaStore} for examples of use.
+ *
+ */
+public class CommonCliOptions {
+ /**
+ * Options for parsing the command line.
+ */
+ protected final Options OPTIONS = new Options();
+
+ protected CommandLine commandLine;
+
+ /**
+ * The name of this cli.
+ */
+ protected final String cliname;
+
+ private boolean verbose = false;
+
+ /**
+ * Create an instance with common options (help, verbose, etc...).
+ *
+ * @param includeHiveConf include "hiveconf" as an option if true
+ */
+ @SuppressWarnings("static-access")
+ public CommonCliOptions(String cliname, boolean includeHiveconf) {
+ this.cliname = cliname;
+
+ // [-v|--verbose]
+ OPTIONS.addOption(new Option("v", "verbose", false, "Verbose mode"));
+
+ // [-h|--help]
+ OPTIONS.addOption(new Option("h", "help", false, "Print help information"));
+
+ if (includeHiveconf) {
+ OPTIONS.addOption(OptionBuilder
+ .withValueSeparator()
+ .hasArgs(2)
+ .withArgName("property=value")
+ .withLongOpt("hiveconf")
+ .withDescription("Use value for given property")
+ .create());
+ }
+ }
+
+ /**
+ * Add the hiveconf properties to the Java system properties, override
+ * anything therein.
+ *
+ * @return a copy of the properties specified in hiveconf
+ */
+ public Properties addHiveconfToSystemProperties() {
+ Properties confProps = commandLine.getOptionProperties("hiveconf");
+ for (String propKey : confProps.stringPropertyNames()) {
+ if (verbose) {
+ System.err.println(
+ "hiveconf: " + propKey + "=" + confProps.getProperty(propKey));
+ }
+ System.setProperty(propKey, confProps.getProperty(propKey));
+ }
+ return confProps;
+ }
+
+ /**
+ * Print usage information for the CLI.
+ */
+ public void printUsage() {
+ new HelpFormatter().printHelp(cliname, OPTIONS);
+ }
+
+ /**
+ * Parse the arguments.
+ * @param args
+ */
+ public void parse(String[] args) {
+ try {
+ commandLine = new GnuParser().parse(OPTIONS, args);
+
+ if (commandLine.hasOption('h')) {
+ printUsage();
+ System.exit(1);
+ }
+ if (commandLine.hasOption('v')) {
+ verbose = true;
+ }
+ } catch (ParseException e) {
+ System.err.println(e.getMessage());
+ printUsage();
+ System.exit(1);
+ }
+
+ }
+
+ /**
+ * Should the client be verbose.
+ */
+ public boolean isVerbose() {
+ return verbose;
+ }
+
+}
Propchange: hive/trunk/common/src/java/org/apache/hadoop/hive/common/cli/CommonCliOptions.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java
URL: http://svn.apache.org/viewvc/hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java (original)
+++ hive/trunk/hwi/src/java/org/apache/hadoop/hive/hwi/HWISessionItem.java Thu Jul 21 18:50:23 2011
@@ -29,6 +29,8 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.cli.CliSessionState;
import org.apache.hadoop.hive.cli.OptionsProcessor;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.CommandNeedRetryException;
import org.apache.hadoop.hive.ql.Driver;
@@ -140,7 +142,11 @@ public class HWISessionItem implements R
}
}
- SessionState.initHiveLog4j();
+ try {
+ LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+ l4j.warn(e);
+ }
conf = new HiveConf(SessionState.class);
ss = new CliSessionState(conf);
SessionState.start(ss);
Modified: hive/trunk/metastore/ivy.xml
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/ivy.xml?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/metastore/ivy.xml (original)
+++ hive/trunk/metastore/ivy.xml Thu Jul 21 18:50:23 2011
@@ -26,6 +26,7 @@
<dependency org="hadoop" name="core" rev="${hadoop.version.ant-internal}">
<artifact name="hadoop" type="source" ext="tar.gz"/>
</dependency>
+ <dependency org="commons-cli" name="commons-cli" rev="${commons-cli.version}"/>
<dependency org="commons-dbcp" name="commons-dbcp" rev="${commons-dbcp.version}">
<exclude module="commons-pool" />
<exclude org="org.apache.geronimo.specs" module="geronimo-jta_1.1_spec"/>
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Jul 21 18:50:23 2011
@@ -32,16 +32,21 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Properties;
import java.util.Timer;
import java.util.regex.Pattern;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.JavaUtils;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.common.classification.InterfaceAudience;
import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.common.cli.CommonCliOptions;
import org.apache.hadoop.hive.common.metrics.Metrics;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
@@ -119,6 +124,11 @@ public class HiveMetaStore extends Thrif
public static final Log LOG = LogFactory.getLog(
HiveMetaStore.class);
+ /**
+ * default port on which to start the Hive server
+ */
+ private static final int DEFAULT_HIVE_METASTORE_PORT = 9083;
+
private static HadoopThriftAuthBridge.Server saslServer;
private static boolean useSasl;
@@ -3367,16 +3377,90 @@ public class HiveMetaStore extends Thrif
}
/**
+ * HiveMetaStore specific CLI
+ *
+ */
+ static public class HiveMetastoreCli extends CommonCliOptions {
+ int port = DEFAULT_HIVE_METASTORE_PORT;
+
+ @SuppressWarnings("static-access")
+ public HiveMetastoreCli() {
+ super("hivemetastore", true);
+
+ // -p port
+ OPTIONS.addOption(OptionBuilder
+ .hasArg()
+ .withArgName("port")
+ .withDescription("Hive Metastore port number, default:"
+ + DEFAULT_HIVE_METASTORE_PORT)
+ .create('p'));
+
+ }
+
+ @Override
+ public void parse(String[] args) {
+ super.parse(args);
+
+ // support the old syntax "hivemetastore [port]" but complain
+ args = commandLine.getArgs();
+ if (args.length > 0) {
+ // complain about the deprecated syntax -- but still run
+ System.err.println(
+ "This usage has been deprecated, consider using the new command "
+ + "line syntax (run with -h to see usage information)");
+
+ port = new Integer(args[0]);
+ }
+
+ // notice that command line options take precedence over the
+ // deprecated (old style) naked args...
+ if (commandLine.hasOption('p')) {
+ port = Integer.parseInt(commandLine.getOptionValue('p'));
+ } else {
+ // legacy handling
+ String metastorePort = System.getenv("METASTORE_PORT");
+ if (metastorePort != null) {
+ port = Integer.parseInt(metastorePort);
+ }
+ }
+ }
+ }
+
+ /**
* @param args
*/
- public static void main(String[] args) throws Throwable{
- int port = 9083;
+ public static void main(String[] args) throws Throwable {
+ HiveMetastoreCli cli = new HiveMetastoreCli();
+
+ cli.parse(args);
+
+ // NOTE: It is critical to do this prior to initializing log4j, otherwise
+ // any log specific settings via hiveconf will be ignored
+ Properties hiveconf = cli.addHiveconfToSystemProperties();
- if (args.length > 0) {
- port = new Integer(args[0]);
+ // NOTE: It is critical to do this here so that log4j is reinitialized
+ // before any of the other core hive classes are loaded
+ try {
+ LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+ HMSHandler.LOG.warn(e.getMessage());
}
+
try {
- startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+ String msg = "Starting hive metastore on port " + cli.port;
+ HMSHandler.LOG.info(msg);
+ if (cli.isVerbose()) {
+ System.err.println(msg);
+ }
+
+ HiveConf conf = new HiveConf(HMSHandler.class);
+
+ // set all properties specified on the command line
+ for (Map.Entry<Object, Object> item : hiveconf.entrySet()) {
+ conf.set((String) item.getKey(), (String) item.getValue());
+ }
+
+ startMetaStore(cli.port, ShimLoader.getHadoopThriftAuthBridge(), conf);
} catch (Throwable t) {
// Catch the exception, log it and rethrow it.
HMSHandler.LOG
@@ -3393,10 +3477,20 @@ public class HiveMetaStore extends Thrif
*/
public static void startMetaStore(int port, HadoopThriftAuthBridge bridge)
throws Throwable {
- try {
+ startMetaStore(port, bridge, new HiveConf(HMSHandler.class));
+ }
- HMSHandler handler = new HMSHandler("new db based metaserver");
- HiveConf conf = handler.getHiveConf();
+ /**
+ * Start Metastore based on a passed {@link HadoopThriftAuthBridge}
+ * @param port
+ * @param bridge
+ * @param hiveconf configuration overrides
+ * @throws Throwable
+ */
+ public static void startMetaStore(int port, HadoopThriftAuthBridge bridge,
+ HiveConf conf) throws Throwable {
+ try {
+ HMSHandler handler = new HMSHandler("new db based metaserver", conf);
// Server will create new threads up to max as necessary. After an idle
// period, it will destory threads to keep the number of threads in the
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java Thu Jul 21 18:50:23 2011
@@ -48,6 +48,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
+import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.Context;
@@ -562,9 +563,9 @@ public class ExecDriver extends Task<Map
*/
private static void setupChildLog4j(Configuration conf) {
- URL hive_l4j = ExecDriver.class.getClassLoader().getResource(SessionState.HIVE_EXEC_L4J);
+ URL hive_l4j = ExecDriver.class.getClassLoader().getResource(LogUtils.HIVE_EXEC_L4J);
if (hive_l4j == null) {
- hive_l4j = ExecDriver.class.getClassLoader().getResource(SessionState.HIVE_L4J);
+ hive_l4j = ExecDriver.class.getClassLoader().getResource(LogUtils.HIVE_L4J);
}
if (hive_l4j != null) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Thu Jul 21 18:50:23 2011
@@ -23,7 +23,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.net.URI;
-import java.net.URL;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.HashMap;
@@ -46,8 +45,6 @@ import org.apache.hadoop.hive.ql.plan.Hi
import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
import org.apache.hadoop.hive.ql.util.DosToUnix;
-import org.apache.log4j.LogManager;
-import org.apache.log4j.PropertyConfigurator;
/**
* SessionState encapsulates common data associated with a session.
@@ -278,29 +275,6 @@ public class SessionState {
.get(Calendar.HOUR_OF_DAY), gc.get(Calendar.MINUTE));
}
- public static final String HIVE_L4J = "hive-log4j.properties";
- public static final String HIVE_EXEC_L4J = "hive-exec-log4j.properties";
-
- public static void initHiveLog4j() {
- // allow hive log4j to override any normal initialized one
- URL hive_l4j = SessionState.class.getClassLoader().getResource(HIVE_L4J);
- if (hive_l4j != null) {
- LogManager.resetConfiguration();
- PropertyConfigurator.configure(hive_l4j);
- }
- }
-
- public void printInitInfo() {
- URL hive_l4j = SessionState.class.getClassLoader().getResource(HIVE_L4J);
- if (!isSilent) {
- if (hive_l4j == null) {
- System.err.println("Unable to initialize logging. " + HIVE_L4J + " not found on CLASSPATH!");
- } else {
- SessionState.getConsole().printInfo("Logging initialized using configuration in " + hive_l4j);
- }
- }
- }
-
/**
* This class provides helper routines to emit informational and error
* messages to the user and log4j files while obeying the current session's
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/history/TestHiveHistory.java Thu Jul 21 18:50:23 2011
@@ -28,6 +28,8 @@ import junit.framework.TestCase;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.MetaStoreUtils;
import org.apache.hadoop.hive.ql.Driver;
@@ -137,7 +139,11 @@ public class TestHiveHistory extends Tes
// NOTE: It is critical to do this here so that log4j is reinitialized
// before
// any of the other core hive classes are loaded
- SessionState.initHiveLog4j();
+ try {
+ LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+ // ignore
+ }
CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
ss.in = System.in;
Modified: hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java?rev=1149311&r1=1149310&r2=1149311&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java (original)
+++ hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java Thu Jul 21 18:50:23 2011
@@ -28,9 +28,15 @@ import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
+import org.apache.hadoop.hive.common.cli.CommonCliOptions;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStore;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -64,6 +70,21 @@ public class HiveServer extends ThriftHi
private static final String VERSION = "1";
/**
+ * default port on which to start the Hive server
+ */
+ private static final int DEFAULT_HIVE_SERVER_PORT = 10000;
+
+ /**
+ * default minimum number of threads serving the Hive server
+ */
+ private static final int DEFAULT_MIN_WORKER_THREADS = 100;
+
+ /**
+ * default maximum number of threads serving the Hive server
+ */
+ private static final int DEFAULT_MAX_WORKER_THREADS = Integer.MAX_VALUE;
+
+ /**
* Handler which implements the Hive Interface This class can be used in lieu
* of the HiveClient class to get an embedded server.
*/
@@ -89,14 +110,26 @@ public class HiveServer extends ThriftHi
public static final Log LOG = LogFactory.getLog(HiveServer.class.getName());
/**
- * A constructor.
+ * Construct a new handler.
+ *
+ * @throws MetaException unable to create metastore
*/
public HiveServerHandler() throws MetaException {
- super(HiveServer.class.getName());
+ this(new HiveConf(SessionState.class));
+ }
+
+ /**
+ * Construct a new handler with the specified hive configuration.
+ *
+ * @param conf caller specified hive configuration
+ * @throws MetaException unable to create metastore
+ */
+ public HiveServerHandler(HiveConf conf) throws MetaException {
+ super(HiveServer.class.getName(), conf);
isHiveQuery = false;
driver = null;
- SessionState session = new SessionState(new HiveConf(SessionState.class));
+ SessionState session = new SessionState(conf);
SessionState.start(session);
setupSessionIO(session);
}
@@ -510,14 +543,17 @@ public class HiveServer extends ThriftHi
*
*/
public static class ThriftHiveProcessorFactory extends TProcessorFactory {
- public ThriftHiveProcessorFactory(TProcessor processor) {
+ private final HiveConf conf;
+
+ public ThriftHiveProcessorFactory(TProcessor processor, HiveConf conf) {
super(processor);
+ this.conf = conf;
}
@Override
public TProcessor getProcessor(TTransport trans) {
try {
- Iface handler = new HiveServerHandler();
+ Iface handler = new HiveServerHandler(conf);
return new ThriftHive.Processor(handler);
} catch (Exception e) {
throw new RuntimeException(e);
@@ -525,27 +561,132 @@ public class HiveServer extends ThriftHi
}
}
- public static void main(String[] args) {
- try {
- //Initializing log for the HiveServer Mode
- SessionState.initHiveLog4j();
+ /**
+ * HiveServer specific CLI
+ *
+ */
+ static public class HiveServerCli extends CommonCliOptions {
+ private static final String OPTION_MAX_WORKER_THREADS = "maxWorkerThreads";
+ private static final String OPTION_MIN_WORKER_THREADS = "minWorkerThreads";
+
+ public int port = DEFAULT_HIVE_SERVER_PORT;
+ public int minWorkerThreads = DEFAULT_MIN_WORKER_THREADS;
+ public int maxWorkerThreads = DEFAULT_MAX_WORKER_THREADS;
+
+ @SuppressWarnings("static-access")
+ public HiveServerCli() {
+ super("hiveserver", true);
+
+ // -p port
+ OPTIONS.addOption(OptionBuilder
+ .hasArg()
+ .withArgName("port")
+ .withDescription("Hive Server port number, default:"
+ + DEFAULT_HIVE_SERVER_PORT)
+ .create('p'));
+
+ // min worker thread count
+ OPTIONS.addOption(OptionBuilder
+ .hasArg()
+ .withLongOpt(OPTION_MIN_WORKER_THREADS)
+ .withDescription("minimum number of worker threads, default:"
+ + DEFAULT_MIN_WORKER_THREADS)
+ .create());
+
+ // max worker thread count
+ OPTIONS.addOption(OptionBuilder
+ .hasArg()
+ .withLongOpt(OPTION_MAX_WORKER_THREADS)
+ .withDescription("maximum number of worker threads, default:"
+ + DEFAULT_MAX_WORKER_THREADS)
+ .create());
+ }
+
+ @Override
+ public void parse(String[] args) {
+ super.parse(args);
- int port = 10000;
- int minWorkerThreads = 100; // default number of threads serving the Hive server
+ // support the old syntax "hiveserver [port [threads]]" but complain
+ args = commandLine.getArgs();
if (args.length >= 1) {
+ // complain about the deprecated syntax -- but still run
+ System.err.println(
+ "This usage has been deprecated, consider using the new command "
+ + "line syntax (run with -h to see usage information)");
+
port = Integer.parseInt(args[0]);
}
if (args.length >= 2) {
minWorkerThreads = Integer.parseInt(args[1]);
}
- TServerTransport serverTransport = new TServerSocket(port);
- ThriftHiveProcessorFactory hfactory = new ThriftHiveProcessorFactory(null);
+
+ // notice that command line options take precedence over the
+ // deprecated (old style) naked args...
+ if (commandLine.hasOption('p')) {
+ port = Integer.parseInt(commandLine.getOptionValue('p'));
+ } else {
+ // legacy handling
+ String hivePort = System.getenv("HIVE_PORT");
+ if (hivePort != null) {
+ port = Integer.parseInt(hivePort);
+ }
+ }
+ if (commandLine.hasOption(OPTION_MIN_WORKER_THREADS)) {
+ minWorkerThreads = Integer.parseInt(
+ commandLine.getOptionValue(OPTION_MIN_WORKER_THREADS));
+ }
+ if (commandLine.hasOption(OPTION_MAX_WORKER_THREADS)) {
+ maxWorkerThreads = Integer.parseInt(
+ commandLine.getOptionValue(OPTION_MAX_WORKER_THREADS));
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ try {
+ HiveServerCli cli = new HiveServerCli();
+
+ cli.parse(args);
+
+ // NOTE: It is critical to do this prior to initializing log4j, otherwise
+ // any log specific settings via hiveconf will be ignored
+ Properties hiveconf = cli.addHiveconfToSystemProperties();
+
+ // NOTE: It is critical to do this here so that log4j is reinitialized
+ // before any of the other core hive classes are loaded
+ try {
+ LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+ HiveServerHandler.LOG.warn(e.getMessage());
+ }
+
+ TServerTransport serverTransport = new TServerSocket(cli.port);
+
+ HiveConf conf = new HiveConf(HiveServerHandler.class);
+
+ // set all properties specified on the command line
+ for (Map.Entry<Object, Object> item : hiveconf.entrySet()) {
+ conf.set((String) item.getKey(), (String) item.getValue());
+ }
+
+ ThriftHiveProcessorFactory hfactory =
+ new ThriftHiveProcessorFactory(null, conf);
+
TThreadPoolServer.Options options = new TThreadPoolServer.Options();
- options.minWorkerThreads = minWorkerThreads;
+ options.minWorkerThreads = cli.minWorkerThreads;
+ options.maxWorkerThreads = cli.maxWorkerThreads;
TServer server = new TThreadPoolServer(hfactory, serverTransport,
new TTransportFactory(), new TTransportFactory(),
new TBinaryProtocol.Factory(), new TBinaryProtocol.Factory(), options);
- HiveServerHandler.LOG.info("Starting hive server on port " + port);
+
+ String msg = "Starting hive server on port " + cli.port
+ + " with " + cli.minWorkerThreads + " min worker threads and "
+ + cli.maxWorkerThreads + " max worker threads";
+ HiveServerHandler.LOG.info(msg);
+ if (cli.isVerbose()) {
+ System.err.println(msg);
+ }
+
server.serve();
} catch (Exception x) {
x.printStackTrace();