svn commit: r1301347 - in /hive/trunk: cli/src/java/org/apache/hadoop/hive/cli/ common/src/java/org/apache/hadoop/hive/conf/ ql/src/java/org/apache/hadoop/hive/ql/processors/ ql/src/java/org/apache/hadoop/hive/ql/session/ ql/src/test/org/apache/hadoop/...

[na...@apache.org]

Author: namit
Date: Fri Mar 16 06:21:39 2012
New Revision: 1301347

URL: http://svn.apache.org/viewvc?rev=1301347&view=rev
Log:
HIVE-2872 Store which configs the user has explicitly changed
(Kevin Wilfng via namit)


Added:
    hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyOverriddenConfigsHook.java
    hive/trunk/ql/src/test/queries/clientpositive/overridden_confs.q
    hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out
Modified:
    hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java

Modified: hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java?rev=1301347&r1=1301346&r2=1301347&view=diff
==============================================================================
--- hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java (original)
+++ hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java Fri Mar 16 06:21:39 2012
@@ -32,12 +32,12 @@ import java.util.Map;
 import java.util.Set;
 
 import jline.ArgumentCompletor;
-import jline.ArgumentCompletor.AbstractArgumentDelimiter;
-import jline.ArgumentCompletor.ArgumentDelimiter;
 import jline.Completor;
 import jline.ConsoleReader;
 import jline.History;
 import jline.SimpleCompletor;
+import jline.ArgumentCompletor.AbstractArgumentDelimiter;
+import jline.ArgumentCompletor.ArgumentDelimiter;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -602,6 +602,7 @@ public class CliDriver {
     HiveConf conf = ss.getConf();
     for (Map.Entry<Object, Object> item : ss.cmdProperties.entrySet()) {
       conf.set((String) item.getKey(), (String) item.getValue());
+      ss.getOverriddenConfigurations().put((String) item.getKey(), (String) item.getValue());
     }
 
     SessionState.start(ss);

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1301347&r1=1301346&r2=1301347&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Mar 16 06:21:39 2012
@@ -23,8 +23,10 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.URL;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Properties;
 
 import javax.security.auth.login.LoginException;
@@ -865,13 +867,28 @@ public class HiveConf extends Configurat
    * and the value is non-null and not an empty string.
    */
   private void applySystemProperties() {
+    Map<String, String> systemProperties = getConfSystemProperties();
+    for (Entry<String, String> systemProperty : systemProperties.entrySet()) {
+      this.set(systemProperty.getKey(), systemProperty.getValue());
+    }
+  }
+
+  /**
+   * This method returns a mapping from config variable name to its value for all config variables
+   * which have been set using System properties
+   */
+  public static Map<String, String> getConfSystemProperties() {
+    Map<String, String> systemProperties = new HashMap<String, String>();
+
     for (ConfVars oneVar : ConfVars.values()) {
       if (System.getProperty(oneVar.varname) != null) {
         if (System.getProperty(oneVar.varname).length() > 0) {
-          this.set(oneVar.varname, System.getProperty(oneVar.varname));
+          systemProperties.put(oneVar.varname, System.getProperty(oneVar.varname));
         }
       }
     }
+
+    return systemProperties;
   }
 
   /**

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java?rev=1301347&r1=1301346&r2=1301347&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java Fri Mar 16 06:21:39 2012
@@ -18,8 +18,8 @@
 
 package org.apache.hadoop.hive.ql.processors;
 
-import static org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME;
 import static org.apache.hadoop.hive.serde.Constants.SERIALIZATION_NULL_FORMAT;
+import static org.apache.hadoop.hive.serde.Constants.STRING_TYPE_NAME;
 import static org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe.defaultNullString;
 
 import java.util.Map;
@@ -27,8 +27,8 @@ import java.util.Properties;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
-import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Schema;
 import org.apache.hadoop.hive.ql.parse.VariableSubstitution;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
@@ -119,7 +119,9 @@ public class SetProcessor implements Com
       ss.getHiveVariables().put(propName, new VariableSubstitution().substitute(ss.getConf(),varvalue));
       return new CommandProcessorResponse(0);
     } else {
-      ss.getConf().set(varname, new VariableSubstitution().substitute(ss.getConf(),varvalue) );
+      String substitutedValue = new VariableSubstitution().substitute(ss.getConf(),varvalue);
+      ss.getConf().set(varname, substitutedValue );
+      ss.getOverriddenConfigurations().put(varname, substitutedValue);
       return new CommandProcessorResponse(0);
     }
   }

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1301347&r1=1301346&r2=1301347&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Fri Mar 16 06:21:39 2012
@@ -116,6 +116,12 @@ public class SessionState {
 
   private Map<String, String> hiveVariables;
 
+  // This mapping collects all the configuration variables which have been set by the user
+  // explicitely, either via SET in the CLI, the hiveconf option, or a System property.
+  // It is a mapping from the variable name to its value.  Note that if a user repeatedly
+  // changes the value of a variable, the corresponding change will be made in this mapping.
+  private Map<String, String> overriddenConfigurations;
+
   /**
    * Lineage state.
    */
@@ -177,6 +183,8 @@ public class SessionState {
     this.conf = conf;
     isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
     ls = new LineageState();
+    overriddenConfigurations = new HashMap<String, String>();
+    overriddenConfigurations.putAll(HiveConf.getConfSystemProperties());
 
     // Register the Hive builtins jar and all of its functions
     try {
@@ -699,4 +707,15 @@ public class SessionState {
   public void setLastMapRedStatsList(List<MapRedStats> lastMapRedStatsList) {
     this.lastMapRedStatsList = lastMapRedStatsList;
   }
+
+  public Map<String, String> getOverriddenConfigurations() {
+    if (overriddenConfigurations == null) {
+      overriddenConfigurations = new HashMap<String, String>();
+    }
+    return overriddenConfigurations;
+  }
+
+  public void setOverriddenConfigurations(Map<String, String> overriddenConfigurations) {
+    this.overriddenConfigurations = overriddenConfigurations;
+  }
 }

Added: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyOverriddenConfigsHook.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyOverriddenConfigsHook.java?rev=1301347&view=auto
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyOverriddenConfigsHook.java (added)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/hooks/VerifyOverriddenConfigsHook.java Fri Mar 16 06:21:39 2012
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.hooks;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
+
+/**
+ *
+ * VerifyOverriddenConfigsHook.
+ *
+ * This hook is meant to be used for testing.  It prints the keys and values of config variables
+ * which have been noted by the session state as changed by the user.  It only prints specific
+ * variables as others may change and that should not affect this test.
+ */
+public class VerifyOverriddenConfigsHook implements ExecuteWithHookContext {
+
+  // A config variable set via a System Propery, a config variable set in the CLI,
+  // a config variable not in the default List of config variables, and a config variable in the
+  // default list of conifg variables, but which has not been overridden
+  private static String[] keysArray =
+    {"mapred.job.tracker", "hive.exec.post.hooks", "hive.config.doesnt.exit",
+     "hive.exec.mode.local.auto"};
+  private static List<String> keysList = Arrays.asList(keysArray);
+
+  public void run(HookContext hookContext) {
+    LogHelper console = SessionState.getConsole();
+    SessionState ss = SessionState.get();
+
+    if (console == null || ss == null) {
+      return;
+    }
+
+    for (Entry<String, String> entry : ss.getOverriddenConfigurations().entrySet()) {
+      if (keysList.contains(entry.getKey())) {
+        console.printError("Key: " + entry.getKey() + ", Value: " + entry.getValue());
+      }
+    }
+  }
+}

Added: hive/trunk/ql/src/test/queries/clientpositive/overridden_confs.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/overridden_confs.q?rev=1301347&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/overridden_confs.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/overridden_confs.q Fri Mar 16 06:21:39 2012
@@ -0,0 +1,4 @@
+set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.VerifyOverriddenConfigsHook;
+set hive.config.doesnt.exit=abc;
+
+select count(*) from src;

Added: hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out?rev=1301347&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/overridden_confs.q.out Fri Mar 16 06:21:39 2012
@@ -0,0 +1,8 @@
+PREHOOK: query: select count(*) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+Key: mapred.job.tracker, Value: local
+Key: hive.exec.post.hooks, Value: org.apache.hadoop.hive.ql.hooks.VerifyOverriddenConfigsHook
+Key: hive.config.doesnt.exit, Value: abc
+500