You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2013/05/11 17:05:10 UTC
svn commit: r1481347 - in /hive/trunk: ./
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/conf/
ql/src/java/org/apache/hadoop/hive/ql/session/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
servi...
Author: omalley
Date: Sat May 11 15:05:10 2013
New Revision: 1481347
URL: http://svn.apache.org/r1481347
Log:
HIVE-4505 Hive can't load transforms with remote scripts. (Prasad Majumdar and Gunther Hagleitner
via omalley)
Added:
hive/trunk/ql/src/test/queries/clientpositive/remote_script.q
- copied unchanged from r1481346, hive/branches/branch-0.11/ql/src/test/queries/clientpositive/remote_script.q
hive/trunk/ql/src/test/results/clientpositive/remote_script.q.out
- copied unchanged from r1481346, hive/branches/branch-0.11/ql/src/test/results/clientpositive/remote_script.q.out
Modified:
hive/trunk/ (props changed)
hive/trunk/build-common.xml
hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Propchange: hive/trunk/
------------------------------------------------------------------------------
Merged /hive/branches/branch-0.11:r1481346
Modified: hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hive/trunk/build-common.xml?rev=1481347&r1=1481346&r2=1481347&view=diff
==============================================================================
--- hive/trunk/build-common.xml (original)
+++ hive/trunk/build-common.xml Sat May 11 15:05:10 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q,remote_script.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Modified: hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java?rev=1481347&r1=1481346&r2=1481347&view=diff
==============================================================================
--- hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java (original)
+++ hive/trunk/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java Sat May 11 15:05:10 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.cli;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -77,11 +78,6 @@ public class CliSessionState extends Ses
private Hive hive; // currently only used (and init'ed) in getCurrentDbName
- public CliSessionState() {
- super();
- remoteMode = false;
- }
-
public CliSessionState(HiveConf conf) {
super(conf);
remoteMode = false;
@@ -112,10 +108,13 @@ public class CliSessionState extends Ses
public void close() {
try {
+ super.close();
if (remoteMode) {
client.clean();
transport.close();
}
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
} catch (TException e) {
e.printStackTrace();
}
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1481347&r1=1481346&r2=1481347&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sat May 11 15:05:10 2013
@@ -201,7 +201,8 @@ public class HiveConf extends Configurat
DYNAMICPARTITIONMAXPARTS("hive.exec.max.dynamic.partitions", 1000),
DYNAMICPARTITIONMAXPARTSPERNODE("hive.exec.max.dynamic.partitions.pernode", 100),
MAXCREATEDFILES("hive.exec.max.created.files", 100000L),
- DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir", "/tmp/"+System.getProperty("user.name")+"/hive_resources"),
+ DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
+ "/tmp/${hive.session.id}_resources"),
DEFAULTPARTITIONNAME("hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__"),
DEFAULT_ZOOKEEPER_PARTITION_NAME("hive.lockmgr.zookeeper.default.partition.name", "__HIVE_DEFAULT_ZOOKEEPER_PARTITION__"),
// Whether to show a link to the most failed task + debugging tips
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java?rev=1481347&r1=1481346&r2=1481347&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java Sat May 11 15:05:10 2013
@@ -22,16 +22,18 @@ import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
+import java.lang.management.ManagementFactory;
import java.net.URI;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.GregorianCalendar;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -57,6 +59,7 @@ import org.apache.hadoop.hive.ql.util.Do
* configuration information
*/
public class SessionState {
+ private static final Log LOG = LogFactory.getLog(SessionState.class);
/**
* current configuration.
@@ -179,18 +182,21 @@ public class SessionState {
this.isVerbose = isVerbose;
}
- public SessionState() {
- this(null);
- }
-
public SessionState(HiveConf conf) {
this.conf = conf;
isSilent = conf.getBoolVar(HiveConf.ConfVars.HIVESESSIONSILENT);
ls = new LineageState();
overriddenConfigurations = new HashMap<String, String>();
overriddenConfigurations.putAll(HiveConf.getConfSystemProperties());
+ // if there isn't already a session name, go ahead and create it.
+ if (StringUtils.isEmpty(conf.getVar(HiveConf.ConfVars.HIVESESSIONID))) {
+ conf.setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
+ }
}
+ private static final SimpleDateFormat DATE_FORMAT =
+ new SimpleDateFormat("yyyyMMddHHmm");
+
public void setCmd(String cmdString) {
conf.setVar(HiveConf.ConfVars.HIVEQUERYSTRING, cmdString);
}
@@ -242,12 +248,6 @@ public class SessionState {
tss.set(startSs);
- if (StringUtils.isEmpty(startSs.getConf().getVar(
- HiveConf.ConfVars.HIVESESSIONID))) {
- startSs.getConf()
- .setVar(HiveConf.ConfVars.HIVESESSIONID, makeSessionId());
- }
-
if (startSs.hiveHist == null) {
startSs.hiveHist = new HiveHistory(startSs);
}
@@ -297,15 +297,15 @@ public class SessionState {
return hiveHist;
}
+ /**
+ * Create a session ID. Looks like:
+ * $user_$pid@$host_$date
+ * @return the unique string
+ */
private static String makeSessionId() {
- GregorianCalendar gc = new GregorianCalendar();
String userid = System.getProperty("user.name");
-
- return userid
- + "_"
- + String.format("%1$4d%2$02d%3$02d%4$02d%5$02d", gc.get(Calendar.YEAR),
- gc.get(Calendar.MONTH) + 1, gc.get(Calendar.DAY_OF_MONTH), gc
- .get(Calendar.HOUR_OF_DAY), gc.get(Calendar.MINUTE));
+ return userid + "_" + ManagementFactory.getRuntimeMXBean().getName() + "_"
+ + DATE_FORMAT.format(new Date());
}
/**
@@ -588,35 +588,15 @@ public class SessionState {
private String downloadResource(String value, boolean convertToUnix) {
if (canDownloadResource(value)) {
getConsole().printInfo("converting to local " + value);
- String location = getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR);
-
+ File resourceDir = new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
String destinationName = new Path(value).getName();
- String prefix = destinationName;
- String postfix = null;
- int index = destinationName.lastIndexOf(".");
- if (index > 0) {
- prefix = destinationName.substring(0, index);
- postfix = destinationName.substring(index);
- }
- if (prefix.length() < 3) {
- prefix += ".tmp"; // prefix should be longer than 3
- }
-
- File resourceDir = new File(location);
- if (resourceDir.exists() && !resourceDir.isDirectory()) {
- throw new RuntimeException("The resource directory is not a directory, " +
- "resourceDir is set to " + resourceDir);
+ File destinationFile = new File(resourceDir, destinationName);
+ if (resourceDir.exists() && ! resourceDir.isDirectory()) {
+ throw new RuntimeException("The resource directory is not a directory, resourceDir is set to" + resourceDir);
}
if (!resourceDir.exists() && !resourceDir.mkdirs()) {
throw new RuntimeException("Couldn't create directory " + resourceDir);
}
-
- File destinationFile;
- try {
- destinationFile = File.createTempFile(prefix, postfix, resourceDir);
- } catch (Exception e) {
- throw new RuntimeException("Failed to create temporary file for " + value, e);
- }
try {
FileSystem fs = FileSystem.get(new URI(value), conf);
fs.copyToLocalFile(new Path(value), new Path(destinationFile.getCanonicalPath()));
@@ -756,4 +736,17 @@ public class SessionState {
public void setLocalMapRedErrors(Map<String, List<String>> localMapRedErrors) {
this.localMapRedErrors = localMapRedErrors;
}
+
+ public void close() throws IOException {
+ File resourceDir =
+ new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));
+ LOG.debug("Removing resource dir " + resourceDir);
+ try {
+ if (resourceDir.exists()) {
+ FileUtils.deleteDirectory(resourceDir);
+ }
+ } catch (IOException e) {
+ LOG.info("Error removing session resource dir " + resourceDir, e);
+ }
+ }
}
Modified: hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java?rev=1481347&r1=1481346&r2=1481347&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java (original)
+++ hive/trunk/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java Sat May 11 15:05:10 2013
@@ -18,13 +18,19 @@
package org.apache.hive.service.cli.session;
+import java.io.File;
+import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -64,6 +70,8 @@ public class HiveSessionImpl implements
private static final String FETCH_WORK_SERDE_CLASS =
"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe";
+ private static final Log LOG = LogFactory.getLog(HiveSessionImpl.class);
+
private SessionManager sessionManager;
private OperationManager operationManager;
@@ -79,7 +87,9 @@ public class HiveSessionImpl implements
hiveConf.set(entry.getKey(), entry.getValue());
}
}
-
+ // set an explicit session name to control the download directory name
+ hiveConf.set(ConfVars.HIVESESSIONID.varname,
+ sessionHandle.getHandleIdentifier().toString());
sessionState = new SessionState(hiveConf);
}
@@ -300,8 +310,11 @@ public class HiveSessionImpl implements
if (null != hiveHist) {
hiveHist.closeStream();
}
- } finally {
+ sessionState.close();
+ release();
+ } catch (IOException ioe) {
release();
+ throw new HiveSQLException("Failure to close", ioe);
}
}