You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jv...@apache.org on 2011/09/23 19:19:46 UTC
svn commit: r1174892 - in /hive/trunk:
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/
service/src/java/org/apache/hadoop/hive/service/
service/src/test/org/apache/hadoop/hive/service/
Author: jvs
Date: Fri Sep 23 17:19:45 2011
New Revision: 1174892
URL: http://svn.apache.org/viewvc?rev=1174892&view=rev
Log:
HIVE-2181. Clean up the scratch.dir (tmp/hive-root) while restarting
Hive server.
(Chinna Rao Lalam via jvs)
Added:
hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java
hive/trunk/service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java
Added: hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java?rev=1174892&view=auto
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java (added)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/common/ServerUtils.java Fri Sep 23 17:19:45 2011
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
+
+/**
+ *
+ * ServerUtils.
+ *
+ */
+public class ServerUtils {
+
+ public static final Log LOG = LogFactory.getLog(ServerUtils.class);
+
+ public static void cleanUpScratchDir(HiveConf hiveConf) {
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_START_CLEANUP_SCRATCHDIR)) {
+ String hiveScratchDir = hiveConf.get(HiveConf.ConfVars.SCRATCHDIR.varname);
+ try {
+ Path jobScratchDir = new Path(hiveScratchDir);
+ LOG.info("Cleaning scratchDir : " + hiveScratchDir);
+ FileSystem fileSystem = jobScratchDir.getFileSystem(hiveConf);
+ fileSystem.delete(jobScratchDir, true);
+ }
+ // Even if the cleanup throws some exception it will continue.
+ catch (Throwable e) {
+ LOG.warn("Unable to delete scratchDir : " + hiveScratchDir, e);
+ }
+ }
+ }
+
+}
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1174892&r1=1174891&r2=1174892&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Sep 23 17:19:45 2011
@@ -478,6 +478,8 @@ public class HiveConf extends Configurat
// The class responsible for logging client side performance metrics
// Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger
HIVE_PERF_LOGGER("hive.exec.perf.logger", "org.apache.hadoop.hive.ql.log.PerfLogger"),
+ // Whether to delete the scratchdir while startup
+ HIVE_START_CLEANUP_SCRATCHDIR("hive.start.cleanup.scratchdir", false),
;
public final String varname;
Modified: hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1174892&r1=1174891&r2=1174892&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Fri Sep 23 17:19:45 2011
@@ -1193,4 +1193,10 @@
<description>The class responsible logging client side performance metrics. Must be a subclass of org.apache.hadoop.hive.ql.log.PerfLogger</description>
</property>
+<property>
+ <name>hive.start.cleanup.scratchdir</name>
+ <value>false</value>
+ <description>To cleanup the hive scratchdir while starting the hive server</description>
+</property>
+
</configuration>
Modified: hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java?rev=1174892&r1=1174891&r2=1174892&view=diff
==============================================================================
--- hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java (original)
+++ hive/trunk/service/src/java/org/apache/hadoop/hive/service/HiveServer.java Fri Sep 23 17:19:45 2011
@@ -34,6 +34,7 @@ import java.util.Properties;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.ServerUtils;
import org.apache.hadoop.hive.common.LogUtils;
import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.common.cli.CommonCliOptions;
@@ -60,7 +61,8 @@ import org.apache.thrift.transport.TServ
import org.apache.thrift.transport.TServerTransport;
import org.apache.thrift.transport.TTransport;
import org.apache.thrift.transport.TTransportFactory;
-
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import com.facebook.fb303.fb_status;
/**
@@ -647,7 +649,7 @@ public class HiveServer extends ThriftHi
}
}
}
-
+
public static void main(String[] args) {
try {
HiveServerCli cli = new HiveServerCli();
@@ -666,9 +668,9 @@ public class HiveServer extends ThriftHi
HiveServerHandler.LOG.warn(e.getMessage());
}
- TServerTransport serverTransport = new TServerSocket(cli.port);
-
HiveConf conf = new HiveConf(HiveServerHandler.class);
+ ServerUtils.cleanUpScratchDir(conf);
+ TServerTransport serverTransport = new TServerSocket(cli.port);
// set all properties specified on the command line
for (Map.Entry<Object, Object> item : hiveconf.entrySet()) {
Modified: hive/trunk/service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java
URL: http://svn.apache.org/viewvc/hive/trunk/service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java?rev=1174892&r1=1174891&r2=1174892&view=diff
==============================================================================
--- hive/trunk/service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java (original)
+++ hive/trunk/service/src/test/org/apache/hadoop/hive/service/TestHiveServer.java Fri Sep 23 17:19:45 2011
@@ -23,7 +23,9 @@ import java.util.Properties;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.ServerUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Schema;
@@ -390,4 +392,33 @@ public class TestHiveServer extends Test
}
}
+ public void testScratchDirShouldNotClearWhileStartup() throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+ Path scratchDirPath = new Path(HiveConf.getVar(conf,
+ HiveConf.ConfVars.SCRATCHDIR));
+ boolean fileExists = fs.exists(scratchDirPath);
+ if (!fileExists) {
+ fileExists = fs.mkdirs(scratchDirPath);
+ }
+ ServerUtils.cleanUpScratchDir(conf);
+ assertTrue("Scratch dir is not available after startup", fs.exists(scratchDirPath));
+ }
+
+ public void testScratchDirShouldClearWhileStartup() throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+ Path scratchDirPath = new Path(HiveConf.getVar(conf,
+ HiveConf.ConfVars.SCRATCHDIR));
+ boolean fileExists = fs.exists(scratchDirPath);
+ if (!fileExists) {
+ fileExists = fs.mkdirs(scratchDirPath);
+ }
+ try {
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_START_CLEANUP_SCRATCHDIR, true);
+ ServerUtils.cleanUpScratchDir(conf);
+ } finally {
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_START_CLEANUP_SCRATCHDIR, false);
+ }
+ assertFalse("Scratch dir is available after startup", fs.exists(scratchDirPath));
+ }
+
}