You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2016/05/26 16:52:38 UTC
[1/4] hive git commit: HIVE-13551: Make cleardanglingscratchdir work
on Windows (Daniel Dai, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/branch-2.1 6edd5056d -> ffbd633fa
HIVE-13551: Make cleardanglingscratchdir work on Windows (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3648d35f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3648d35f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3648d35f
Branch: refs/heads/branch-2.1
Commit: 3648d35f51c92c44f4ba96495241daa9114aee42
Parents: 6edd505
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed May 25 15:00:05 2016 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu May 26 09:50:38 2016 -0700
----------------------------------------------------------------------
bin/ext/cleardanglingscratchdir.cmd | 1 -
bin/hive.cmd | 3 ++
.../ql/session/TestClearDanglingScratchDir.java | 5 +++
.../ql/session/ClearDanglingScratchDir.java | 46 +++++++++++++++-----
4 files changed, 44 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/bin/ext/cleardanglingscratchdir.cmd
----------------------------------------------------------------------
diff --git a/bin/ext/cleardanglingscratchdir.cmd b/bin/ext/cleardanglingscratchdir.cmd
index 31104af..1bb0453 100644
--- a/bin/ext/cleardanglingscratchdir.cmd
+++ b/bin/ext/cleardanglingscratchdir.cmd
@@ -16,7 +16,6 @@
set CLASS=org.apache.hadoop.hive.ql.session.ClearDanglingScratchDir
set HIVE_OPTS=
-set HADOOP_CLASSPATH=
pushd %HIVE_LIB%
for /f %%a IN ('dir /b hive-exec-*.jar') do (
http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/bin/hive.cmd
----------------------------------------------------------------------
diff --git a/bin/hive.cmd b/bin/hive.cmd
index 9080796..79d6d1b 100644
--- a/bin/hive.cmd
+++ b/bin/hive.cmd
@@ -361,6 +361,9 @@ goto :EOF
set /a SERVICE_COUNT = %SERVICE_COUNT% + 1
set VAR%SERVICE_COUNT%=schematool
+
+ set /a SERVICE_COUNT = %SERVICE_COUNT% + 1
+ set VAR%SERVICE_COUNT%=cleardanglingscratchdir
goto :EOF
:AddToAuxParam
http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
index 3cb80a7..185dbd5 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/session/TestClearDanglingScratchDir.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.WindowsPathUtil;
+import org.apache.hadoop.util.Shell;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -47,6 +49,9 @@ public class TestClearDanglingScratchDir {
static public void oneTimeSetup() throws Exception {
m_dfs = new MiniDFSCluster.Builder(new Configuration()).numDataNodes(1).format(true).build();
conf = new HiveConf();
+ if (Shell.WINDOWS) {
+ WindowsPathUtil.convertPathsFromWindowsToHdfs(conf);
+ }
conf.set(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK.toString(), "true");
conf.set(HiveConf.ConfVars.METASTORE_AUTO_CREATE_ALL.toString(), "true");
LoggerFactory.getLogger("SessionState");
http://git-wip-us.apache.org/repos/asf/hive/blob/3648d35f/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
index 8543768..ee012c2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
@@ -25,6 +25,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -66,6 +67,7 @@ public class ClearDanglingScratchDir {
if (cli.hasOption("r")) {
dryRun = true;
+ SessionState.getConsole().printInfo("dry-run mode on");
}
if (cli.hasOption("v")) {
@@ -99,24 +101,48 @@ public class ClearDanglingScratchDir {
}
continue;
}
+ boolean removable = false;
+ boolean inuse = false;
try {
IOUtils.closeStream(fs.append(lockFilePath));
- scratchDirToRemove.add(scratchDir.getPath());
- } catch (RemoteException e) {
+ removable = true;
+ } catch (RemoteException eAppend) {
// RemoteException with AlreadyBeingCreatedException will be thrown
// if the file is currently held by a writer
- if(AlreadyBeingCreatedException.class.getName().equals(e.getClassName())){
- // Cannot open the lock file for writing, must be held by a live process
- String message = scratchDir.getPath() + " is being used by live process";
- if (verbose) {
- SessionState.getConsole().printInfo(message);
- } else {
- SessionState.getConsole().logInfo(message);
+ if(AlreadyBeingCreatedException.class.getName().equals(eAppend.getClassName())){
+ inuse = true;
+ } else if (UnsupportedOperationException.class.getName().equals(eAppend.getClassName())) {
+ // Append is not supported in the cluster, try to use create
+ try {
+ IOUtils.closeStream(fs.create(lockFilePath, false));
+ } catch (RemoteException eCreate) {
+ if (AlreadyBeingCreatedException.class.getName().equals(eCreate.getClassName())){
+ // If the file is held by a writer, will throw AlreadyBeingCreatedException
+ inuse = true;
+ } else {
+ SessionState.getConsole().printInfo("Unexpected error:" + eCreate.getMessage());
+ }
+ } catch (FileAlreadyExistsException eCreateNormal) {
+ // Otherwise, throw FileAlreadyExistsException, which means the file owner is
+ // dead
+ removable = true;
}
} else {
- throw e;
+ SessionState.getConsole().printInfo("Unexpected error:" + eAppend.getMessage());
}
}
+ if (inuse) {
+ // Cannot open the lock file for writing, must be held by a live process
+ String message = scratchDir.getPath() + " is being used by live process";
+ if (verbose) {
+ SessionState.getConsole().printInfo(message);
+ } else {
+ SessionState.getConsole().logInfo(message);
+ }
+ }
+ if (removable) {
+ scratchDirToRemove.add(scratchDir.getPath());
+ }
}
}
[3/4] hive git commit: HIVE-12279: Testcase to verify session
temporary files are removed after HIVE-11768
Posted by da...@apache.org.
HIVE-12279: Testcase to verify session temporary files are removed after HIVE-11768
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2209cab5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2209cab5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2209cab5
Branch: refs/heads/branch-2.1
Commit: 2209cab57486f70aebc25312c1f5f04cfce6f009
Parents: b448993
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed May 25 15:44:32 2016 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu May 26 09:51:01 2016 -0700
----------------------------------------------------------------------
.../service/cli/session/TestSessionCleanup.java | 77 ++++++++++++++++++++
1 file changed, 77 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2209cab5/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java b/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java
new file mode 100644
index 0000000..e38a52a
--- /dev/null
+++ b/service/src/test/org/apache/hive/service/cli/session/TestSessionCleanup.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.service.cli.session;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hive.service.cli.SessionHandle;
+import org.apache.hive.service.cli.thrift.EmbeddedThriftBinaryCLIService;
+import org.apache.hive.service.cli.thrift.ThriftCLIServiceClient;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestSessionCleanup extends TestCase {
+
+ @Test
+ // This is to test session temporary files are cleaned up after HIVE-11768
+ public void testTempSessionFileCleanup() throws Exception {
+ EmbeddedThriftBinaryCLIService service = new EmbeddedThriftBinaryCLIService();
+ service.init(null);
+ ThriftCLIServiceClient client = new ThriftCLIServiceClient(service);
+
+ Set<String> existingPipeoutFiles = new HashSet<String>(Arrays.asList(getPipeoutFiles()));
+ SessionHandle sessionHandle = client.openSession("user1", "foobar",
+ Collections.<String, String>emptyMap());
+ client.executeStatement(sessionHandle, "set a=b", null);
+ File operationLogRootDir = new File(
+ new HiveConf().getVar(ConfVars.HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION));
+ Assert.assertNotEquals(operationLogRootDir.list().length, 0);
+ client.closeSession(sessionHandle);
+
+ // Check if session files are removed
+ Assert.assertEquals(operationLogRootDir.list().length, 0);
+
+ // Check if the pipeout files are removed
+ Set<String> finalPipeoutFiles = new HashSet<String>(Arrays.asList(getPipeoutFiles()));
+ finalPipeoutFiles.removeAll(existingPipeoutFiles);
+ Assert.assertTrue(finalPipeoutFiles.isEmpty());
+ }
+
+ private String[] getPipeoutFiles() {
+ File localScratchDir = new File(
+ new HiveConf().getVar(HiveConf.ConfVars.LOCALSCRATCHDIR));
+ String[] pipeoutFiles = localScratchDir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ if (name.endsWith("pipeout")) return true;
+ return false;
+ }
+ });
+ return pipeoutFiles;
+ }
+}
[4/4] hive git commit: HIVE-13729: FileSystem leaks in
FileUtils.checkFileAccessWithImpersonation (Daniel Dai,
reviewed by Thejas Nair)
Posted by da...@apache.org.
HIVE-13729: FileSystem leaks in FileUtils.checkFileAccessWithImpersonation (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ffbd633f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ffbd633f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ffbd633f
Branch: refs/heads/branch-2.1
Commit: ffbd633fa448651e5c15c61826427537b31c160c
Parents: 2209cab
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed May 25 21:44:54 2016 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu May 26 09:52:00 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/common/FileUtils.java | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ffbd633f/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
index 5cf4d39..d755798 100644
--- a/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
+++ b/common/src/java/org/apache/hadoop/hive/common/FileUtils.java
@@ -387,14 +387,18 @@ public final class FileUtils {
// Otherwise, try user impersonation. Current user must be configured to do user impersonation.
UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
user, UserGroupInformation.getLoginUser());
- proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf());
- ShimLoader.getHadoopShims().checkFileAccess(fsAsUser, stat, action);
- return null;
- }
- });
+ try {
+ proxyUser.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ FileSystem fsAsUser = FileSystem.get(fs.getUri(), fs.getConf());
+ ShimLoader.getHadoopShims().checkFileAccess(fsAsUser, stat, action);
+ return null;
+ }
+ });
+ } finally {
+ FileSystem.closeAllForUGI(proxyUser);
+ }
}
/**
[2/4] hive git commit: HIVE-13513: cleardanglingscratchdir does not
work in some version of HDFS (Daniel Dai, reviewed by Thejas Nair)
Posted by da...@apache.org.
HIVE-13513: cleardanglingscratchdir does not work in some version of HDFS (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4489936
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4489936
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4489936
Branch: refs/heads/branch-2.1
Commit: b4489936e84b5f9fdc13bc9d12f152c111e779f9
Parents: 3648d35
Author: Daniel Dai <da...@hortonworks.com>
Authored: Wed May 25 15:23:57 2016 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu May 26 09:50:44 2016 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/session/ClearDanglingScratchDir.java | 6 ++++++
.../org/apache/hadoop/hive/ql/session/SessionState.java | 9 ++++++---
2 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4489936/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
index ee012c2..725f954 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/ClearDanglingScratchDir.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
+import org.apache.hadoop.hive.common.LogUtils;
+import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
@@ -52,6 +54,10 @@ import org.apache.hadoop.ipc.RemoteException;
public class ClearDanglingScratchDir {
public static void main(String[] args) throws Exception {
+ try {
+ LogUtils.initHiveLog4j();
+ } catch (LogInitializationException e) {
+ }
Options opts = createOptions();
CommandLine cli = new GnuParser().parse(opts, args);
http://git-wip-us.apache.org/repos/asf/hive/blob/b4489936/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 37ef165..ce43f7d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -109,6 +109,7 @@ public class SessionState {
private static final String HDFS_SESSION_PATH_KEY = "_hive.hdfs.session.path";
private static final String TMP_TABLE_SPACE_KEY = "_hive.tmp_table_space";
static final String LOCK_FILE_NAME = "inuse.lck";
+ static final String INFO_FILE_NAME = "inuse.info";
private final Map<String, Map<String, Table>> tempTables = new HashMap<String, Map<String, Table>>();
private final Map<String, Map<String, ColumnStatisticsObj>> tempTableColStats =
@@ -643,10 +644,12 @@ public class SessionState {
// 5. hold a lock file in HDFS session dir to indicate the it is in use
if (conf.getBoolVar(HiveConf.ConfVars.HIVE_SCRATCH_DIR_LOCK)) {
FileSystem fs = hdfsSessionPath.getFileSystem(conf);
+ FSDataOutputStream hdfsSessionPathInfoFile = fs.create(new Path(hdfsSessionPath, INFO_FILE_NAME),
+ true);
+ hdfsSessionPathInfoFile.writeUTF("process: " + ManagementFactory.getRuntimeMXBean().getName()
+ +"\n");
+ hdfsSessionPathInfoFile.close();
hdfsSessionPathLockFile = fs.create(new Path(hdfsSessionPath, LOCK_FILE_NAME), true);
- hdfsSessionPathLockFile.writeUTF("hostname: " + InetAddress.getLocalHost().getHostName() + "\n");
- hdfsSessionPathLockFile.writeUTF("process: " + ManagementFactory.getRuntimeMXBean().getName() + "\n");
- hdfsSessionPathLockFile.hsync();
}
// 6. Local session path
localSessionPath = new Path(HiveConf.getVar(conf, HiveConf.ConfVars.LOCALSCRATCHDIR), sessionId);