You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by va...@apache.org on 2018/07/12 14:54:56 UTC
sqoop git commit: SQOOP-3042: Sqoop does not clear compile directory
under /tmp/sqoop-/compile automatically
Repository: sqoop
Updated Branches:
refs/heads/trunk b790c606a -> 0cfbf5671
SQOOP-3042: Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile automatically
(Eric Lin via Szabolcs Vasas)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0cfbf567
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0cfbf567
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0cfbf567
Branch: refs/heads/trunk
Commit: 0cfbf56713f7574568ea3754f6854e82f5540954
Parents: b790c60
Author: Szabolcs Vasas <va...@apache.org>
Authored: Thu Jul 12 16:26:33 2018 +0200
Committer: Szabolcs Vasas <va...@apache.org>
Committed: Thu Jul 12 16:26:33 2018 +0200
----------------------------------------------------------------------
src/docs/man/codegen-args.txt | 3 ++
src/docs/user/common-args.txt | 2 +
src/java/org/apache/sqoop/SqoopOptions.java | 14 ++++++
src/java/org/apache/sqoop/orm/ClassWriter.java | 7 +++
.../apache/sqoop/orm/CompilationManager.java | 13 ++---
.../org/apache/sqoop/tool/BaseSqoopTool.java | 9 ++++
.../org/apache/sqoop/util/DirCleanupHook.java | 52 ++++++++++++++++++++
.../apache/sqoop/util/TestDirCleanupHook.java | 48 ++++++++++++++++++
8 files changed, 142 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/docs/man/codegen-args.txt
----------------------------------------------------------------------
diff --git a/src/docs/man/codegen-args.txt b/src/docs/man/codegen-args.txt
index c1afaef..adee7d9 100644
--- a/src/docs/man/codegen-args.txt
+++ b/src/docs/man/codegen-args.txt
@@ -34,6 +34,9 @@ Code generation options
--outdir (dir)::
Output directory for generated code
+--delete-compile-dir::
+ Remove temporarily generated class Jar files after job finishes
+
--package-name (package)::
Puts auto-generated classes in the named Java package
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/docs/user/common-args.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/common-args.txt b/src/docs/user/common-args.txt
index 98f19be..6d7560a 100644
--- a/src/docs/user/common-args.txt
+++ b/src/docs/user/common-args.txt
@@ -34,6 +34,8 @@ Argument Description
+-P+ Read password from console
+\--password <password>+ Set authentication password
+\--username <username>+ Set authentication username
++\--delete-compile-dir+ Remove temporarily generated class Jar\
+ files after job finishes
+\--verbose+ Print more information while working
+\--connection-param-file <filename>+ Optional properties file that\
provides connection parameters
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 3a19aea..cc1b752 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -204,6 +204,7 @@ public class SqoopOptions implements Cloneable {
@StoredAsProperty("codegen.output.dir") private String codeOutputDir;
@StoredAsProperty("codegen.compile.dir") private String jarOutputDir;
+ @StoredAsProperty("codegen.delete.compile.dir") private boolean deleteJarOutputDir;
// Boolean specifying whether jarOutputDir is a nonce tmpdir (true), or
// explicitly set by the user (false). If the former, disregard any value
// for jarOutputDir saved in the metastore.
@@ -1086,6 +1087,8 @@ public class SqoopOptions implements Cloneable {
this.jarOutputDir = getNonceJarDir(tmpDir + "sqoop-" + localUsername
+ "/compile");
this.jarDirIsAuto = true;
+ // Default to false, so behaviour does not change
+ this.deleteJarOutputDir = false;
this.layout = FileLayout.TextFile;
this.areOutputDelimsManuallySet = false;
@@ -1707,6 +1710,17 @@ public class SqoopOptions implements Cloneable {
}
/**
+ * @return boolean - whether or not to delete the compile JAR directory
+ */
+ public Boolean getDeleteJarOutputDir() {
+ return this.deleteJarOutputDir;
+ }
+
+ public void setDeleteJarOutputDir(Boolean delete) {
+ this.deleteJarOutputDir = delete;
+ }
+
+ /**
* Return the value of $HADOOP_MAPRED_HOME.
* @return $HADOOP_MAPRED_HOME, or null if it's not set.
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/orm/ClassWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java
index a4a768a..46d0698 100644
--- a/src/java/org/apache/sqoop/orm/ClassWriter.java
+++ b/src/java/org/apache/sqoop/orm/ClassWriter.java
@@ -51,6 +51,7 @@ import org.apache.sqoop.lib.LobSerializer;
import org.apache.sqoop.lib.RecordParser;
import org.apache.sqoop.lib.SqoopRecord;
import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.util.DirCleanupHook;
/**
* Creates an ORM class to represent a table from a database.
@@ -1785,6 +1786,12 @@ public class ClassWriter {
LOG.debug("sourceFilename is " + sourceFilename);
}
+ // SQOOP-3042 - Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile
+ // Add shutdown hook so that all files under the jar output dir can be cleared
+ if(options.getDeleteJarOutputDir()) {
+ Runtime.getRuntime().addShutdownHook(new DirCleanupHook(options.getJarOutputDir()));
+ }
+
compileManager.addSourceFile(sourceFilename);
// Create any missing parent directories.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/orm/CompilationManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java
index 6590cac..f0f5de7 100644
--- a/src/java/org/apache/sqoop/orm/CompilationManager.java
+++ b/src/java/org/apache/sqoop/orm/CompilationManager.java
@@ -245,13 +245,14 @@ public class CompilationManager {
}
}
try {
- FileUtils.moveFile(fOrig, fDest);
+ LOG.debug("moving file from " + fOrig.getAbsolutePath() + " to " + fDest.getAbsolutePath());
+ FileUtils.moveFile(fOrig, fDest);
} catch (IOException e) {
- /*Removed the exception being thrown
- *even if the .java file can not be renamed
- *or can not be moved a "dest" directory for
- *any reason.*/
- LOG.debug("Could not rename " + orig + " to " + dest);
+ /*Removed the exception being thrown
+ *even if the .java file can not be renamed
+ *or can not be moved a "dest" directory for
+ *any reason.*/
+ LOG.error("Could not rename " + orig + " to " + dest + ". Error: " + e.getMessage());
}
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 8d31832..e505c26 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -102,6 +102,7 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
public static final String TARGET_DIR_ARG = "target-dir";
public static final String APPEND_ARG = "append";
public static final String DELETE_ARG = "delete-target-dir";
+ public static final String DELETE_COMPILE_ARG = "delete-compile-dir";
public static final String NULL_STRING = "null-string";
public static final String INPUT_NULL_STRING = "input-null-string";
public static final String NULL_NON_STRING = "null-non-string";
@@ -805,6 +806,10 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
.withDescription("Output directory for compiled objects")
.withLongOpt(BIN_OUT_DIR_ARG)
.create());
+ codeGenOpts.addOption(OptionBuilder
+ .withDescription("Delete compile directory")
+ .withLongOpt(DELETE_COMPILE_ARG)
+ .create());
codeGenOpts.addOption(OptionBuilder.withArgName("name")
.hasArg()
.withDescription("Put auto-generated classes in this package")
@@ -1423,6 +1428,10 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool {
out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG));
}
+ if (in.hasOption(DELETE_COMPILE_ARG)) {
+ out.setDeleteJarOutputDir(true);
+ }
+
if (in.hasOption(PACKAGE_NAME_ARG)) {
out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/util/DirCleanupHook.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/DirCleanupHook.java b/src/java/org/apache/sqoop/util/DirCleanupHook.java
new file mode 100644
index 0000000..120bf0d
--- /dev/null
+++ b/src/java/org/apache/sqoop/util/DirCleanupHook.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.util;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * This Hook is currently used to clean up the temp directory that is used
+ * to hold the generated JAR files for each table class under /tmp/sqoop-<username>/compile.
+ *
+ * But it is generic so it can also be used to clean up other directories if needed
+ */
+public class DirCleanupHook extends Thread {
+
+ private File dir;
+
+ public static final Log LOG = LogFactory.getLog(DirCleanupHook.class.getName());
+
+ public DirCleanupHook(String dirPath) {
+ dir = new File(dirPath);
+ }
+
+ public void run() {
+ try {
+ LOG.debug("Removing directory: " + dir + " in the clean up hook.");
+ FileUtils.deleteDirectory(dir);
+ } catch (IOException e) {
+ LOG.error("Unable to remove directory: " + dir + ". Error was: " + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/test/org/apache/sqoop/util/TestDirCleanupHook.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/util/TestDirCleanupHook.java b/src/test/org/apache/sqoop/util/TestDirCleanupHook.java
new file mode 100644
index 0000000..41146fd
--- /dev/null
+++ b/src/test/org/apache/sqoop/util/TestDirCleanupHook.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.util;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertFalse;
+
+public class TestDirCleanupHook {
+ @Rule
+ public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ private DirCleanupHook dirCleanupHook;
+
+ @Before
+ public void before() throws Exception {
+ // Make sure the directory is not empty.
+ tmpFolder.newFile();
+
+ dirCleanupHook = new DirCleanupHook(tmpFolder.getRoot().getAbsolutePath());
+ }
+
+ @Test
+ public void testDirCleanup() {
+ dirCleanupHook.run();
+
+ assertFalse(tmpFolder.getRoot().exists());
+ }
+}