You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ve...@apache.org on 2014/01/29 20:30:19 UTC
git commit: SQOOP-1273: Multiple append jobs can easily end up
sharing directories (Jarek Jarcec Cecho via Venkat Ranganathan)
Updated Branches:
refs/heads/trunk da5ab302e -> ad12695b5
SQOOP-1273: Multiple append jobs can easily end up sharing directories
(Jarek Jarcec Cecho via Venkat Ranganathan)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ad12695b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ad12695b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ad12695b
Branch: refs/heads/trunk
Commit: ad12695b59e7f0af09e27da8dca08e9a2be9b6a2
Parents: da5ab30
Author: Venkat Ranganathan <ve...@hortonworks.com>
Authored: Wed Jan 29 11:29:47 2014 -0800
Committer: Venkat Ranganathan <ve...@hortonworks.com>
Committed: Wed Jan 29 11:29:47 2014 -0800
----------------------------------------------------------------------
src/java/org/apache/sqoop/tool/ImportTool.java | 6 +++-
src/java/org/apache/sqoop/util/AppendUtils.java | 11 ++++---
.../com/cloudera/sqoop/TestAppendUtils.java | 32 +++++++++++++++-----
3 files changed, 37 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 59279b6..6cbb873 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -458,7 +458,11 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
Path outputPath = null;
if (options.isAppendMode()) {
// Use temporary path, later removed when appending
- outputPath = AppendUtils.getTempAppendDir(tableName);
+ String salt = tableName;
+ if(salt == null && options.getSqlQuery() != null) {
+ salt = Integer.toHexString(options.getSqlQuery().hashCode());
+ }
+ outputPath = AppendUtils.getTempAppendDir(salt);
LOG.debug("Using temporary folder: " + outputPath.getName());
} else {
// Try in this order: target-dir or warehouse-dir
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/java/org/apache/sqoop/util/AppendUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index d7cd6c5..f5a9a60 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -19,6 +19,7 @@
package org.apache.sqoop.util;
import java.io.IOException;
+import java.lang.management.ManagementFactory;
import java.text.NumberFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
@@ -41,7 +42,7 @@ public class AppendUtils {
public static final Log LOG = LogFactory.getLog(AppendUtils.class.getName());
private static final SimpleDateFormat DATE_FORM = new SimpleDateFormat(
- "ddHHmmssSSS");
+ "ddHHmmssSSSSSSSSS");
private static final String TEMP_IMPORT_ROOT =
System.getProperty("sqoop.test.import.rootDir", "_sqoop");
@@ -263,12 +264,14 @@ public class AppendUtils {
/**
* Creates a unique path object inside the sqoop temporary directory.
*
- * @param tableName
+ * @param salt Salt that will be appended at the end of the generated directory.
+ * Can be arbitrary string, for example table name or query checksum.
* @return a path pointing to the temporary directory
*/
- public static Path getTempAppendDir(String tableName) {
+ public static Path getTempAppendDir(String salt) {
String timeId = DATE_FORM.format(new Date(System.currentTimeMillis()));
- String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + tableName;
+ String jvmName = ManagementFactory.getRuntimeMXBean().getName().replaceAll("@", "_");
+ String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + "_" + jvmName + "_" + salt;
return new Path(tempDir);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/test/com/cloudera/sqoop/TestAppendUtils.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAppendUtils.java b/src/test/com/cloudera/sqoop/TestAppendUtils.java
index ed66b4a..b09dfc6 100644
--- a/src/test/com/cloudera/sqoop/TestAppendUtils.java
+++ b/src/test/com/cloudera/sqoop/TestAppendUtils.java
@@ -58,7 +58,7 @@ public class TestAppendUtils extends ImportJobTestCase {
*
* @return the argv as an array of strings.
*/
- protected ArrayList getOutputlessArgv(boolean includeHadoopFlags,
+ protected ArrayList getOutputlessArgv(boolean includeHadoopFlags, boolean queryBased,
String[] colNames, Configuration conf) {
if (null == colNames) {
colNames = getColNames();
@@ -76,8 +76,13 @@ public class TestAppendUtils extends ImportJobTestCase {
CommonArgs.addHadoopFlags(args);
}
- args.add("--table");
- args.add(getTableName());
+ if(queryBased) {
+ args.add("--query");
+ args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS");
+ } else {
+ args.add("--table");
+ args.add(getTableName());
+ }
args.add("--columns");
args.add(columnsString);
args.add("--split-by");
@@ -241,8 +246,7 @@ public class TestAppendUtils extends ImportJobTestCase {
/** independent to target-dir. */
public void testAppend() throws IOException {
- ArrayList args = getOutputlessArgv(false, HsqldbTestServer.getFieldNames(),
- getConf());
+ ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf());
args.add("--warehouse-dir");
args.add(getWarehouseDir());
@@ -252,8 +256,7 @@ public class TestAppendUtils extends ImportJobTestCase {
/** working with target-dir. */
public void testAppendToTargetDir() throws IOException {
- ArrayList args = getOutputlessArgv(false, HsqldbTestServer.getFieldNames(),
- getConf());
+ ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf());
String targetDir = getWarehouseDir() + "/tempTargetDir";
args.add("--target-dir");
args.add(targetDir);
@@ -265,6 +268,21 @@ public class TestAppendUtils extends ImportJobTestCase {
}
/**
+ * Query based import should also work in append mode.
+ *
+ * @throws IOException
+ */
+ public void testAppendWithQuery() throws IOException {
+ ArrayList args = getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf());
+ String targetDir = getWarehouseDir() + "/tempTargetDir";
+ args.add("--target-dir");
+ args.add(targetDir);
+
+ Path output = new Path(targetDir);
+ runAppendTest(args, output);
+ }
+
+ /**
* If the append source does not exist, don't crash.
*/
public void testAppendSrcDoesNotExist() throws IOException {