You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/04/29 22:37:18 UTC
sqoop git commit: SQOOP-2339: Move sub-directory might fail in append
mode
Repository: sqoop
Updated Branches:
refs/heads/trunk d54dddf0b -> fa4e90365
SQOOP-2339: Move sub-directory might fail in append mode
(Qian Xu via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/fa4e9036
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/fa4e9036
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/fa4e9036
Branch: refs/heads/trunk
Commit: fa4e90365024b7aa0b07591a873b07d87c75e071
Parents: d54dddf
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Apr 29 13:36:34 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Wed Apr 29 13:36:34 2015 -0700
----------------------------------------------------------------------
src/java/org/apache/sqoop/util/AppendUtils.java | 2 +-
.../com/cloudera/sqoop/TestParquetImport.java | 53 +++++++++++++++++++-
.../sqoop/testutil/ImportJobTestCase.java | 16 +++++-
3 files changed, 68 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fa4e9036/src/java/org/apache/sqoop/util/AppendUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index c53a6e2..0102ee7 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -177,7 +177,7 @@ public class AppendUtils {
if (fileStatus.isDir()) { // move all subdirectories
// pass target dir as initial dest to prevent nesting inside preexisting dir
- if (fs.rename(fileStatus.getPath(), targetDir)) {
+ if (!fs.exists(targetDir) && fs.rename(fileStatus.getPath(), targetDir)) {
LOG.debug("Directory: " + sourceFilename + " renamed to: " + sourceFilename);
} else {
int dirNumber = 0;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fa4e9036/src/test/com/cloudera/sqoop/TestParquetImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestParquetImport.java b/src/test/com/cloudera/sqoop/TestParquetImport.java
index ae5427c..07e140a 100644
--- a/src/test/com/cloudera/sqoop/TestParquetImport.java
+++ b/src/test/com/cloudera/sqoop/TestParquetImport.java
@@ -112,7 +112,7 @@ public class TestParquetImport extends ImportJobTestCase {
DatasetReader<GenericRecord> reader = getReader();
try {
GenericRecord record1 = reader.next();
- //assertNull(record1);
+ assertNotNull(record1);
assertEquals("DATA_COL0", true, record1.get("DATA_COL0"));
assertEquals("DATA_COL1", 100, record1.get("DATA_COL1"));
assertEquals("DATA_COL2", 200L, record1.get("DATA_COL2"));
@@ -124,6 +124,7 @@ public class TestParquetImport extends ImportJobTestCase {
ByteBuffer b = ((ByteBuffer) object);
assertEquals((byte) 1, b.get(0));
assertEquals((byte) 2, b.get(1));
+ assertFalse(reader.hasNext());
} finally {
reader.close();
}
@@ -145,8 +146,10 @@ public class TestParquetImport extends ImportJobTestCase {
DatasetReader<GenericRecord> reader = getReader();
try {
+ assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals("DATA_COL0", "10", record1.get("DATA_COL0"));
+ assertFalse(reader.hasNext());
} finally {
reader.close();
}
@@ -168,8 +171,10 @@ public class TestParquetImport extends ImportJobTestCase {
DatasetReader<GenericRecord> reader = getReader();
try {
+ assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertEquals("__NAME", 1987, record1.get("__NAME"));
+ assertFalse(reader.hasNext());
} finally {
reader.close();
}
@@ -184,13 +189,50 @@ public class TestParquetImport extends ImportJobTestCase {
DatasetReader<GenericRecord> reader = getReader();
try {
+ assertTrue(reader.hasNext());
GenericRecord record1 = reader.next();
assertNull(record1.get("DATA_COL0"));
+ assertFalse(reader.hasNext());
} finally {
reader.close();
}
}
+ public void testIncrementalParquetImport() throws IOException, SQLException {
+ String [] types = { "INT" };
+ String [] vals = { "1" };
+ createTableWithColTypes(types, vals);
+
+ runImport(getOutputArgv(true, null));
+ runImportAgain(getOutputArgv(true, new String[]{"--append"}));
+
+ DatasetReader<GenericRecord> reader = getReader();
+ try {
+ assertTrue(reader.hasNext());
+ GenericRecord record1 = reader.next();
+ assertEquals(1, record1.get("DATA_COL0"));
+ record1 = reader.next();
+ assertEquals(1, record1.get("DATA_COL0"));
+ assertFalse(reader.hasNext());
+ } finally {
+ reader.close();
+ }
+ }
+
+ public void testOverwriteParquetDatasetFail() throws IOException, SQLException {
+ String [] types = { "INT" };
+ String [] vals = {};
+ createTableWithColTypes(types, vals);
+
+ runImport(getOutputArgv(true, null));
+ try {
+ runImportAgain(getOutputArgv(true, null));
+ fail("");
+ } catch (IOException ex) {
+ // ok
+ }
+ }
+
private CompressionType getCompressionType() {
return getDataset().getDescriptor().getCompressionType();
}
@@ -208,6 +250,15 @@ public class TestParquetImport extends ImportJobTestCase {
return Datasets.load(uri, GenericRecord.class);
}
+ @Override
+ public void tearDown() {
+ super.tearDown();
+ String uri = "dataset:file:" + getTablePath();
+ if (Datasets.exists(uri)) {
+ Datasets.delete(uri);
+ }
+ }
+
private void checkField(Field field, String name, Type type) {
assertEquals(name, field.name());
assertEquals(Type.UNION, field.schema().getType());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/fa4e9036/src/test/com/cloudera/sqoop/testutil/ImportJobTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/testutil/ImportJobTestCase.java b/src/test/com/cloudera/sqoop/testutil/ImportJobTestCase.java
index dbd9061..293bf10 100644
--- a/src/test/com/cloudera/sqoop/testutil/ImportJobTestCase.java
+++ b/src/test/com/cloudera/sqoop/testutil/ImportJobTestCase.java
@@ -206,7 +206,15 @@ public abstract class ImportJobTestCase extends BaseSqoopTestCase {
* execution).
*/
protected void runImport(SqoopTool tool, String [] argv) throws IOException {
- removeTableDir();
+ boolean cleanup = true;
+ runImport(cleanup, tool, argv);
+ }
+
+ private void runImport(boolean cleanup, SqoopTool tool,
+ String [] argv) throws IOException {
+ if (cleanup) {
+ removeTableDir();
+ }
// run the tool through the normal entry-point.
int ret;
@@ -234,4 +242,10 @@ public abstract class ImportJobTestCase extends BaseSqoopTestCase {
runImport(new ImportTool(), argv);
}
+ protected void runImportAgain(String[] argv)
+ throws IOException {
+ boolean cleanup = false;
+ runImport(cleanup, new ImportTool(), argv);
+ }
+
}