You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2018/01/18 14:04:11 UTC
[25/32] sqoop git commit: SQOOP-3273: Removing com.cloudera.sqoop
packages
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 807ec8c..e992005 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -38,28 +38,28 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.sqoop.avro.AvroSchemaMismatchException;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.RelatedOptions;
-import com.cloudera.sqoop.cli.ToolOptions;
-import com.cloudera.sqoop.hive.HiveImport;
-import com.cloudera.sqoop.manager.ImportJobContext;
-import com.cloudera.sqoop.mapreduce.MergeJob;
-import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.metastore.JobStorage;
-import com.cloudera.sqoop.metastore.JobStorageFactory;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.RelatedOptions;
+import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.hive.HiveImport;
+import org.apache.sqoop.manager.ImportJobContext;
+import org.apache.sqoop.mapreduce.MergeJob;
+import org.apache.sqoop.metastore.JobData;
+import org.apache.sqoop.metastore.JobStorage;
+import org.apache.sqoop.metastore.JobStorageFactory;
import org.apache.sqoop.orm.ClassWriter;
-import com.cloudera.sqoop.orm.TableClassName;
-import com.cloudera.sqoop.util.AppendUtils;
-import com.cloudera.sqoop.util.ClassLoaderStack;
-import com.cloudera.sqoop.util.ImportException;
+import org.apache.sqoop.orm.TableClassName;
+import org.apache.sqoop.util.AppendUtils;
+import org.apache.sqoop.util.ClassLoaderStack;
+import org.apache.sqoop.util.ImportException;
import static org.apache.sqoop.manager.SupportedManagers.MYSQL;
/**
* Tool that performs database imports to HDFS.
*/
-public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class ImportTool extends BaseSqoopTool {
public static final Log LOG = LogFactory.getLog(ImportTool.class.getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/JobTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/JobTool.java b/src/java/org/apache/sqoop/tool/JobTool.java
index 72234ba..cf5f320 100644
--- a/src/java/org/apache/sqoop/tool/JobTool.java
+++ b/src/java/org/apache/sqoop/tool/JobTool.java
@@ -18,9 +18,9 @@
package org.apache.sqoop.tool;
-import static com.cloudera.sqoop.metastore.GenericJobStorage.META_CONNECT_KEY;
-import static com.cloudera.sqoop.metastore.GenericJobStorage.META_PASSWORD_KEY;
-import static com.cloudera.sqoop.metastore.GenericJobStorage.META_USERNAME_KEY;
+import static org.apache.sqoop.metastore.GenericJobStorage.META_CONNECT_KEY;
+import static org.apache.sqoop.metastore.GenericJobStorage.META_PASSWORD_KEY;
+import static org.apache.sqoop.metastore.GenericJobStorage.META_USERNAME_KEY;
import java.io.IOException;
@@ -39,19 +39,21 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolRunner;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.ToolOptions;
-import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.metastore.JobStorage;
-import com.cloudera.sqoop.metastore.JobStorageFactory;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.metastore.GenericJobStorage;
+import org.apache.sqoop.metastore.JobData;
+import org.apache.sqoop.metastore.JobStorage;
+import org.apache.sqoop.metastore.JobStorageFactory;
+import org.apache.sqoop.manager.JdbcDrivers;
import org.apache.sqoop.metastore.PasswordRedactor;
import org.apache.sqoop.util.LoggingUtils;
/**
* Tool that creates and executes saved jobs.
*/
-public class JobTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class JobTool extends BaseSqoopTool {
public static final Log LOG = LogFactory.getLog(
JobTool.class.getName());
@@ -180,8 +182,7 @@ public class JobTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
// Now that the tool is fully configured, materialize the job.
//TODO(jarcec): Remove the cast when JobData will be moved to apache package
- JobData jobData = new JobData(jobOptions,
- (com.cloudera.sqoop.tool.SqoopTool)jobTool);
+ JobData jobData = new JobData(jobOptions, jobTool);
this.storage.create(jobName, jobData);
return 0; // Success.
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/ListDatabasesTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ListDatabasesTool.java b/src/java/org/apache/sqoop/tool/ListDatabasesTool.java
index 2dfbfb5..86c0444 100644
--- a/src/java/org/apache/sqoop/tool/ListDatabasesTool.java
+++ b/src/java/org/apache/sqoop/tool/ListDatabasesTool.java
@@ -22,14 +22,14 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.ToolOptions;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.ToolOptions;
/**
* Tool that lists available databases on a server.
*/
-public class ListDatabasesTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class ListDatabasesTool extends BaseSqoopTool {
public static final Log LOG = LogFactory.getLog(
ListDatabasesTool.class.getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/ListTablesTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ListTablesTool.java b/src/java/org/apache/sqoop/tool/ListTablesTool.java
index 5d47bc9..0441352 100644
--- a/src/java/org/apache/sqoop/tool/ListTablesTool.java
+++ b/src/java/org/apache/sqoop/tool/ListTablesTool.java
@@ -22,14 +22,14 @@ import org.apache.commons.cli.CommandLine;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.ToolOptions;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.ToolOptions;
/**
* Tool that lists available tables in a database.
*/
-public class ListTablesTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class ListTablesTool extends BaseSqoopTool {
public static final Log LOG = LogFactory.getLog(
ListTablesTool.class.getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/MainframeImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/MainframeImportTool.java b/src/java/org/apache/sqoop/tool/MainframeImportTool.java
index 0cb91db..8883301 100644
--- a/src/java/org/apache/sqoop/tool/MainframeImportTool.java
+++ b/src/java/org/apache/sqoop/tool/MainframeImportTool.java
@@ -25,10 +25,10 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.ToolRunner;
import org.apache.sqoop.mapreduce.mainframe.MainframeConfiguration;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.RelatedOptions;
-import com.cloudera.sqoop.cli.ToolOptions;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.RelatedOptions;
+import org.apache.sqoop.cli.ToolOptions;
/**
* Tool that performs mainframe dataset imports to HDFS.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/MergeTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/MergeTool.java b/src/java/org/apache/sqoop/tool/MergeTool.java
index a710740..311fee8 100644
--- a/src/java/org/apache/sqoop/tool/MergeTool.java
+++ b/src/java/org/apache/sqoop/tool/MergeTool.java
@@ -25,17 +25,17 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.util.StringUtils;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.RelatedOptions;
-import com.cloudera.sqoop.cli.ToolOptions;
-import com.cloudera.sqoop.mapreduce.MergeJob;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.RelatedOptions;
+import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.mapreduce.MergeJob;
import org.apache.sqoop.util.LoggingUtils;
/**
* Tool that merges a more recent dataset on top of an older one.
*/
-public class MergeTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class MergeTool extends BaseSqoopTool {
public static final Log LOG = LogFactory.getLog(MergeTool.class.getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/MetastoreTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/MetastoreTool.java b/src/java/org/apache/sqoop/tool/MetastoreTool.java
index 53e56f0..596e911 100644
--- a/src/java/org/apache/sqoop/tool/MetastoreTool.java
+++ b/src/java/org/apache/sqoop/tool/MetastoreTool.java
@@ -23,16 +23,16 @@ import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.RelatedOptions;
-import com.cloudera.sqoop.cli.ToolOptions;
-import com.cloudera.sqoop.metastore.hsqldb.HsqldbMetaStore;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.RelatedOptions;
+import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.metastore.hsqldb.HsqldbMetaStore;
/**
* Tool that runs a standalone Sqoop metastore.
*/
-public class MetastoreTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class MetastoreTool extends BaseSqoopTool {
public static final Log LOG = LogFactory.getLog(
MetastoreTool.class.getName());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/SqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/SqoopTool.java b/src/java/org/apache/sqoop/tool/SqoopTool.java
index 5b8453d..732cbe9 100644
--- a/src/java/org/apache/sqoop/tool/SqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/SqoopTool.java
@@ -42,11 +42,10 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.sqoop.util.ClassLoaderStack;
import org.apache.sqoop.config.ConfigurationHelper;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
-import com.cloudera.sqoop.cli.SqoopParser;
-import com.cloudera.sqoop.cli.ToolOptions;
-import com.cloudera.sqoop.tool.ToolDesc;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
+import org.apache.sqoop.cli.SqoopParser;
+import org.apache.sqoop.cli.ToolOptions;
/**
* Base class for Sqoop subprograms (e.g., SqoopImport, SqoopExport, etc.)
@@ -412,7 +411,7 @@ public abstract class SqoopTool {
// This tool is the "active" tool; bind it in the SqoopOptions.
//TODO(jarcec): Remove the cast when SqoopOptions will be moved
// to apache package
- out.setActiveSqoopTool((com.cloudera.sqoop.tool.SqoopTool)this);
+ out.setActiveSqoopTool(this);
String [] toolArgs = args; // args after generic parser is done.
if (useGenericOptions) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/ToolPlugin.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ToolPlugin.java b/src/java/org/apache/sqoop/tool/ToolPlugin.java
index 5fa5e6f..d5ec199 100644
--- a/src/java/org/apache/sqoop/tool/ToolPlugin.java
+++ b/src/java/org/apache/sqoop/tool/ToolPlugin.java
@@ -20,8 +20,6 @@ package org.apache.sqoop.tool;
import java.util.List;
-import com.cloudera.sqoop.tool.ToolDesc;
-
/**
* Abstract base class that defines the ToolPlugin API; additional SqoopTool
* implementations may be registered with the system via ToolPlugin classes.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/tool/VersionTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/VersionTool.java b/src/java/org/apache/sqoop/tool/VersionTool.java
index bcd824d..aafddb2 100644
--- a/src/java/org/apache/sqoop/tool/VersionTool.java
+++ b/src/java/org/apache/sqoop/tool/VersionTool.java
@@ -18,13 +18,13 @@
package org.apache.sqoop.tool;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.cli.ToolOptions;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.cli.ToolOptions;
/**
* Tool that prints Sqoop's version.
*/
-public class VersionTool extends com.cloudera.sqoop.tool.BaseSqoopTool {
+public class VersionTool extends BaseSqoopTool {
public VersionTool() {
super("version");
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/AppendUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index a3082c4..fa85280 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -27,8 +27,8 @@ import java.util.regex.Pattern;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import com.cloudera.sqoop.manager.ImportJobContext;
-import com.cloudera.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.ImportJobContext;
+import org.apache.sqoop.SqoopOptions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/CredentialsUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/CredentialsUtil.java b/src/java/org/apache/sqoop/util/CredentialsUtil.java
index c627b32..fee0bdc 100644
--- a/src/java/org/apache/sqoop/util/CredentialsUtil.java
+++ b/src/java/org/apache/sqoop/util/CredentialsUtil.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.util;
-import com.cloudera.sqoop.SqoopOptions;
+import org.apache.sqoop.SqoopOptions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/DirectImportUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/DirectImportUtils.java b/src/java/org/apache/sqoop/util/DirectImportUtils.java
index d801c8f..a0a5efc 100644
--- a/src/java/org/apache/sqoop/util/DirectImportUtils.java
+++ b/src/java/org/apache/sqoop/util/DirectImportUtils.java
@@ -31,13 +31,13 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.io.CodecMap;
-import com.cloudera.sqoop.io.SplittingOutputStream;
-import com.cloudera.sqoop.io.SplittableBufferedWriter;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.io.CodecMap;
+import org.apache.sqoop.io.SplittingOutputStream;
+import org.apache.sqoop.io.SplittableBufferedWriter;
import org.apache.hadoop.util.Shell;
-import com.cloudera.sqoop.manager.ImportJobContext;
+import org.apache.sqoop.manager.ImportJobContext;
/**
* Utility methods that are common to various the direct import managers.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/ErrorableAsyncSink.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/ErrorableAsyncSink.java b/src/java/org/apache/sqoop/util/ErrorableAsyncSink.java
index aaea322..47d2857 100644
--- a/src/java/org/apache/sqoop/util/ErrorableAsyncSink.java
+++ b/src/java/org/apache/sqoop/util/ErrorableAsyncSink.java
@@ -18,8 +18,6 @@
package org.apache.sqoop.util;
-import com.cloudera.sqoop.util.AsyncSink;
-
/**
* Partial implementation of AsyncSink that relies on ErrorableThread to
* provide a status bit for the join() method.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/Jars.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/Jars.java b/src/java/org/apache/sqoop/util/Jars.java
index 476d59a..3809ada 100644
--- a/src/java/org/apache/sqoop/util/Jars.java
+++ b/src/java/org/apache/sqoop/util/Jars.java
@@ -26,7 +26,7 @@ import java.util.Enumeration;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.manager.ConnManager;
+import org.apache.sqoop.manager.ConnManager;
/**
* Utility class; returns the locations of various jars.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/LoggingAsyncSink.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/LoggingAsyncSink.java b/src/java/org/apache/sqoop/util/LoggingAsyncSink.java
index 5f20539..58a3c55 100644
--- a/src/java/org/apache/sqoop/util/LoggingAsyncSink.java
+++ b/src/java/org/apache/sqoop/util/LoggingAsyncSink.java
@@ -26,8 +26,6 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.util.AsyncSink;
-
/**
* An AsyncSink that takes the contents of a stream and writes
* it to log4j.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/LoggingUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/LoggingUtils.java b/src/java/org/apache/sqoop/util/LoggingUtils.java
index 06b012d..c478245 100644
--- a/src/java/org/apache/sqoop/util/LoggingUtils.java
+++ b/src/java/org/apache/sqoop/util/LoggingUtils.java
@@ -52,7 +52,6 @@ public final class LoggingUtils {
public static void setDebugLevel() {
Logger.getLogger("org.apache.sqoop").setLevel(Level.DEBUG);
- Logger.getLogger("com.cloudera.apache").setLevel(Level.DEBUG);
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/NullAsyncSink.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/NullAsyncSink.java b/src/java/org/apache/sqoop/util/NullAsyncSink.java
index a42e4e9..ffe4560 100644
--- a/src/java/org/apache/sqoop/util/NullAsyncSink.java
+++ b/src/java/org/apache/sqoop/util/NullAsyncSink.java
@@ -26,8 +26,6 @@ import java.io.IOException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.util.AsyncSink;
-
/**
* An AsyncSink that takes the contents of a stream and ignores it.
*/
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/OptionsFileUtil.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/OptionsFileUtil.java b/src/java/org/apache/sqoop/util/OptionsFileUtil.java
index 993ac1b..b487941 100644
--- a/src/java/org/apache/sqoop/util/OptionsFileUtil.java
+++ b/src/java/org/apache/sqoop/util/OptionsFileUtil.java
@@ -30,7 +30,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.Sqoop;
+import org.apache.sqoop.Sqoop;
/**
* Provides utility functions to read in options file. An options file is a
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/SubprocessSecurityManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/SubprocessSecurityManager.java b/src/java/org/apache/sqoop/util/SubprocessSecurityManager.java
index 7a861a0..ea39c6c 100644
--- a/src/java/org/apache/sqoop/util/SubprocessSecurityManager.java
+++ b/src/java/org/apache/sqoop/util/SubprocessSecurityManager.java
@@ -80,7 +80,7 @@ public class SubprocessSecurityManager extends SecurityManager {
*/
public void checkExit(int status) {
LOG.debug("Rejecting System.exit call with status=" + status);
- throw new com.cloudera.sqoop.util.ExitSecurityException(status);
+ throw new org.apache.sqoop.util.ExitSecurityException(status);
}
@Override
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/java/org/apache/sqoop/util/TaskId.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/TaskId.java b/src/java/org/apache/sqoop/util/TaskId.java
index c543754..4caca73 100644
--- a/src/java/org/apache/sqoop/util/TaskId.java
+++ b/src/java/org/apache/sqoop/util/TaskId.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
-import com.cloudera.sqoop.config.ConfigurationConstants;
+import org.apache.sqoop.config.ConfigurationConstants;
/**
* Utility class; returns task attempt Id of the current job
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/perftest/ExportStressTest.java
----------------------------------------------------------------------
diff --git a/src/perftest/ExportStressTest.java b/src/perftest/ExportStressTest.java
index b5710e0..8c6f0f3 100644
--- a/src/perftest/ExportStressTest.java
+++ b/src/perftest/ExportStressTest.java
@@ -23,10 +23,10 @@ import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.*;
-import com.cloudera.sqoop.Sqoop;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.tool.ExportTool;
-import com.cloudera.sqoop.tool.SqoopTool;
+import org.apache.sqoop.Sqoop;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.tool.ExportTool;
+import org.apache.sqoop.tool.SqoopTool;
/**
* Stress test export procedure by running a large-scale export to MySQL.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/perftest/ExtConnFactoryTest.java
----------------------------------------------------------------------
diff --git a/src/perftest/ExtConnFactoryTest.java b/src/perftest/ExtConnFactoryTest.java
index 6d2dec5..c614011 100644
--- a/src/perftest/ExtConnFactoryTest.java
+++ b/src/perftest/ExtConnFactoryTest.java
@@ -20,13 +20,13 @@ import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
-import com.cloudera.sqoop.Sqoop;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.ExportJobContext;
-import com.cloudera.sqoop.manager.ImportJobContext;
-import com.cloudera.sqoop.manager.SqlManager;
-import com.cloudera.sqoop.util.ExportException;
-import com.cloudera.sqoop.util.ImportException;
+import org.apache.sqoop.Sqoop;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.ExportJobContext;
+import org.apache.sqoop.manager.ImportJobContext;
+import org.apache.sqoop.manager.SqlManager;
+import org.apache.sqoop.util.ExportException;
+import org.apache.sqoop.util.ImportException;
/**
* Test external connection factory classes on the classpath.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/perftest/ExtFactory.java
----------------------------------------------------------------------
diff --git a/src/perftest/ExtFactory.java b/src/perftest/ExtFactory.java
index ecde08c..3e3aec8 100644
--- a/src/perftest/ExtFactory.java
+++ b/src/perftest/ExtFactory.java
@@ -19,9 +19,9 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.ConnManager;
-import com.cloudera.sqoop.manager.ManagerFactory;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.manager.ManagerFactory;
/**
* An external ConnFactory used by ExtConnFactoryTest.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/perftest/LobFilePerfTest.java
----------------------------------------------------------------------
diff --git a/src/perftest/LobFilePerfTest.java b/src/perftest/LobFilePerfTest.java
index a16bb5e..f822f16 100644
--- a/src/perftest/LobFilePerfTest.java
+++ b/src/perftest/LobFilePerfTest.java
@@ -19,7 +19,7 @@
import java.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import com.cloudera.sqoop.io.*;
+import org.apache.sqoop.io.*;
/**
* A simple benchmark to performance test LobFile reader/writer speed.
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/perftest/LobFileStressTest.java
----------------------------------------------------------------------
diff --git a/src/perftest/LobFileStressTest.java b/src/perftest/LobFileStressTest.java
index dafebb4..eebc0c1 100644
--- a/src/perftest/LobFileStressTest.java
+++ b/src/perftest/LobFileStressTest.java
@@ -20,7 +20,7 @@ import java.io.*;
import java.util.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import com.cloudera.sqoop.io.*;
+import org.apache.sqoop.io.*;
/**
* Stress test LobFiles by writing a bunch of different files and reading
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/scripts/write-version-info.cmd
----------------------------------------------------------------------
diff --git a/src/scripts/write-version-info.cmd b/src/scripts/write-version-info.cmd
index 247f5f3..8b70a68 100644
--- a/src/scripts/write-version-info.cmd
+++ b/src/scripts/write-version-info.cmd
@@ -29,9 +29,6 @@ set buildroot=%1
set version=%2
set specifiedgithash=%3
-set outputdir=%buildroot%\src\com\cloudera\sqoop
-set outputfile=%outputdir%\SqoopVersion.java
-
set newoutputdir=%buildroot%\src\org\apache\sqoop
set newoutputfile=%newoutputdir%\SqoopVersion.java
@@ -45,31 +42,6 @@ if "%signature%"=="" (
set host=%COMPUTERNAME%
set compiledate=%date%-%time%
-mkdir %outputdir%
-
-(
- echo.// generated by src/scripts/write-version-info.cmd
- echo.package com.cloudera.sqoop;
- echo.
- echo./**
- echo. * @deprecated use org.apache.sqoop.SqoopVersion instead
- echo. * @see org.apache.sqoop.SqoopVersion
- echo. */
- echo.public final class SqoopVersion extends org.apache.sqoop.SqoopVersion {
- echo. public SqoopVersion^(^) {
- echo. super^(^);
- echo. }
- echo. public static final String VERSION =
- echo. org.apache.sqoop.SqoopVersion.VERSION;
- echo. public static final String GIT_HASH =
- echo. org.apache.sqoop.SqoopVersion.GIT_HASH;
- echo. public static final String COMPILE_USER =
- echo. org.apache.sqoop.SqoopVersion.COMPILE_USER;
- echo. public static final String COMPILE_DATE =
- echo. org.apache.sqoop.SqoopVersion.COMPILE_DATE;
- echo.}
-) > %outputfile%
-
mkdir %newoutputdir%
(
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/scripts/write-version-info.sh
----------------------------------------------------------------------
diff --git a/src/scripts/write-version-info.sh b/src/scripts/write-version-info.sh
index 70ce3c7..a88ebf8 100755
--- a/src/scripts/write-version-info.sh
+++ b/src/scripts/write-version-info.sh
@@ -30,9 +30,6 @@ buildroot=$1
version=$2
specifiedgithash=$3
-outputdir="${buildroot}/src/com/cloudera/sqoop"
-outputfile="${outputdir}/SqoopVersion.java"
-
newoutputdir="${buildroot}/src/org/apache/sqoop"
newoutputfile="${newoutputdir}/SqoopVersion.java"
@@ -44,31 +41,6 @@ fi
host=`hostname`
compiledate=`date`
-mkdir -p "${outputdir}"
-cat > "${outputfile}" <<EOF
-// generated by src/scripts/write-version-info.sh
-package com.cloudera.sqoop;
-
-/**
- * @deprecated use org.apache.sqoop.SqoopVersion instead
- * @see org.apache.sqoop.SqoopVersion
- */
-public final class SqoopVersion extends org.apache.sqoop.SqoopVersion {
- public SqoopVersion() {
- super();
- }
- public static final String VERSION =
- org.apache.sqoop.SqoopVersion.VERSION;
- public static final String GIT_HASH =
- org.apache.sqoop.SqoopVersion.GIT_HASH;
- public static final String COMPILE_USER =
- org.apache.sqoop.SqoopVersion.COMPILE_USER;
- public static final String COMPILE_DATE =
- org.apache.sqoop.SqoopVersion.COMPILE_DATE;
-}
-EOF
-
-
mkdir -p "${newoutputdir}"
cat > "${newoutputfile}" <<EOF
// generated by src/scripts/write-version-info.sh
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/TestAllTables.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAllTables.java b/src/test/com/cloudera/sqoop/TestAllTables.java
deleted file mode 100644
index 232b82f..0000000
--- a/src/test/com/cloudera/sqoop/TestAllTables.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop;
-
-import java.io.*;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.avro.generic.GenericRecord;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
-import org.junit.Before;
-import org.junit.After;
-
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.tool.ImportAllTablesTool;
-import org.junit.Test;
-import org.kitesdk.data.Dataset;
-import org.kitesdk.data.DatasetReader;
-import org.kitesdk.data.Datasets;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-/**
- * Test the --all-tables functionality that can import multiple tables.
- */
-public class TestAllTables extends ImportJobTestCase {
-
- /**
- * Create the argv to pass to Sqoop.
- * @return the argv as an array of strings.
- */
- private String [] getArgv(String[] extraArgs, String[] excludeTables) {
- ArrayList<String> args = new ArrayList<String>();
-
- CommonArgs.addHadoopFlags(args);
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--connect");
- args.add(getConnectString());
- args.add("--num-mappers");
- args.add("1");
- args.add("--escaped-by");
- args.add("\\");
- if (excludeTables != null) {
- args.add("--exclude-tables");
- args.add(StringUtils.join(excludeTables, ","));
- }
- if (extraArgs != null) {
- for (String arg : extraArgs) {
- args.add(arg);
- }
- }
-
- return args.toArray(new String[0]);
- }
-
- /** the names of the tables we're creating. */
- private List<String> tableNames;
-
- /** The strings to inject in the (ordered) tables. */
- private List<String> expectedStrings;
-
- @Before
- public void setUp() {
- // start the server
- super.setUp();
-
- if (useHsqldbTestServer()) {
- // throw away TWOINTTABLE and things we don't care about.
- try {
- this.getTestServer().dropExistingSchema();
- } catch (SQLException sqlE) {
- fail(sqlE.toString());
- }
- }
-
- this.tableNames = new ArrayList<String>();
- this.expectedStrings = new ArrayList<String>();
-
- // create two tables.
- this.expectedStrings.add("A winner");
- this.expectedStrings.add("is you!");
- this.expectedStrings.add(null);
-
- int i = 0;
- for (String expectedStr: this.expectedStrings) {
- String wrappedStr = null;
- if (expectedStr != null) {
- wrappedStr = "'" + expectedStr + "'";
- }
-
- String [] types = { "INT NOT NULL PRIMARY KEY", "VARCHAR(32)" };
- String [] vals = { Integer.toString(i++) , wrappedStr };
- this.createTableWithColTypes(types, vals);
- this.tableNames.add(this.getTableName());
- this.removeTableDir();
- incrementTableNum();
- }
- }
-
- @After
- public void tearDown() {
- try {
- for (String table : tableNames) {
- dropTableIfExists(table);
- }
- } catch(SQLException e) {
- LOG.error("Can't clean up the database:", e);
- }
- super.tearDown();
- }
-
- @Test
- public void testMultiTableImport() throws IOException {
- String [] argv = getArgv(null, null);
- runImport(new ImportAllTablesTool(), argv);
-
- Path warehousePath = new Path(this.getWarehouseDir());
- int i = 0;
- for (String tableName : this.tableNames) {
- Path tablePath = new Path(warehousePath, tableName);
- Path filePath = new Path(tablePath, "part-m-00000");
-
- // dequeue the expected value for this table. This
- // list has the same order as the tableNames list.
- String expectedVal = Integer.toString(i++) + ","
- + this.expectedStrings.get(0);
- this.expectedStrings.remove(0);
-
- BufferedReader reader = null;
- if (!isOnPhysicalCluster()) {
- reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(
- new File(filePath.toString()))));
- } else {
- FileSystem dfs = FileSystem.get(getConf());
- FSDataInputStream dis = dfs.open(filePath);
- reader = new BufferedReader(new InputStreamReader(dis));
- }
- try {
- String line = reader.readLine();
- assertEquals("Table " + tableName + " expected a different string",
- expectedVal, line);
- } finally {
- IOUtils.closeStream(reader);
- }
- }
- }
-
- @Test
- public void testMultiTableImportAsParquetFormat() throws IOException {
- String [] argv = getArgv(new String[]{"--as-parquetfile"}, null);
- runImport(new ImportAllTablesTool(), argv);
-
- Path warehousePath = new Path(this.getWarehouseDir());
- int i = 0;
- for (String tableName : this.tableNames) {
- Path tablePath = new Path(warehousePath, tableName);
- Dataset dataset = Datasets.load("dataset:file:" + tablePath);
-
- // dequeue the expected value for this table. This
- // list has the same order as the tableNames list.
- String expectedVal = Integer.toString(i++) + ","
- + this.expectedStrings.get(0);
- this.expectedStrings.remove(0);
-
- DatasetReader<GenericRecord> reader = dataset.newReader();
- try {
- GenericRecord record = reader.next();
- String line = record.get(0) + "," + record.get(1);
- assertEquals("Table " + tableName + " expected a different string",
- expectedVal, line);
- assertFalse(reader.hasNext());
- } finally {
- reader.close();
- }
- }
- }
-
- @Test
- public void testMultiTableImportWithExclude() throws IOException {
- String exclude = this.tableNames.get(0);
- String [] argv = getArgv(null, new String[]{ exclude });
- runImport(new ImportAllTablesTool(), argv);
-
- Path warehousePath = new Path(this.getWarehouseDir());
- int i = 0;
- for (String tableName : this.tableNames) {
- Path tablePath = new Path(warehousePath, tableName);
- Path filePath = new Path(tablePath, "part-m-00000");
-
- // dequeue the expected value for this table. This
- // list has the same order as the tableNames list.
- String expectedVal = Integer.toString(i++) + ","
- + this.expectedStrings.get(0);
- this.expectedStrings.remove(0);
-
- BufferedReader reader = null;
- if (!isOnPhysicalCluster()) {
- reader = new BufferedReader(
- new InputStreamReader(new FileInputStream(
- new File(filePath.toString()))));
- } else {
- FSDataInputStream dis;
- FileSystem dfs = FileSystem.get(getConf());
- if (tableName.equals(exclude)) {
- try {
- dis = dfs.open(filePath);
- assertFalse(true);
- } catch (FileNotFoundException e) {
- // Success
- continue;
- }
- } else {
- dis = dfs.open(filePath);
- }
- reader = new BufferedReader(new InputStreamReader(dis));
- }
- try {
- String line = reader.readLine();
- assertEquals("Table " + tableName + " expected a different string",
- expectedVal, line);
- } finally {
- IOUtils.closeStream(reader);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/TestAppendUtils.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAppendUtils.java b/src/test/com/cloudera/sqoop/TestAppendUtils.java
deleted file mode 100644
index 486afee..0000000
--- a/src/test/com/cloudera/sqoop/TestAppendUtils.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import com.cloudera.sqoop.manager.ImportJobContext;
-
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.HsqldbTestServer;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import com.cloudera.sqoop.tool.ImportTool;
-import com.cloudera.sqoop.util.AppendUtils;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test that --append works.
- */
-public class TestAppendUtils extends ImportJobTestCase {
-
- private static final int PARTITION_DIGITS = 5;
- private static final String FILEPART_SEPARATOR = "-";
-
- public static final Log LOG = LogFactory.getLog(TestAppendUtils.class
- .getName());
-
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- protected ArrayList getOutputlessArgv(boolean includeHadoopFlags, boolean queryBased,
- String[] colNames, Configuration conf) {
- if (null == colNames) {
- colNames = getColNames();
- }
-
- String splitByCol = colNames[0];
- String columnsString = "";
- for (String col : colNames) {
- columnsString += col + ",";
- }
-
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
-
- if(queryBased) {
- args.add("--query");
- args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS");
- } else {
- args.add("--table");
- args.add(getTableName());
- }
- args.add("--columns");
- args.add(columnsString);
- args.add("--split-by");
- args.add(splitByCol);
- args.add("--connect");
- args.add(getConnectString());
- args.add("--as-sequencefile");
- args.add("--num-mappers");
- args.add("1");
-
- args.addAll(getExtraArgs(conf));
-
- return args;
- }
-
- // this test just uses the two int table.
- protected String getTableName() {
- return HsqldbTestServer.getTableName();
- }
-
- /** the same than ImportJobTestCase but without removing tabledir. */
- protected void runUncleanImport(String[] argv) throws IOException {
- // run the tool through the normal entry-point.
- int ret;
- try {
- Configuration conf = getConf();
- SqoopOptions opts = getSqoopOptions(conf);
- Sqoop sqoop = new Sqoop(new ImportTool(), conf, opts);
- ret = Sqoop.runSqoop(sqoop, argv);
- } catch (Exception e) {
- LOG.error("Got exception running Sqoop: " + e.toString());
- e.printStackTrace();
- ret = 1;
- }
-
- // expect a successful return.
- if (0 != ret) {
- throw new IOException("Failure during job; return status " + ret);
- }
- }
-
- /** @return FileStatus for data files only. */
- private FileStatus[] listFiles(FileSystem fs, Path path) throws IOException {
- FileStatus[] fileStatuses = fs.listStatus(path);
- ArrayList files = new ArrayList();
- Pattern patt = Pattern.compile("part.*-([0-9][0-9][0-9][0-9][0-9]).*");
- for (FileStatus fstat : fileStatuses) {
- String fname = fstat.getPath().getName();
- if (!fstat.isDir()) {
- Matcher mat = patt.matcher(fname);
- if (mat.matches()) {
- files.add(fstat);
- }
- }
- }
- return (FileStatus[]) files.toArray(new FileStatus[files.size()]);
- }
-
- private class StatusPathComparator implements Comparator<FileStatus> {
-
- @Override
- public int compare(FileStatus fs1, FileStatus fs2) {
- return fs1.getPath().toString().compareTo(fs2.getPath().toString());
- }
- }
-
- /** @return a concat. string with file-creation dates excluding folders. */
- private String getFileCreationTimeImage(FileSystem fs, Path outputPath,
- int fileCount) throws IOException {
- // create string image with all file creation dates
- StringBuffer image = new StringBuffer();
- FileStatus[] fileStatuses = listFiles(fs, outputPath);
- // sort the file statuses by path so we have a stable order for
- // using 'fileCount'.
- Arrays.sort(fileStatuses, new StatusPathComparator());
- for (int i = 0; i < fileStatuses.length && i < fileCount; i++) {
- image.append(fileStatuses[i].getPath() + "="
- + fileStatuses[i].getModificationTime());
- }
- return image.toString();
- }
-
- /** @return the number part of a partition */
- private int getFilePartition(Path file) {
- String filename = file.getName();
- int pos = filename.lastIndexOf(FILEPART_SEPARATOR);
- if (pos != -1) {
- String part = filename.substring(pos + 1, pos + 1 + PARTITION_DIGITS);
- return Integer.parseInt(part);
- } else {
- return 0;
- }
- }
-
- /**
- * Test for ouput path file-count increase, current files untouched and new
- * correct partition number.
- *
- * @throws IOException
- */
- public void runAppendTest(ArrayList args, Path outputPath)
- throws IOException {
-
- try {
-
- // ensure non-existing output dir for insert phase
- FileSystem fs = FileSystem.get(getConf());
- if (fs.exists(outputPath)) {
- fs.delete(outputPath, true);
- }
-
- // run Sqoop in INSERT mode
- String[] argv = (String[]) args.toArray(new String[0]);
- runUncleanImport(argv);
-
- // get current file count
- FileStatus[] fileStatuses = listFiles(fs, outputPath);
- Arrays.sort(fileStatuses, new StatusPathComparator());
- int previousFileCount = fileStatuses.length;
-
- // get string image with all file creation dates
- String previousImage = getFileCreationTimeImage(fs, outputPath,
- previousFileCount);
-
- // get current last partition number
- Path lastFile = fileStatuses[fileStatuses.length - 1].getPath();
- int lastPartition = getFilePartition(lastFile);
-
- // run Sqoop in APPEND mode
- args.add("--append");
- argv = (String[]) args.toArray(new String[0]);
- runUncleanImport(argv);
-
- // check directory file increase
- fileStatuses = listFiles(fs, outputPath);
- Arrays.sort(fileStatuses, new StatusPathComparator());
- int currentFileCount = fileStatuses.length;
- assertTrue("Output directory didn't got increased in file count ",
- currentFileCount > previousFileCount);
-
- // check previous files weren't modified, also works for partition
- // overlapping
- String currentImage = getFileCreationTimeImage(fs, outputPath,
- previousFileCount);
- assertEquals("Previous files to appending operation were modified",
- currentImage, previousImage);
-
- // check that exists at least 1 new correlative partition
- // let's use a different way than the code being tested
- Path newFile = fileStatuses[previousFileCount].getPath(); // there is a
- // new bound now
- int newPartition = getFilePartition(newFile);
- assertTrue("New partition file isn't correlative",
- lastPartition + 1 == newPartition);
-
- } catch (Exception e) {
- LOG.error("Got Exception: " + StringUtils.stringifyException(e));
- fail(e.toString());
- }
- }
-
- /** independent to target-dir. */
- @Test
- public void testAppend() throws IOException {
- ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf());
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
-
- Path output = new Path(getWarehouseDir(), HsqldbTestServer.getTableName());
- runAppendTest(args, output);
- }
-
- /** working with target-dir. */
- @Test
- public void testAppendToTargetDir() throws IOException {
- ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf());
- String targetDir = getWarehouseDir() + "/tempTargetDir";
- args.add("--target-dir");
- args.add(targetDir);
-
- // there's no need for a new param
- // in diff. w/--warehouse-dir there will no be $tablename dir
- Path output = new Path(targetDir);
- runAppendTest(args, output);
- }
-
- /**
- * Query based import should also work in append mode.
- *
- * @throws IOException
- */
- @Test
- public void testAppendWithQuery() throws IOException {
- ArrayList args = getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf());
- String targetDir = getWarehouseDir() + "/tempTargetDir";
- args.add("--target-dir");
- args.add(targetDir);
-
- Path output = new Path(targetDir);
- runAppendTest(args, output);
- }
-
- /**
- * If the append source does not exist, don't crash.
- */
- @Test
- public void testAppendSrcDoesNotExist() throws IOException {
- Configuration conf = new Configuration();
- if (!isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- SqoopOptions options = new SqoopOptions(conf);
- options.setTableName("meep");
- Path missingPath = new Path("doesNotExistForAnyReason");
- FileSystem local = FileSystem.getLocal(conf);
- assertFalse(local.exists(missingPath));
- ImportJobContext importContext = new ImportJobContext("meep", null,
- options, missingPath);
- AppendUtils utils = new AppendUtils(importContext);
- utils.append();
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/TestAvroExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAvroExport.java b/src/test/com/cloudera/sqoop/TestAvroExport.java
deleted file mode 100644
index ea456e2..0000000
--- a/src/test/com/cloudera/sqoop/TestAvroExport.java
+++ /dev/null
@@ -1,536 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.ExportJobTestCase;
-import com.google.common.collect.Lists;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.avro.Conversions;
-import org.apache.avro.LogicalTypes;
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumWriter;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
-
-/**
- * Test that we can export Avro Data Files from HDFS into databases.
- */
-
-public class TestAvroExport extends ExportJobTestCase {
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- /**
- * @return an argv for the CodeGenTool to use when creating tables to export.
- */
- protected String [] getCodeGenArgv(String... extraArgs) {
- List<String> codeGenArgv = new ArrayList<String>();
-
- if (null != extraArgs) {
- for (String arg : extraArgs) {
- codeGenArgv.add(arg);
- }
- }
-
- codeGenArgv.add("--table");
- codeGenArgv.add(getTableName());
- codeGenArgv.add("--connect");
- codeGenArgv.add(getConnectString());
-
- return codeGenArgv.toArray(new String[0]);
- }
-
- /** When generating data for export tests, each column is generated
- according to a ColumnGenerator. Methods exist for determining
- what to put into Avro objects in the files to export, as well
- as what the object representation of the column as returned by
- the database should look like.
- */
- public interface ColumnGenerator {
- /** For a row with id rowNum, what should we write into that
- Avro record to export?
- */
- Object getExportValue(int rowNum);
-
- /** Return the Avro schema for the field. */
- Schema getColumnAvroSchema();
-
- /** For a row with id rowNum, what should the database return
- for the given column's value?
- */
- Object getVerifyValue(int rowNum);
-
- /** Return the column type to put in the CREATE TABLE statement. */
- String getColumnType();
- }
-
- private ColumnGenerator colGenerator(final Object exportValue,
- final Schema schema, final Object verifyValue,
- final String columnType) {
- return new ColumnGenerator() {
- @Override
- public Object getVerifyValue(int rowNum) {
- return verifyValue;
- }
- @Override
- public Object getExportValue(int rowNum) {
- return exportValue;
- }
- @Override
- public String getColumnType() {
- return columnType;
- }
- @Override
- public Schema getColumnAvroSchema() {
- return schema;
- }
- };
- }
-
- /**
- * Create a data file that gets exported to the db.
- * @param fileNum the number of the file (for multi-file export)
- * @param numRecords how many records to write to the file.
- */
- protected void createAvroFile(int fileNum, int numRecords,
- ColumnGenerator... extraCols) throws IOException {
-
- Path tablePath = getTablePath();
- Path filePath = new Path(tablePath, "part" + fileNum);
-
- Configuration conf = new Configuration();
- if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- FileSystem fs = FileSystem.get(conf);
- fs.mkdirs(tablePath);
- OutputStream os = fs.create(filePath);
-
- Schema schema = buildAvroSchema(extraCols);
- DatumWriter<GenericRecord> datumWriter =
- new GenericDatumWriter<GenericRecord>();
- DataFileWriter<GenericRecord> dataFileWriter =
- new DataFileWriter<GenericRecord>(datumWriter);
- dataFileWriter.create(schema, os);
-
- for (int i = 0; i < numRecords; i++) {
- GenericRecord record = new GenericData.Record(schema);
- record.put("id", i);
- record.put("msg", getMsgPrefix() + i);
- addExtraColumns(record, i, extraCols);
- dataFileWriter.append(record);
- }
-
- dataFileWriter.close();
- os.close();
- }
-
- private Schema buildAvroSchema(ColumnGenerator... extraCols) {
- List<Field> fields = new ArrayList<Field>();
- fields.add(buildAvroField("id", Schema.Type.INT));
- fields.add(buildAvroField("msg", Schema.Type.STRING));
- int colNum = 0;
- // Issue [SQOOP-2846]
- if (null != extraCols) {
- for (ColumnGenerator gen : extraCols) {
- if (gen.getColumnAvroSchema() != null) {
- fields.add(buildAvroField(forIdx(colNum++), gen.getColumnAvroSchema()));
- }
- }
- }
- Schema schema = Schema.createRecord("myschema", null, null, false);
- schema.setFields(fields);
- return schema;
- }
-
- private void addExtraColumns(GenericRecord record, int rowNum,
- ColumnGenerator[] extraCols) {
- int colNum = 0;
- // Issue [SQOOP-2846]
- if (null != extraCols) {
- for (ColumnGenerator gen : extraCols) {
- if (gen.getColumnAvroSchema() != null) {
- record.put(forIdx(colNum++), gen.getExportValue(rowNum));
- }
- }
- }
- }
-
- private Field buildAvroField(String name, Schema.Type type) {
- return new Field(name, Schema.create(type), null, null);
- }
-
- private Field buildAvroField(String name, Schema schema) {
- return new Field(name, schema, null, null);
- }
-
- /** Return the column name for a column index.
- * Each table contains two columns named 'id' and 'msg', and then an
- * arbitrary number of additional columns defined by ColumnGenerators.
- * These columns are referenced by idx 0, 1, 2...
- * @param idx the index of the ColumnGenerator in the array passed to
- * createTable().
- * @return the name of the column
- */
- protected String forIdx(int idx) {
- return "col" + idx;
- }
-
- /**
- * Return a SQL statement that drops a table, if it exists.
- * @param tableName the table to drop.
- * @return the SQL statement to drop that table.
- */
- protected String getDropTableStatement(String tableName) {
- return "DROP TABLE " + tableName + " IF EXISTS";
- }
-
- /** Create the table definition to export to, removing any prior table.
- By specifying ColumnGenerator arguments, you can add extra columns
- to the table of arbitrary type.
- */
- private void createTable(ColumnGenerator... extraColumns)
- throws SQLException {
- Connection conn = getConnection();
- PreparedStatement statement = conn.prepareStatement(
- getDropTableStatement(getTableName()),
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append("CREATE TABLE ");
- sb.append(getTableName());
- sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
- int colNum = 0;
- for (ColumnGenerator gen : extraColumns) {
- if (gen.getColumnType() != null) {
- sb.append(", " + forIdx(colNum++) + " " + gen.getColumnType());
- }
- }
- sb.append(")");
-
- statement = conn.prepareStatement(sb.toString(),
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
- /**
- * Create the table definition to export and also inserting one records for
- * identifying the updates. Issue [SQOOP-2846]
- */
- private void createTableWithInsert() throws SQLException {
- Connection conn = getConnection();
- PreparedStatement statement = conn.prepareStatement(getDropTableStatement(getTableName()),
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- conn.commit();
- } finally {
- statement.close();
- }
-
- StringBuilder sb = new StringBuilder();
- sb.append("CREATE TABLE ");
- sb.append(getTableName());
- sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
- sb.append(")");
- statement = conn.prepareStatement(sb.toString(), ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- try {
- statement.executeUpdate();
- Statement statement2 = conn.createStatement();
- String insertCmd = "INSERT INTO " + getTableName() + " (ID,MSG) VALUES(" + 0 + ",'testMsg');";
- statement2.execute(insertCmd);
- conn.commit();
- } finally {
- statement.close();
- }
- }
-
-
- /** Verify that on a given row, a column has a given value.
- * @param id the id column specifying the row to test.
- */
- private void assertColValForRowId(int id, String colName, Object expectedVal)
- throws SQLException {
- Connection conn = getConnection();
- LOG.info("Verifying column " + colName + " has value " + expectedVal);
-
- PreparedStatement statement = conn.prepareStatement(
- "SELECT " + colName + " FROM " + getTableName() + " WHERE ID = " + id,
- ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
- Object actualVal = null;
- try {
- ResultSet rs = statement.executeQuery();
- try {
- rs.next();
- actualVal = rs.getObject(1);
- } finally {
- rs.close();
- }
- } finally {
- statement.close();
- }
-
- if (expectedVal != null && expectedVal instanceof byte[]) {
- assertArrayEquals((byte[]) expectedVal, (byte[]) actualVal);
- } else {
- assertEquals("Got unexpected column value", expectedVal, actualVal);
- }
- }
-
- /** Verify that for the max and min values of the 'id' column, the values
- for a given column meet the expected values.
- */
- protected void assertColMinAndMax(String colName, ColumnGenerator generator)
- throws SQLException {
- Connection conn = getConnection();
- int minId = getMinRowId(conn);
- int maxId = getMaxRowId(conn);
-
- LOG.info("Checking min/max for column " + colName + " with type "
- + generator.getColumnType());
-
- Object expectedMin = generator.getVerifyValue(minId);
- Object expectedMax = generator.getVerifyValue(maxId);
-
- assertColValForRowId(minId, colName, expectedMin);
- assertColValForRowId(maxId, colName, expectedMax);
- }
-
- @Test
- public void testSupportedAvroTypes() throws IOException, SQLException {
- GenericData.get().addLogicalTypeConversion(new Conversions.DecimalConversion());
-
- String[] argv = {};
- final int TOTAL_RECORDS = 1 * 10;
-
- byte[] b = new byte[] { (byte) 1, (byte) 2 };
- Schema fixed = Schema.createFixed("myfixed", null, null, 2);
- Schema enumeration = Schema.createEnum("myenum", null, null,
- Lists.newArrayList("a", "b"));
- Schema decimalSchema = LogicalTypes.decimal(3,2)
- .addToSchema(Schema.createFixed("dec1", null, null, 2));
-
- ColumnGenerator[] gens = new ColumnGenerator[] {
- colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"),
- colGenerator(100, Schema.create(Schema.Type.INT), 100, "INTEGER"),
- colGenerator(200L, Schema.create(Schema.Type.LONG), 200L, "BIGINT"),
- // HSQLDB maps REAL to double, not float:
- colGenerator(1.0f, Schema.create(Schema.Type.FLOAT), 1.0d, "REAL"),
- colGenerator(2.0d, Schema.create(Schema.Type.DOUBLE), 2.0d, "DOUBLE"),
- colGenerator("s", Schema.create(Schema.Type.STRING), "s", "VARCHAR(8)"),
- colGenerator(ByteBuffer.wrap(b), Schema.create(Schema.Type.BYTES),
- b, "VARBINARY(8)"),
- colGenerator(new GenericData.Fixed(fixed, b), fixed,
- b, "BINARY(2)"),
- colGenerator(new GenericData.EnumSymbol(enumeration, "a"), enumeration,
- "a", "VARCHAR(8)"),
- colGenerator(new BigDecimal("2.00"), decimalSchema,
- new BigDecimal("2.00"), "DECIMAL(3,2)"),
- colGenerator("22.00", Schema.create(Schema.Type.STRING),
- new BigDecimal("22.00"), "DECIMAL(4,2)"),
- };
- createAvroFile(0, TOTAL_RECORDS, gens);
- createTable(gens);
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- verifyExport(TOTAL_RECORDS);
- for (int i = 0; i < gens.length; i++) {
- assertColMinAndMax(forIdx(i), gens[i]);
- }
- }
-
- @Test
- public void testPathPatternInExportDir() throws IOException, SQLException {
- final int TOTAL_RECORDS = 10;
-
- ColumnGenerator[] gens = new ColumnGenerator[] {
- colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"),
- };
-
- createAvroFile(0, TOTAL_RECORDS, gens);
- createTable(gens);
-
- // Converts path to an unary set while preserving the leading '/'
- String pathPattern = new StringBuilder(getTablePath().toString())
- .insert(1, "{")
- .append("}")
- .toString();
-
- runExport(getArgv(true, 10, 10, "--export-dir", pathPattern));
- verifyExport(TOTAL_RECORDS);
- }
-
- @Test
- public void testNullableField() throws IOException, SQLException {
- String[] argv = {};
- final int TOTAL_RECORDS = 1 * 10;
-
- List<Schema> childSchemas = new ArrayList<Schema>();
- childSchemas.add(Schema.create(Schema.Type.NULL));
- childSchemas.add(Schema.create(Schema.Type.STRING));
- Schema schema = Schema.createUnion(childSchemas);
- ColumnGenerator gen0 = colGenerator(null, schema, null, "VARCHAR(64)");
- ColumnGenerator gen1 = colGenerator("s", schema, "s", "VARCHAR(64)");
- createAvroFile(0, TOTAL_RECORDS, gen0, gen1);
- createTable(gen0, gen1);
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- verifyExport(TOTAL_RECORDS);
- assertColMinAndMax(forIdx(0), gen0);
- assertColMinAndMax(forIdx(1), gen1);
- }
-
- @Test
- public void testAvroRecordsNotSupported() throws IOException, SQLException {
- String[] argv = {};
- final int TOTAL_RECORDS = 1;
-
- Schema schema = Schema.createRecord("nestedrecord", null, null, false);
- schema.setFields(Lists.newArrayList(buildAvroField("myint",
- Schema.Type.INT)));
- GenericRecord record = new GenericData.Record(schema);
- record.put("myint", 100);
- // DB type is not used so can be anything:
- ColumnGenerator gen = colGenerator(record, schema, null, "VARCHAR(64)");
- createAvroFile(0, TOTAL_RECORDS, gen);
- createTable(gen);
-
- thrown.expect(Exception.class);
- thrown.reportMissingExceptionWithMessage("Expected Exception as Avro records are not supported");
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- }
-
- @Test
- public void testMissingDatabaseFields() throws IOException, SQLException {
- String[] argv = {};
- final int TOTAL_RECORDS = 1;
-
- // null column type means don't create a database column
- // the Avro value will not be exported
- ColumnGenerator gen = colGenerator(100, Schema.create(Schema.Type.INT),
- null, null);
- createAvroFile(0, TOTAL_RECORDS, gen);
- createTable(gen);
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- verifyExport(TOTAL_RECORDS);
- }
-
- // Test Case for Issue [SQOOP-2846]
- @Test
- public void testAvroWithUpsert() throws IOException, SQLException {
- String[] argv = { "--update-key", "ID", "--update-mode", "allowinsert" };
- final int TOTAL_RECORDS = 2;
- // ColumnGenerator gen = colGenerator("100",
- // Schema.create(Schema.Type.STRING), null, "VARCHAR(64)");
- createAvroFile(0, TOTAL_RECORDS, null);
- createTableWithInsert();
-
- thrown.expect(Exception.class);
- thrown.reportMissingExceptionWithMessage("Expected Exception during Avro export with --update-mode");
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- }
-
- // Test Case for Issue [SQOOP-2846]
- @Test
- public void testAvroWithUpdateKey() throws IOException, SQLException {
- String[] argv = { "--update-key", "ID" };
- final int TOTAL_RECORDS = 1;
- // ColumnGenerator gen = colGenerator("100",
- // Schema.create(Schema.Type.STRING), null, "VARCHAR(64)");
- createAvroFile(0, TOTAL_RECORDS, null);
- createTableWithInsert();
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- verifyExport(getMsgPrefix() + "0");
- }
-
- @Test
- public void testMissingAvroFields() throws IOException, SQLException {
- String[] argv = {};
- final int TOTAL_RECORDS = 1;
-
- // null Avro schema means don't create an Avro field
- ColumnGenerator gen = colGenerator(null, null, null, "VARCHAR(64)");
- createAvroFile(0, TOTAL_RECORDS, gen);
- createTable(gen);
-
- thrown.expect(Exception.class);
- thrown.reportMissingExceptionWithMessage("Expected Exception on missing Avro fields");
- runExport(getArgv(true, 10, 10, newStrArray(argv, "-m", "" + 1)));
- }
-
- @Test
- public void testSpecifiedColumnsAsAvroFields() throws IOException, SQLException {
- final int TOTAL_RECORDS = 10;
- ColumnGenerator[] gens = new ColumnGenerator[] {
- colGenerator(000, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col0
- colGenerator(111, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col1
- colGenerator(222, Schema.create(Schema.Type.INT), 100, "INTEGER"), //col2
- colGenerator(333, Schema.create(Schema.Type.INT), 100, "INTEGER") //col3
- };
- createAvroFile(0, TOTAL_RECORDS, gens);
- createTable(gens);
- runExport(getArgv(true, 10, 10, newStrArray(null, "-m", "" + 1, "--columns", "ID,MSG,COL1,COL2")));
- verifyExport(TOTAL_RECORDS);
- assertColValForRowId(0, "col0", null);
- assertColValForRowId(0, "col1", 111);
- assertColValForRowId(0, "col2", 222);
- assertColValForRowId(0, "col3", null);
- assertColValForRowId(9, "col0", null);
- assertColValForRowId(9, "col1", 111);
- assertColValForRowId(9, "col2", 222);
- assertColValForRowId(9, "col3", null);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/TestAvroImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAvroImport.java b/src/test/com/cloudera/sqoop/TestAvroImport.java
deleted file mode 100644
index da79c7a..0000000
--- a/src/test/com/cloudera/sqoop/TestAvroImport.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.avro.Schema;
-import org.apache.avro.Schema.Field;
-import org.apache.avro.Schema.Type;
-import org.apache.avro.file.DataFileConstants;
-import org.apache.avro.file.DataFileReader;
-import org.apache.avro.generic.GenericDatumReader;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.mapred.FsInput;
-import org.apache.avro.util.Utf8;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.HsqldbTestServer;
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Tests --as-avrodatafile.
- */
-public class TestAvroImport extends ImportJobTestCase {
-
- public static final Log LOG = LogFactory
- .getLog(TestAvroImport.class.getName());
-
- /**
- * Create the argv to pass to Sqoop.
- *
- * @return the argv as an array of strings.
- */
- protected String[] getOutputArgv(boolean includeHadoopFlags,
- String[] extraArgs) {
- ArrayList<String> args = new ArrayList<String>();
-
- if (includeHadoopFlags) {
- CommonArgs.addHadoopFlags(args);
- }
- args.add("-m");
- args.add("1");
- args.add("--table");
- args.add(getTableName());
- args.add("--connect");
- args.add(HsqldbTestServer.getUrl());
- args.add("--warehouse-dir");
- args.add(getWarehouseDir());
- args.add("--split-by");
- args.add("INTFIELD1");
- args.add("--as-avrodatafile");
- if (extraArgs != null) {
- args.addAll(Arrays.asList(extraArgs));
- }
-
- return args.toArray(new String[0]);
- }
-
- @Test
- public void testAvroImport() throws IOException {
- this.setCurTableName("Avro_Import_Test");
- avroImportTestHelper(null, null);
- }
-
- @Test
- public void testDeflateCompressedAvroImport() throws IOException {
- this.setCurTableName("Deflate_Compressed_Avro_Import_Test_1");
- avroImportTestHelper(new String[] {"--compression-codec",
- "org.apache.hadoop.io.compress.DefaultCodec", }, "deflate");
- }
-
- @Test
- public void testDefaultCompressedAvroImport() throws IOException {
- this.setCurTableName("Deflate_Compressed_Avro_Import_Test_2");
- avroImportTestHelper(new String[] {"--compress", }, "deflate");
- }
-
- @Test
- public void testUnsupportedCodec() throws IOException {
- try {
- this.setCurTableName("Deflate_Compressed_Avro_Import_Test_3");
- avroImportTestHelper(new String[] {"--compression-codec", "foobar", },
- null);
- fail("Expected IOException");
- } catch (IOException e) {
- // Exception is expected
- }
- }
-
- /**
- * Helper method that runs an import using Avro with optional command line
- * arguments and checks that the created file matches the expectations.
- * <p/>
- * This can be used to test various extra options that are implemented for
- * the Avro input.
- *
- * @param extraArgs extra command line arguments to pass to Sqoop in addition
- * to those that {@link #getOutputArgv(boolean, String[])}
- * returns
- */
- protected void avroImportTestHelper(String[] extraArgs, String codec)
- throws IOException {
- String[] types =
- {"BIT", "INTEGER", "BIGINT", "REAL", "DOUBLE", "VARCHAR(6)",
- "VARBINARY(2)", "DECIMAL(3,2)"};
- String[] vals = {"true", "100", "200", "1.0", "2.0", "'s'", "'0102'", "'1.00'"};
- createTableWithColTypes(types, vals);
-
- runImport(getOutputArgv(true, extraArgs));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- Schema schema = reader.getSchema();
- assertEquals(Schema.Type.RECORD, schema.getType());
- List<Field> fields = schema.getFields();
- assertEquals(types.length, fields.size());
-
- checkField(fields.get(0), "DATA_COL0", Schema.Type.BOOLEAN);
- checkField(fields.get(1), "DATA_COL1", Schema.Type.INT);
- checkField(fields.get(2), "DATA_COL2", Schema.Type.LONG);
- checkField(fields.get(3), "DATA_COL3", Schema.Type.FLOAT);
- checkField(fields.get(4), "DATA_COL4", Schema.Type.DOUBLE);
- checkField(fields.get(5), "DATA_COL5", Schema.Type.STRING);
- checkField(fields.get(6), "DATA_COL6", Schema.Type.BYTES);
- checkField(fields.get(7), "DATA_COL7", Schema.Type.STRING);
-
- GenericRecord record1 = reader.next();
- assertEquals("DATA_COL0", true, record1.get("DATA_COL0"));
- assertEquals("DATA_COL1", 100, record1.get("DATA_COL1"));
- assertEquals("DATA_COL2", 200L, record1.get("DATA_COL2"));
- assertEquals("DATA_COL3", 1.0f, record1.get("DATA_COL3"));
- assertEquals("DATA_COL4", 2.0, record1.get("DATA_COL4"));
- assertEquals("DATA_COL5", new Utf8("s"), record1.get("DATA_COL5"));
- Object object = record1.get("DATA_COL6");
- assertTrue(object instanceof ByteBuffer);
- ByteBuffer b = ((ByteBuffer) object);
- assertEquals((byte) 1, b.get(0));
- assertEquals((byte) 2, b.get(1));
- assertEquals("DATA_COL7", "1.00", record1.get("DATA_COL7").toString());
-
- if (codec != null) {
- assertEquals(codec, reader.getMetaString(DataFileConstants.CODEC));
- }
-
- checkSchemaFile(schema);
- }
-
- @Test
- public void testOverrideTypeMapping() throws IOException {
- String [] types = { "INT" };
- String [] vals = { "10" };
- createTableWithColTypes(types, vals);
-
- String [] extraArgs = { "--map-column-java", "DATA_COL0=String"};
-
- runImport(getOutputArgv(true, extraArgs));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- Schema schema = reader.getSchema();
- assertEquals(Schema.Type.RECORD, schema.getType());
- List<Field> fields = schema.getFields();
- assertEquals(types.length, fields.size());
-
- checkField(fields.get(0), "DATA_COL0", Schema.Type.STRING);
-
- GenericRecord record1 = reader.next();
- assertEquals("DATA_COL0", new Utf8("10"), record1.get("DATA_COL0"));
- }
-
- @Test
- public void testFirstUnderscoreInColumnName() throws IOException {
- String [] names = { "_NAME" };
- String [] types = { "INT" };
- String [] vals = { "1987" };
- createTableWithColTypesAndNames(names, types, vals);
-
- runImport(getOutputArgv(true, null));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- Schema schema = reader.getSchema();
- assertEquals(Schema.Type.RECORD, schema.getType());
- List<Field> fields = schema.getFields();
- assertEquals(types.length, fields.size());
-
- checkField(fields.get(0), "__NAME", Type.INT);
-
- GenericRecord record1 = reader.next();
- assertEquals("__NAME", 1987, record1.get("__NAME"));
- }
-
- @Test
- public void testNonstandardCharactersInColumnName() throws IOException {
- String [] names = { "avro\uC3A11" };
- String [] types = { "INT" };
- String [] vals = { "1987" };
- this.setCurTableName("Non_Std_Character_Test");
- createTableWithColTypesAndNames(names, types, vals);
-
- runImport(getOutputArgv(true, null));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- Schema schema = reader.getSchema();
- assertEquals(Schema.Type.RECORD, schema.getType());
- List<Field> fields = schema.getFields();
- assertEquals(types.length, fields.size());
-
- checkField(fields.get(0), "AVRO\uC3A11", Type.INT);
-
- GenericRecord record1 = reader.next();
- assertEquals("AVRO\uC3A11", 1987, record1.get("AVRO\uC3A11"));
- }
-
- @Test
- public void testNonIdentCharactersInColumnName() throws IOException {
- String [] names = { "test_a-v+r/o" };
- String [] types = { "INT" };
- String [] vals = { "2015" };
- createTableWithColTypesAndNames(names, types, vals);
-
- runImport(getOutputArgv(true, null));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- Schema schema = reader.getSchema();
- assertEquals(Schema.Type.RECORD, schema.getType());
- List<Field> fields = schema.getFields();
- assertEquals(types.length, fields.size());
-
- checkField(fields.get(0), "TEST_A_V_R_O", Type.INT);
-
- GenericRecord record1 = reader.next();
- assertEquals("TEST_A_V_R_O", 2015, record1.get("TEST_A_V_R_O"));
- }
-
- /*
- * Test Case For checking multiple columns having non standard characters in multiple columns
- */
- @Test
- public void testNonstandardCharactersInMultipleColumns() throws IOException {
- String[] names = { "id$1", "id1$" };
- String[] types = { "INT", "INT" };
- String[] vals = { "1987", "1988" };
- this.setCurTableName("Non_Std_Character_Test_For_Multiple_Columns");
- createTableWithColTypesAndNames(names, types, vals);
-
- runImport(getOutputArgv(true, null));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- Schema schema = reader.getSchema();
- assertEquals(Schema.Type.RECORD, schema.getType());
- List<Field> fields = schema.getFields();
- assertEquals(types.length, fields.size());
-
- checkField(fields.get(0), "ID_1", Type.INT);
-
- GenericRecord record1 = reader.next();
- assertEquals("ID_1", 1987, record1.get("ID_1"));
- checkField(fields.get(1), "ID1_", Type.INT);
- assertEquals("ID1_", 1988, record1.get("ID1_"));
- }
-
- protected void checkField(Field field, String name, Type type) {
- assertEquals(name, field.name());
- assertEquals(Schema.Type.UNION, field.schema().getType());
- assertEquals(Schema.Type.NULL, field.schema().getTypes().get(0).getType());
- assertEquals(type, field.schema().getTypes().get(1).getType());
- }
-
- @Test
- public void testNullableAvroImport() throws IOException, SQLException {
- String [] types = { "INT" };
- String [] vals = { null };
- createTableWithColTypes(types, vals);
-
- runImport(getOutputArgv(true, null));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
-
- GenericRecord record1 = reader.next();
- assertNull(record1.get("DATA_COL0"));
- }
-
- @Test
- public void testSpecialCharactersInColumnMappingWithConvertion() throws IOException, SQLException {
- // escaping enabled by default
- String [] extraArgsEscapeColNamesWithMapping = { "--map-column-java",
- "INTFIELD1=String,DATA_#_COL0=String,DATA#COL1=String,DATA___COL2=String"};
-
- // disable escaping
- String [] extraArgsEscapingDisables = {"--escape-mapping-column-names", "false"};
-
- // escaping enabled but mapping not provided
- String [] extraArgsEscapingWithoutMapping = {};
-
- checkRecordWithExtraArgs(extraArgsEscapeColNamesWithMapping, "TABLE1");
- checkRecordWithExtraArgs(extraArgsEscapingDisables, "TABLE2");
- checkRecordWithExtraArgs(extraArgsEscapingWithoutMapping, "TABLE3");
- }
-
- private void checkRecordWithExtraArgs(String[] extraArgs, String tableName) throws IOException {
- String date = "2017-01-19";
- String timeStamp = "2017-01-19 14:47:57.112000";
-
- String [] names = {"INTFIELD1", "DATA_#_COL0", "DATA#COL1", "DATA___COL2"};
- String [] types = { "INT", "DATE", "TIMESTAMP", "DECIMAL(2,20)" };
- String [] vals = {"1", "{ts \'" + date + "\'}", "{ts \'" + timeStamp + "\'}", "2e20"};
-
- String [] checkNames = {"INTFIELD1", "DATA___COL0", "DATA_COL1", "DATA___COL2"};
-
- setCurTableName(tableName);
-
- createTableWithColTypesAndNames(names, types, vals);
- runImport(getOutputArgv(true, extraArgs));
-
- Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
- DataFileReader<GenericRecord> reader = read(outputFile);
- GenericRecord record = reader.next();
-
- for (String columnName : checkNames) {
- assertNotNull(record.get(columnName));
- }
-
- removeTableDir();
- }
-
- protected DataFileReader<GenericRecord> read(Path filename) throws IOException {
- Configuration conf = new Configuration();
- if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- FsInput fsInput = new FsInput(filename, conf);
- DatumReader<GenericRecord> datumReader =
- new GenericDatumReader<GenericRecord>();
- return new DataFileReader<GenericRecord>(fsInput, datumReader);
- }
-
- protected void checkSchemaFile(final Schema schema) throws IOException {
- final File schemaFile = new File(schema.getName() + ".avsc");
- assertTrue(schemaFile.exists());
- assertEquals(schema, new Schema.Parser().parse(schemaFile));
- }
-}