You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2015/08/14 02:11:03 UTC
sqoop git commit: SQOOP-2470: Incremental Hive import with append not
working after validation check for --hive-import and --import
Repository: sqoop
Updated Branches:
refs/heads/trunk 73ef0c133 -> de6fa8279
SQOOP-2470: Incremental Hive import with append not working after validation check for --hive-import and --import
(Aaditya Raj via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/de6fa827
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/de6fa827
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/de6fa827
Branch: refs/heads/trunk
Commit: de6fa8279d8125afb4fd2d7f1e8c7e731d70451d
Parents: 73ef0c1
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Aug 13 17:10:42 2015 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Aug 13 17:10:42 2015 -0700
----------------------------------------------------------------------
.../org/apache/sqoop/tool/BaseSqoopTool.java | 4 +-
.../cloudera/sqoop/TestIncrementalImport.java | 63 +++++++++++++++++++-
testdata/hive/scripts/incrementalHiveAppend10.q | 17 ++++++
testdata/hive/scripts/incrementalHiveAppend20.q | 17 ++++++
.../hive/scripts/incrementalHiveAppendEmpty.q | 17 ++++++
5 files changed, 114 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 4e2e66d..a7f6aec 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -40,6 +40,7 @@ import org.apache.sqoop.util.password.CredentialProviderHelper;
import com.cloudera.sqoop.ConnFactory;
import com.cloudera.sqoop.Sqoop;
import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.IncrementalMode;
import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
import com.cloudera.sqoop.cli.RelatedOptions;
import com.cloudera.sqoop.cli.ToolOptions;
@@ -1398,7 +1399,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool {
}
if (options.doHiveImport()
- && options.isAppendMode()) {
+ && options.isAppendMode()
+ && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) {
throw new InvalidOptionsException("Append mode for hive imports is not "
+ " yet supported. Please remove the parameter --append-mode");
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/src/test/com/cloudera/sqoop/TestIncrementalImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java
index a6680e8..3e3f929 100644
--- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java
+++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java
@@ -19,6 +19,7 @@
package com.cloudera.sqoop;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.Connection;
@@ -29,9 +30,7 @@ import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-
import junit.framework.TestCase;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -39,7 +38,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
-
+import org.apache.sqoop.hive.HiveImport;
import com.cloudera.sqoop.manager.ConnManager;
import com.cloudera.sqoop.manager.HsqldbManager;
import com.cloudera.sqoop.manager.ManagerFactory;
@@ -1216,6 +1215,64 @@ public class TestIncrementalImport extends TestCase {
runJob(TABLE_NAME);
assertDirOfNumbers(TABLE_NAME, 20);
}
+
+ public void testIncrementalHiveAppendEmptyThenFull() throws Exception {
+ // This is to test Incremental Hive append feature. SQOOP-2470
+ final String TABLE_NAME = "incrementalHiveAppendEmptyThenFull";
+ Configuration conf = newConf();
+ conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY,
+ InstrumentHsqldbManagerFactory.class.getName());
+ clearDir(TABLE_NAME);
+ createIdTable(TABLE_NAME, 0);
+ List<String> args = new ArrayList<String>();
+ args.add("--connect");
+ args.add(SOURCE_DB_URL);
+ args.add("--table");
+ args.add(TABLE_NAME);
+ args.add("--warehouse-dir");
+ args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
+ args.add("--hive-import");
+ args.add("--hive-table");
+ args.add(TABLE_NAME + "hive");
+ args.add("--incremental");
+ args.add("append");
+ args.add("--check-column");
+ args.add("id");
+ args.add("-m");
+ args.add("1");
+ createJob(TABLE_NAME, args, conf);
+ HiveImport.setTestMode(true);
+ String hiveHome = org.apache.sqoop.SqoopOptions.getHiveHomeDefault();
+ assertNotNull("hive.home was not set", hiveHome);
+ String testDataPath = new Path(new Path(hiveHome), "scripts/"
+ + "incrementalHiveAppendEmpty.q").toString();
+ System.clearProperty("expected.script");
+ System.setProperty("expected.script",
+ new File(testDataPath).getAbsolutePath());
+ runJob(TABLE_NAME);
+ assertDirOfNumbers(TABLE_NAME, 0);
+ // Now add some rows.
+ insertIdRows(TABLE_NAME, 0, 10);
+ String testDataPath10 = new Path(new Path(hiveHome), "scripts/"
+ + "incrementalHiveAppend10.q").toString();
+ System.clearProperty("expected.script");
+ System.setProperty("expected.script",
+ new File(testDataPath10).getAbsolutePath());
+ System.getProperty("expected.script");
+ // Running the job a second time should import 10 rows.
+ runJob(TABLE_NAME);
+ assertDirOfNumbers(TABLE_NAME, 10);
+ // Add some more rows.
+ insertIdRows(TABLE_NAME, 10, 20);
+ String testDataPath20 = new Path(new Path(hiveHome), "scripts/"
+ + "incrementalHiveAppend20.q").toString();
+ System.clearProperty("expected.script");
+ System.setProperty("expected.script",
+ new File(testDataPath20).getAbsolutePath());
+ // Import only those rows.
+ runJob(TABLE_NAME);
+ assertDirOfNumbers(TABLE_NAME, 20);
+ }
// SQOOP-1890
public void testTableNameWithSpecialCharacters() throws Exception {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppend10.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/incrementalHiveAppend10.q b/testdata/hive/scripts/incrementalHiveAppend10.q
new file mode 100644
index 0000000..383c7b4
--- /dev/null
+++ b/testdata/hive/scripts/incrementalHiveAppend10.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppend20.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/incrementalHiveAppend20.q b/testdata/hive/scripts/incrementalHiveAppend20.q
new file mode 100644
index 0000000..383c7b4
--- /dev/null
+++ b/testdata/hive/scripts/incrementalHiveAppend20.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppendEmpty.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/incrementalHiveAppendEmpty.q b/testdata/hive/scripts/incrementalHiveAppendEmpty.q
new file mode 100644
index 0000000..383c7b4
--- /dev/null
+++ b/testdata/hive/scripts/incrementalHiveAppendEmpty.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO TABLE `incrementalHiveAppendEmptyThenFullhive`;
\ No newline at end of file