You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2014/08/14 02:28:51 UTC
git commit: SQOOP-1429: Fix native characters usage for SqlServer
object names
Repository: sqoop
Updated Branches:
refs/heads/trunk d2b3ab971 -> 74ec89452
SQOOP-1429: Fix native characters usage for SqlServer object names
(Venkat Ranganathan via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/74ec8945
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/74ec8945
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/74ec8945
Branch: refs/heads/trunk
Commit: 74ec89452b574f29fb0ef3615c18184c86a559cc
Parents: d2b3ab9
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Wed Aug 13 17:28:32 2014 -0700
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Wed Aug 13 17:28:32 2014 -0700
----------------------------------------------------------------------
.../apache/sqoop/manager/SQLServerManager.java | 23 +++++++++++++---
.../mapreduce/db/DataDrivenDBInputFormat.java | 6 +++--
.../sqoop/mapreduce/db/NTextSplitter.java | 28 ++++++++++++++++++++
.../apache/sqoop/mapreduce/db/TextSplitter.java | 17 +++++++++---
.../sqoop/mapreduce/db/TestTextSplitter.java | 9 +++++++
5 files changed, 75 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/74ec8945/src/java/org/apache/sqoop/manager/SQLServerManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/SQLServerManager.java b/src/java/org/apache/sqoop/manager/SQLServerManager.java
index 625f7cd..fdd4e91 100644
--- a/src/java/org/apache/sqoop/manager/SQLServerManager.java
+++ b/src/java/org/apache/sqoop/manager/SQLServerManager.java
@@ -38,6 +38,7 @@ import com.cloudera.sqoop.mapreduce.JdbcExportJob;
import com.cloudera.sqoop.mapreduce.JdbcUpdateExportJob;
import com.cloudera.sqoop.util.ExportException;
import com.cloudera.sqoop.util.ImportException;
+
import org.apache.sqoop.cli.RelatedOptions;
import org.apache.sqoop.mapreduce.sqlserver.SqlServerExportBatchOutputFormat;
import org.apache.sqoop.mapreduce.sqlserver.SqlServerInputFormat;
@@ -226,15 +227,31 @@ public class SQLServerManager
return "'" + schema + "'";
}
-
@Override
protected String getListColumnsQuery(String tableName) {
return
- super.getListColumnsQuery(tableName)
- + " ORDER BY ORDINAL_POSITION";
+ "SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS "
+ + "WHERE TABLE_SCHEMA = (" + getSchemaQuery() + ") "
+ + " AND TABLE_NAME = N'" + tableName + "' "
+ + " ORDER BY ORDINAL_POSITION";
}
@Override
+ protected String getPrimaryKeyQuery(String tableName) {
+ return
+ "SELECT kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc, "
+ + " INFORMATION_SCHEMA.KEY_COLUMN_USAGE kcu "
+ + "WHERE tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA "
+ + " AND tc.TABLE_NAME = kcu.TABLE_NAME "
+ + " AND tc.CONSTRAINT_SCHEMA = kcu.CONSTRAINT_SCHEMA "
+ + " AND tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME "
+ + " AND tc.TABLE_SCHEMA = (" + getSchemaQuery() + ") "
+ + " AND tc.TABLE_NAME = N'" + tableName + "' "
+ + " AND tc.CONSTRAINT_TYPE = 'PRIMARY KEY'";
+ }
+
+
+ @Override
public String escapeColName(String colName) {
return escapeObjectName(colName);
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/74ec8945/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java b/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
index 6b30820..2c59fe5 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
@@ -97,10 +97,12 @@ public class DataDrivenDBInputFormat<T extends DBWritable>
case Types.DOUBLE:
return new FloatSplitter();
- case Types.CHAR:
+ case Types.NVARCHAR:
case Types.NCHAR:
+ return new NTextSplitter();
+
+ case Types.CHAR:
case Types.VARCHAR:
- case Types.NVARCHAR:
case Types.LONGVARCHAR:
return new TextSplitter();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/74ec8945/src/java/org/apache/sqoop/mapreduce/db/NTextSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/NTextSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/NTextSplitter.java
new file mode 100644
index 0000000..805deed
--- /dev/null
+++ b/src/java/org/apache/sqoop/mapreduce/db/NTextSplitter.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.mapreduce.db;
+
+/**
+ * Implement DBSplitter over native text strings.
+ */
+public class NTextSplitter extends TextSplitter {
+
+ public NTextSplitter() {
+ setUseNCharStrings(true);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/74ec8945/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
index 6c512aa..d3085cd 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
@@ -39,6 +39,8 @@ public class TextSplitter extends BigDecimalSplitter {
private static final Log LOG = LogFactory.getLog(TextSplitter.class);
+ private boolean useNCharStrings = false;
+
/**
* This method needs to determine the splits between two user-provided
* strings. In the case where the user's strings are 'A' and 'Z', this is
@@ -90,8 +92,8 @@ public class TextSplitter extends BigDecimalSplitter {
// divide cleanly.
int numSplits = ConfigurationHelper.getConfNumMaps(conf);
- String lowClausePrefix = colName + " >= '";
- String highClausePrefix = colName + " < '";
+ String lowClausePrefix = colName + " >= " + (useNCharStrings ? "N'" : "'");
+ String highClausePrefix = colName + " < " + (useNCharStrings ? "N'" : "'");
// If there is a common prefix between minString and maxString, establish
// it and pull it out of minString and maxString.
@@ -123,7 +125,8 @@ public class TextSplitter extends BigDecimalSplitter {
if (i == splitStrings.size() - 1) {
// This is the last one; use a closed interval.
splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(
- lowClausePrefix + start + "'", colName + " <= '" + end + "'"));
+ lowClausePrefix + start + "'", colName
+ + " <= " + (useNCharStrings ? "N'" : "'") + end + "'"));
} else {
// Normal open-interval case.
splits.add(new DataDrivenDBInputFormat.DataDrivenDBInputSplit(
@@ -225,4 +228,12 @@ public class TextSplitter extends BigDecimalSplitter {
return sb.toString();
}
+
+ public void setUseNCharStrings(boolean use) {
+ useNCharStrings = use;
+ }
+
+ public boolean isUseNCharStrings() {
+ return useNCharStrings;
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/74ec8945/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java b/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
index e3d0c0c..9c007d3 100644
--- a/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
+++ b/src/test/org/apache/sqoop/mapreduce/db/TestTextSplitter.java
@@ -134,4 +134,13 @@ public class TestTextSplitter extends TestCase {
assertEquals("Hardy", splits.get(splits.size() -1));
assertEquals(6, splits.size());
}
+
+ public void testNChar() throws SQLException {
+ // Splits between 'Hand' and 'Hardy'
+ NTextSplitter splitter = new NTextSplitter();
+ assertEquals(true, splitter.isUseNCharStrings());
+ TextSplitter splitter2 = new TextSplitter();
+ assertEquals(false, splitter2.isUseNCharStrings());
+ }
+
}