You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ve...@apache.org on 2016/03/04 20:18:04 UTC
sqoop git commit: SQOOP-2863 Properly escape column names for
generated INSERT statements (Jarek Jarcec Cecho via Venkat Ranganathan)
Repository: sqoop
Updated Branches:
refs/heads/trunk 2eecf6801 -> d21699e2a
SQOOP-2863 Properly escape column names for generated INSERT statements
(Jarek Jarcec Cecho via Venkat Ranganathan)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d21699e2
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d21699e2
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d21699e2
Branch: refs/heads/trunk
Commit: d21699e2ab00829a90ffaf5ada1b87da69683157
Parents: 2eecf68
Author: Venkat Ranganathan <ve...@hortonworks.com>
Authored: Fri Mar 4 11:16:53 2016 -0800
Committer: Venkat Ranganathan <ve...@hortonworks.com>
Committed: Fri Mar 4 11:17:51 2016 -0800
----------------------------------------------------------------------
src/java/org/apache/sqoop/manager/ConnManager.java | 15 +++++++++++++++
.../apache/sqoop/mapreduce/JdbcCallExportJob.java | 2 +-
.../org/apache/sqoop/mapreduce/JdbcExportJob.java | 4 ++--
.../apache/sqoop/mapreduce/JdbcUpdateExportJob.java | 3 +--
.../apache/sqoop/mapreduce/JdbcUpsertExportJob.java | 2 +-
.../netezza/NetezzaExternalTableExportJob.java | 4 ++--
src/test/com/cloudera/sqoop/TestAvroExport.java | 6 +++---
src/test/com/cloudera/sqoop/TestExport.java | 11 ++++++-----
src/test/com/cloudera/sqoop/TestParquetExport.java | 6 +++---
.../cloudera/sqoop/testutil/ExportJobTestCase.java | 12 ++++++------
.../org/apache/sqoop/TestExportUsingProcedure.java | 5 +++--
11 files changed, 43 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/java/org/apache/sqoop/manager/ConnManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ConnManager.java b/src/java/org/apache/sqoop/manager/ConnManager.java
index f98feb3..1811ce0 100644
--- a/src/java/org/apache/sqoop/manager/ConnManager.java
+++ b/src/java/org/apache/sqoop/manager/ConnManager.java
@@ -604,6 +604,21 @@ public abstract class ConnManager {
}
/**
+ * Variant of escapeColName() method that will escape whole column name array.
+ *
+ * @param colNames Column names as provided by the user, etc.
+ * @return
+ */
+ public String [] escapeColNames(String ...colNames) {
+ String [] escaped = new String[colNames.length];
+ int i = 0;
+ for(String colName : colNames) {
+ escaped[i++] = escapeColName(colName);
+ }
+ return escaped;
+ }
+
+ /**
* When using a table name in a generated SQL query, how (if at all)
* should we escape that column name? e.g., a table named "table"
* may need to be quoted with backtiks: "`table`".
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java b/src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java
index 2459698..f8594c7 100644
--- a/src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java
@@ -91,7 +91,7 @@ public class JdbcCallExportJob extends JdbcExportJob {
DBOutputFormat.setOutput(
job,
mgr.escapeTableName(procedureName),
- colNames);
+ mgr.escapeColNames(colNames));
job.setOutputFormatClass(getOutputFormatClass());
job.getConfiguration().set(SQOOP_EXPORT_TABLE_CLASS_KEY, tableClassName);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java b/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
index 78df33c..626119b 100644
--- a/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
@@ -172,9 +172,9 @@ public class JdbcExportJob extends ExportJobBase {
}
if (mgr.escapeTableNameOnExport()) {
- DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), colNames);
+ DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), mgr.escapeColNames(colNames));
} else {
- DBOutputFormat.setOutput(job, tableName, colNames);
+ DBOutputFormat.setOutput(job, tableName, mgr.escapeColNames(colNames));
}
job.setOutputFormatClass(getOutputFormatClass());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/java/org/apache/sqoop/mapreduce/JdbcUpdateExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JdbcUpdateExportJob.java b/src/java/org/apache/sqoop/mapreduce/JdbcUpdateExportJob.java
index 8fa420e..c85602c 100644
--- a/src/java/org/apache/sqoop/mapreduce/JdbcUpdateExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/JdbcUpdateExportJob.java
@@ -134,8 +134,7 @@ public class JdbcUpdateExportJob extends ExportJobBase {
outColNames[j++] = colNames[i];
}
}
- DBOutputFormat.setOutput(job,
- mgr.escapeTableName(tableName), outColNames);
+ DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), mgr.escapeColNames(outColNames));
job.setOutputFormatClass(getOutputFormatClass());
job.getConfiguration().set(SQOOP_EXPORT_TABLE_CLASS_KEY, tableClassName);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/java/org/apache/sqoop/mapreduce/JdbcUpsertExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JdbcUpsertExportJob.java b/src/java/org/apache/sqoop/mapreduce/JdbcUpsertExportJob.java
index 0a9bf7f..8e9d1b5 100644
--- a/src/java/org/apache/sqoop/mapreduce/JdbcUpsertExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/JdbcUpsertExportJob.java
@@ -74,7 +74,7 @@ public class JdbcUpsertExportJob extends JdbcUpdateExportJob {
throw new IOException(
"Export column names could not be determined for " + tableName);
}
- DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), colNames);
+ DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), mgr.escapeColNames(colNames));
String updateKeyColumns = options.getUpdateKeyCol();
if (null == updateKeyColumns) {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java b/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
index 117cc3f..b035d86 100644
--- a/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/netezza/NetezzaExternalTableExportJob.java
@@ -129,9 +129,9 @@ public class NetezzaExternalTableExportJob extends ExportJobBase {
}
if (mgr.escapeTableNameOnExport()) {
- DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), colNames);
+ DBOutputFormat.setOutput(job, mgr.escapeTableName(tableName), mgr.escapeColNames(colNames));
} else {
- DBOutputFormat.setOutput(job, tableName, colNames);
+ DBOutputFormat.setOutput(job, tableName, mgr.escapeColNames(colNames));
}
job.setOutputFormatClass(getOutputFormatClass());
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/test/com/cloudera/sqoop/TestAvroExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAvroExport.java b/src/test/com/cloudera/sqoop/TestAvroExport.java
index 137a6e1..f91cd48 100644
--- a/src/test/com/cloudera/sqoop/TestAvroExport.java
+++ b/src/test/com/cloudera/sqoop/TestAvroExport.java
@@ -234,11 +234,11 @@ public class TestAvroExport extends ExportJobTestCase {
StringBuilder sb = new StringBuilder();
sb.append("CREATE TABLE ");
sb.append(getTableName());
- sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
+ sb.append(" (\"id\" INT NOT NULL PRIMARY KEY, \"msg\" VARCHAR(64)");
int colNum = 0;
for (ColumnGenerator gen : extraColumns) {
if (gen.getColumnType() != null) {
- sb.append(", " + forIdx(colNum++) + " " + gen.getColumnType());
+ sb.append(", \"" + forIdx(colNum++) + "\" " + gen.getColumnType());
}
}
sb.append(")");
@@ -262,7 +262,7 @@ public class TestAvroExport extends ExportJobTestCase {
LOG.info("Verifying column " + colName + " has value " + expectedVal);
PreparedStatement statement = conn.prepareStatement(
- "SELECT " + colName + " FROM " + getTableName() + " WHERE id = " + id,
+ "SELECT \"" + colName + "\" FROM " + getTableName() + " WHERE \"id\" = " + id,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
Object actualVal = null;
try {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/test/com/cloudera/sqoop/TestExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestExport.java b/src/test/com/cloudera/sqoop/TestExport.java
index 0b650af..c4c264d 100644
--- a/src/test/com/cloudera/sqoop/TestExport.java
+++ b/src/test/com/cloudera/sqoop/TestExport.java
@@ -1,4 +1,5 @@
/**
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -279,10 +280,10 @@ public class TestExport extends ExportJobTestCase {
StringBuilder sb = new StringBuilder();
sb.append("CREATE TABLE ");
sb.append(getTableName());
- sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
+ sb.append(" (\"id\" INT NOT NULL PRIMARY KEY, \"msg\" VARCHAR(64)");
int colNum = 0;
for (ColumnGenerator gen : extraColumns) {
- sb.append(", " + forIdx(colNum++) + " " + gen.getType());
+ sb.append(", \"" + forIdx(colNum++) + "\" " + gen.getType());
}
sb.append(")");
@@ -325,10 +326,10 @@ public class TestExport extends ExportJobTestCase {
StringBuilder sb = new StringBuilder();
sb.append("CREATE TABLE ");
sb.append(stageTableName);
- sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
+ sb.append(" (\"id\" INT NOT NULL PRIMARY KEY, \"msg\" VARCHAR(64)");
int colNum = 0;
for (ColumnGenerator gen : extraColumns) {
- sb.append(", " + forIdx(colNum++) + " " + gen.getType());
+ sb.append(", \"" + forIdx(colNum++) + "\" " + gen.getType());
}
sb.append(")");
@@ -362,7 +363,7 @@ public class TestExport extends ExportJobTestCase {
LOG.info("Verifying column " + colName + " has value " + expectedVal);
PreparedStatement statement = conn.prepareStatement(
- "SELECT " + colName + " FROM " + getTableName() + " WHERE id = " + id,
+ "SELECT \"" + colName + "\" FROM " + getTableName() + " WHERE \"id\" = " + id,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
String actualVal = null;
try {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/test/com/cloudera/sqoop/TestParquetExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestParquetExport.java b/src/test/com/cloudera/sqoop/TestParquetExport.java
index 86b40fb..b938bf8 100644
--- a/src/test/com/cloudera/sqoop/TestParquetExport.java
+++ b/src/test/com/cloudera/sqoop/TestParquetExport.java
@@ -213,11 +213,11 @@ public class TestParquetExport extends ExportJobTestCase {
StringBuilder sb = new StringBuilder();
sb.append("CREATE TABLE ");
sb.append(getTableName());
- sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");
+ sb.append(" (\"id\" INT NOT NULL PRIMARY KEY, \"msg\" VARCHAR(64)");
int colNum = 0;
for (ColumnGenerator gen : extraColumns) {
if (gen.getColumnType() != null) {
- sb.append(", " + forIdx(colNum++) + " " + gen.getColumnType());
+ sb.append(", \"" + forIdx(colNum++) + "\" " + gen.getColumnType());
}
}
sb.append(")");
@@ -241,7 +241,7 @@ public class TestParquetExport extends ExportJobTestCase {
LOG.info("Verifying column " + colName + " has value " + expectedVal);
PreparedStatement statement = conn.prepareStatement(
- "SELECT " + colName + " FROM " + getTableName() + " WHERE id = " + id,
+ "SELECT \"" + colName + "\" FROM " + getTableName() + " WHERE \"id\" = " + id,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
Object actualVal = null;
try {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java b/src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java
index 9a6e8da..ed0dc31 100644
--- a/src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java
+++ b/src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java
@@ -173,7 +173,7 @@ public abstract class ExportJobTestCase extends BaseSqoopTestCase {
/** @return the minimum 'id' value in the table */
protected int getMinRowId(Connection conn) throws SQLException {
PreparedStatement statement = conn.prepareStatement(
- "SELECT MIN(id) FROM " + getTableName(),
+ "SELECT MIN(\"id\") FROM " + getTableName(),
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
int minVal = 0;
try {
@@ -194,7 +194,7 @@ public abstract class ExportJobTestCase extends BaseSqoopTestCase {
/** @return the maximum 'id' value in the table */
protected int getMaxRowId(Connection conn) throws SQLException {
PreparedStatement statement = conn.prepareStatement(
- "SELECT MAX(id) FROM " + getTableName(),
+ "SELECT MAX(\"id\") FROM " + getTableName(),
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
int maxVal = 0;
try {
@@ -266,8 +266,8 @@ public abstract class ExportJobTestCase extends BaseSqoopTestCase {
assertEquals("Maximum row had invalid id", expectedNumRecords - 1, maxVal);
// Check that the string values associated with these points match up.
- statement = conn.prepareStatement("SELECT msg FROM " + getTableName()
- + " WHERE id = " + minVal,
+ statement = conn.prepareStatement("SELECT \"msg\" FROM " + getTableName()
+ + " WHERE \"id\" = " + minVal,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
String minMsg = "";
try {
@@ -285,8 +285,8 @@ public abstract class ExportJobTestCase extends BaseSqoopTestCase {
assertEquals("Invalid msg field for min value", getMsgPrefix() + minVal,
minMsg);
- statement = conn.prepareStatement("SELECT msg FROM " + getTableName()
- + " WHERE id = " + maxVal,
+ statement = conn.prepareStatement("SELECT \"msg\" FROM " + getTableName()
+ + " WHERE \"id\" = " + maxVal,
ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
String maxMsg = "";
try {
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d21699e2/src/test/org/apache/sqoop/TestExportUsingProcedure.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/TestExportUsingProcedure.java b/src/test/org/apache/sqoop/TestExportUsingProcedure.java
index 98ebf3c..6509124 100644
--- a/src/test/org/apache/sqoop/TestExportUsingProcedure.java
+++ b/src/test/org/apache/sqoop/TestExportUsingProcedure.java
@@ -235,10 +235,11 @@ public class TestExportUsingProcedure extends TestExport {
StringBuilder sql = new StringBuilder("insert into ");
sql.append(instanceForProcedure.getTableName());
- sql.append("(id, msg");
+ sql.append("(\"id\", \"msg\"");
for (int i = 0; i < instanceForProcedure.names.length; ++i) {
- sql.append(", ");
+ sql.append(", \"");
sql.append(instanceForProcedure.names[i]);
+ sql.append("\"");
}
sql.append(") values (");
for (int i = 0; i < instanceForProcedure.names.length + 2; ++i) {