You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/07/12 04:53:48 UTC
[32/50] [abbrv] hadoop git commit: MAPREDUCE-6246. DBOutputFormat.java appending extra semicolon to query which is incompatible with DB2. Contributed by ramtin and Gergely Novák.
MAPREDUCE-6246. DBOutputFormat.java appending extra semicolon to query which is incompatible with DB2. Contributed by ramtin and Gergely Novák.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f484a6ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f484a6ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f484a6ff
Branch: refs/heads/YARN-5972
Commit: f484a6ff602d48413556a1d046670e2003c71c2e
Parents: f10864a
Author: Junping Du <ju...@apache.org>
Authored: Fri Jul 7 13:23:43 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Fri Jul 7 13:26:16 2017 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/lib/db/DBOutputFormat.java | 15 ++++++-
.../mapreduce/lib/db/TestDBOutputFormat.java | 45 ++++++++++++++++++++
2 files changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f484a6ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
index 2e3a9d8..c222bf5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBOutputFormat.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.lib.db;
import java.io.IOException;
import java.sql.Connection;
+import java.sql.DatabaseMetaData;
import java.sql.PreparedStatement;
import java.sql.SQLException;
@@ -51,6 +52,8 @@ public class DBOutputFormat<K extends DBWritable, V>
extends OutputFormat<K,V> {
private static final Log LOG = LogFactory.getLog(DBOutputFormat.class);
+ public String dbProductName = "DEFAULT";
+
public void checkOutputSpecs(JobContext context)
throws IOException, InterruptedException {}
@@ -158,7 +161,12 @@ extends OutputFormat<K,V> {
query.append(",");
}
}
- query.append(");");
+
+ if (dbProductName.startsWith("DB2") || dbProductName.startsWith("ORACLE")) {
+ query.append(")");
+ } else {
+ query.append(");");
+ }
return query.toString();
}
@@ -177,7 +185,10 @@ extends OutputFormat<K,V> {
try {
Connection connection = dbConf.getConnection();
PreparedStatement statement = null;
-
+
+ DatabaseMetaData dbMeta = connection.getMetaData();
+ this.dbProductName = dbMeta.getDatabaseProductName().toUpperCase();
+
statement = connection.prepareStatement(
constructQuery(tableName, fieldNames));
return new DBRecordWriter(connection, statement);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f484a6ff/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
index 014855f..e547c8a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/lib/db/TestDBOutputFormat.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.mapreduce.lib.db;
import java.io.IOException;
+import java.lang.reflect.Field;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
@@ -26,6 +28,7 @@ import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
public class TestDBOutputFormat {
private String[] fieldNames = new String[] { "id", "name", "value" };
@@ -47,6 +50,48 @@ public class TestDBOutputFormat {
}
@Test
+ public void testDB2ConstructQuery() {
+ String db2expected = StringUtils.removeEnd(expected, ";");
+ String db2nullExpected = StringUtils.removeEnd(nullExpected, ";");
+
+ try {
+ Class<?> clazz = this.format.getClass();
+ Field field = clazz.getDeclaredField("dbProductName");
+ field.setAccessible(true);
+ field.set(format, "DB2");
+ } catch (IllegalAccessException | NoSuchFieldException e) {
+ fail(e.getMessage());
+ }
+
+ String actual = format.constructQuery("hadoop_output", fieldNames);
+ assertEquals(db2expected, actual);
+
+ actual = format.constructQuery("hadoop_output", nullFieldNames);
+ assertEquals(db2nullExpected, actual);
+ }
+
+ @Test
+ public void testORACLEConstructQuery() {
+ String oracleExpected = StringUtils.removeEnd(expected, ";");
+ String oracleNullExpected = StringUtils.removeEnd(nullExpected, ";");
+
+ try {
+ Class<?> clazz = this.format.getClass();
+ Field field = clazz.getDeclaredField("dbProductName");
+ field.setAccessible(true);
+ field.set(format, "ORACLE");
+ } catch (IllegalAccessException | NoSuchFieldException e) {
+ fail(e.getMessage());
+ }
+
+ String actual = format.constructQuery("hadoop_output", fieldNames);
+ assertEquals(oracleExpected, actual);
+
+ actual = format.constructQuery("hadoop_output", nullFieldNames);
+ assertEquals(oracleNullExpected, actual);
+ }
+
+ @Test
public void testSetOutput() throws IOException {
Job job = Job.getInstance(new Configuration());
DBOutputFormat.setOutput(job, "hadoop_output", fieldNames);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org