You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2009/05/05 08:47:28 UTC
svn commit: r771590 - in /hadoop/core/trunk: CHANGES.txt
src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java
src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java
src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java
Author: cdouglas
Date: Tue May 5 06:47:27 2009
New Revision: 771590
URL: http://svn.apache.org/viewvc?rev=771590&view=rev
Log:
HADOOP-5500. Where field names are absent permit the number of fields to be
sufficient to construct the select query. Contributed by Enis Soztutar
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java
hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java
hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Tue May 5 06:47:27 2009
@@ -308,6 +308,10 @@
HADOOP-5727. Simplify hashcode for ID types. (Shevek via cdouglas)
+ HADOOP-5500. In DBOutputFormat, where field names are absent permit the
+ number of fields to be sufficient to construct the select query. (Enis
+ Soztutar via cdouglas)
+
OPTIMIZATIONS
HADOOP-5595. NameNode does not need to run a replicator to choose a
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBConfiguration.java Tue May 5 06:47:27 2009
@@ -80,6 +80,9 @@
/** Field names in the Output table */
public static final String OUTPUT_FIELD_NAMES_PROPERTY = "mapred.jdbc.output.field.names";
+ /** Number of fields in the Output table */
+ public static final String OUTPUT_FIELD_COUNT_PROPERTY = "mapred.jdbc.output.field.count";
+
/**
* Sets the DB access related fields in the JobConf.
* @param job the job
@@ -212,5 +215,13 @@
job.setStrings(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, fieldNames);
}
+ void setOutputFieldCount(int fieldCount) {
+ job.setInt(DBConfiguration.OUTPUT_FIELD_COUNT_PROPERTY, fieldCount);
+ }
+
+ int getOutputFieldCount() {
+ return job.getInt(OUTPUT_FIELD_COUNT_PROPERTY, 0);
+ }
+
}
Modified: hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java (original)
+++ hadoop/core/trunk/src/mapred/org/apache/hadoop/mapred/lib/db/DBOutputFormat.java Tue May 5 06:47:27 2009
@@ -151,6 +151,10 @@
String tableName = dbConf.getOutputTableName();
String[] fieldNames = dbConf.getOutputFieldNames();
+ if(fieldNames == null) {
+ fieldNames = new String[dbConf.getOutputFieldCount()];
+ }
+
try {
Connection connection = dbConf.getConnection();
PreparedStatement statement = null;
@@ -166,21 +170,42 @@
/**
* Initializes the reduce-part of the job with the appropriate output settings
*
- * @param job
- * The job
- * @param tableName
- * The table to insert data into
- * @param fieldNames
- * The field names in the table. If unknown, supply the appropriate
- * number of nulls.
+ * @param job The job
+ * @param tableName The table to insert data into
+ * @param fieldNames The field names in the table.
*/
public static void setOutput(JobConf job, String tableName, String... fieldNames) {
+ if(fieldNames.length > 0 && fieldNames[0] != null) {
+ DBConfiguration dbConf = setOutput(job, tableName);
+ dbConf.setOutputFieldNames(fieldNames);
+ } else {
+ if(fieldNames.length > 0)
+ setOutput(job, tableName, fieldNames.length);
+ else
+ throw new IllegalArgumentException("Field names must be greater than 0");
+ }
+ }
+
+ /**
+ * Initializes the reduce-part of the job with the appropriate output settings
+ *
+ * @param job The job
+ * @param tableName The table to insert data into
+ * @param fieldCount the number of fields in the table.
+ */
+ public static void setOutput(JobConf job, String tableName, int fieldCount) {
+ DBConfiguration dbConf = setOutput(job, tableName);
+ dbConf.setOutputFieldCount(fieldCount);
+ }
+
+ private static DBConfiguration setOutput(JobConf job, String tableName) {
job.setOutputFormat(DBOutputFormat.class);
job.setReduceSpeculativeExecution(false);
DBConfiguration dbConf = new DBConfiguration(job);
dbConf.setOutputTableName(tableName);
- dbConf.setOutputFieldNames(fieldNames);
+ return dbConf;
}
+
}
Modified: hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java?rev=771590&r1=771589&r2=771590&view=diff
==============================================================================
--- hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java (original)
+++ hadoop/core/trunk/src/test/org/apache/hadoop/mapred/lib/db/TestConstructQuery.java Tue May 5 06:47:27 2009
@@ -1,19 +1,50 @@
package org.apache.hadoop.mapred.lib.db;
-import org.apache.hadoop.io.NullWritable;
+import java.io.IOException;
import junit.framework.TestCase;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.mapred.JobConf;
+
public class TestConstructQuery extends TestCase {
- public void testConstructQuery() {
- DBOutputFormat<DBWritable, NullWritable> format = new DBOutputFormat<DBWritable, NullWritable>();
- String expected = "INSERT INTO hadoop_output (id,name,value) VALUES (?,?,?);";
- String[] fieldNames = new String[] { "id", "name", "value" };
+
+ private String[] fieldNames = new String[] { "id", "name", "value" };
+ private String[] nullFieldNames = new String[] { null, null, null };
+ private String expected = "INSERT INTO hadoop_output (id,name,value) VALUES (?,?,?);";
+ private String nullExpected = "INSERT INTO hadoop_output VALUES (?,?,?);";
+
+ private DBOutputFormat<DBWritable, NullWritable> format
+ = new DBOutputFormat<DBWritable, NullWritable>();
+
+ public void testConstructQuery() {
String actual = format.constructQuery("hadoop_output", fieldNames);
assertEquals(expected, actual);
- expected = "INSERT INTO hadoop_output VALUES (?,?,?);";
- fieldNames = new String[] { null, null, null };
- actual = format.constructQuery("hadoop_output", fieldNames);
+
+ actual = format.constructQuery("hadoop_output", nullFieldNames);
+ assertEquals(nullExpected, actual);
+ }
+
+ public void testSetOutput() throws IOException {
+ JobConf job = new JobConf();
+ DBOutputFormat.setOutput(job, "hadoop_output", fieldNames);
+
+ DBConfiguration dbConf = new DBConfiguration(job);
+ String actual = format.constructQuery(dbConf.getOutputTableName()
+ , dbConf.getOutputFieldNames());
+
assertEquals(expected, actual);
+
+ job = new JobConf();
+ dbConf = new DBConfiguration(job);
+ DBOutputFormat.setOutput(job, "hadoop_output", nullFieldNames.length);
+ assertNull(dbConf.getOutputFieldNames());
+ assertEquals(nullFieldNames.length, dbConf.getOutputFieldCount());
+
+ actual = format.constructQuery(dbConf.getOutputTableName()
+ , new String[dbConf.getOutputFieldCount()]);
+
+ assertEquals(nullExpected, actual);
}
+
}