You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by nz...@apache.org on 2010/09/27 21:17:09 UTC
svn commit: r1001881 - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/files/
data/scripts/ jdbc/src/java/org/apache/hadoop/hive/jdbc/
jdbc/src/test/org/apache/hadoop/hive/jdbc/
ql/src/java/org/apache/hadoop/hive/ql/ex...
Author: nzhang
Date: Mon Sep 27 19:17:08 2010
New Revision: 1001881
URL: http://svn.apache.org/viewvc?rev=1001881&view=rev
Log:
HIVE-1378. Return value for map, array, and struct needs to return a string (Steven Wong via Ning Zhang)
Added:
hadoop/hive/trunk/data/files/datatypes.txt
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/build.xml
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/conf/hive-default.xml
hadoop/hive/trunk/data/scripts/input20_script
hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java
hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveConnection.java
hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java
hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java
hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java
hadoop/hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out_0.17
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Sep 27 19:17:08 2010
@@ -157,6 +157,9 @@ Trunk - Unreleased
codec 1.4
(Todd Lipcon via jvs)
+ HIVE-1378. Return value for map, array, and struct needs to return a string
+ (Steven Wong via Ning Zhang)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.xml?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/build.xml (original)
+++ hadoop/hive/trunk/build.xml Mon Sep 27 19:17:08 2010
@@ -172,6 +172,7 @@
<!-- create an html report from junit output files -->
<target name="testreport"
description="Generate JUnit HTML test report">
+ <mkdir dir="${test.build.dir}"/>
<junitreport todir="${test.build.dir}">
<fileset dir="${build.dir.hive}">
<include name="**/TEST-*.xml"/>
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Sep 27 19:17:08 2010
@@ -301,6 +301,8 @@ public class HiveConf extends Configurat
HIVEHARPARENTDIRSETTABLE("hive.archive.har.parentdir.settable", false),
HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true),
+ // Serde for FetchTask
+ HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe")
;
Modified: hadoop/hive/trunk/conf/hive-default.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/conf/hive-default.xml?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/conf/hive-default.xml (original)
+++ hadoop/hive/trunk/conf/hive-default.xml Mon Sep 27 19:17:08 2010
@@ -705,6 +705,12 @@
</property>
<property>
+ <name>hive.fetch.output.serde</name>
+ <value>org.apache.hadoop.hive.serde2.DelimitedJSONSerDe</value>
+ <description>The serde used by FetchTask to serialize the fetch output.</description>
+</property>
+
+<property>
<name>hive.exec.mode.local.auto</name>
<value>false</value>
<description> Let hive determine whether to run in local mode automatically </description>
Added: hadoop/hive/trunk/data/files/datatypes.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/datatypes.txt?rev=1001881&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/datatypes.txt (added)
+++ hadoop/hive/trunk/data/files/datatypes.txt Mon Sep 27 19:17:08 2010
@@ -0,0 +1,3 @@
+\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N\N
+-1false-1.1\N\N\N-1-1-1.0-1\N\N
+1true1.11121x2ykva92.2111.01abcd1111213142212212x1abcd2
Modified: hadoop/hive/trunk/data/scripts/input20_script
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/scripts/input20_script?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/data/scripts/input20_script (original)
+++ hadoop/hive/trunk/data/scripts/input20_script Mon Sep 27 19:17:08 2010
@@ -2,4 +2,4 @@
# This script outputs a row of the following format
# <# of values for the key> <key>_<key>
-uniq -c | sed "s@^ *@@" | sed "s@\t@_@" | sed "s@ @\t@"
+uniq -c | sed "s@^ *@@" | sed "s@ @_@" | sed "s@ @ @"
Modified: hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java (original)
+++ hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveBaseResultSet.java Mon Sep 27 19:17:08 2010
@@ -48,7 +48,7 @@ import java.util.Map;
public abstract class HiveBaseResultSet implements ResultSet{
protected SQLWarning warningChain = null;
protected boolean wasNull = false;
- protected List<?> row;
+ protected List<Object> row;
protected List<String> columnNames;
protected List<String> columnTypes;
@@ -135,8 +135,14 @@ public abstract class HiveBaseResultSet
public boolean getBoolean(int columnIndex) throws SQLException {
Object obj = getObject(columnIndex);
- if (Number.class.isInstance(obj)) {
+ if (Boolean.class.isInstance(obj)) {
+ return (Boolean) obj;
+ } else if (obj == null) {
+ return false;
+ } else if (Number.class.isInstance(obj)) {
return ((Number) obj).intValue() != 0;
+ } else if (String.class.isInstance(obj)) {
+ return !((String) obj).equals("0");
}
throw new SQLException("Cannot convert column " + columnIndex + " to boolean");
}
@@ -149,6 +155,8 @@ public abstract class HiveBaseResultSet
Object obj = getObject(columnIndex);
if (Number.class.isInstance(obj)) {
return ((Number) obj).byteValue();
+ } else if (obj == null) {
+ return 0;
}
throw new SQLException("Cannot convert column " + columnIndex + " to byte");
}
Modified: hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveConnection.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveConnection.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveConnection.java (original)
+++ hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveConnection.java Mon Sep 27 19:17:08 2010
@@ -58,7 +58,6 @@ public class HiveConnection implements j
private HiveInterface client;
private boolean isClosed = true;
private SQLWarning warningChain = null;
- private String uri;
private static final String URI_PREFIX = "jdbc:hive://";
@@ -66,7 +65,6 @@ public class HiveConnection implements j
* TODO: - parse uri (use java.net.URI?).
*/
public HiveConnection(String uri, Properties info) throws SQLException {
- this.uri = uri;
session = new JdbcSessionState(new HiveConf(SessionState.class));
session.in = null;
session.out = null;
@@ -110,6 +108,14 @@ public class HiveConnection implements j
}
}
isClosed = false;
+ configureConnection();
+ }
+
+ private void configureConnection() throws SQLException {
+ Statement stmt = createStatement();
+ stmt.execute(
+ "set hive.fetch.output.serde = org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
+ stmt.close();
}
/*
Modified: hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java (original)
+++ hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveQueryResultSet.java Mon Sep 27 19:17:08 2010
@@ -18,97 +18,106 @@
package org.apache.hadoop.hive.jdbc;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.serde.Constants;
-import org.apache.hadoop.hive.serde2.dynamic_type.DynamicSerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.service.HiveInterface;
import org.apache.hadoop.io.BytesWritable;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-
/**
* HiveQueryResultSet.
*
*/
public class HiveQueryResultSet extends HiveBaseResultSet {
+
+ public static final Log LOG = LogFactory.getLog(HiveQueryResultSet.class);
+
private HiveInterface client;
- private DynamicSerDe ds;
+ private SerDe serde;
private int maxRows = 0;
private int rowsFetched = 0;
- @SuppressWarnings("unchecked")
public HiveQueryResultSet(HiveInterface client, int maxRows) throws SQLException {
this.client = client;
- row = new ArrayList();
this.maxRows = maxRows;
- initDynamicSerde();
+ initSerde();
+ row = Arrays.asList(new Object[columnNames.size()]);
}
- @SuppressWarnings("unchecked")
public HiveQueryResultSet(HiveInterface client) throws SQLException {
this(client, 0);
}
/**
- * Instantiate the dynamic serde used to deserialize the result row.
+ * Instantiate the serde used to deserialize the result rows.
*/
- private void initDynamicSerde() throws SQLException {
+ private void initSerde() throws SQLException {
try {
- Schema fullSchema = client.getThriftSchema();
+ Schema fullSchema = client.getSchema();
List<FieldSchema> schema = fullSchema.getFieldSchemas();
columnNames = new ArrayList<String>();
columnTypes = new ArrayList<String>();
-
- String serDDL;
+ StringBuilder namesSb = new StringBuilder();
+ StringBuilder typesSb = new StringBuilder();
if ((schema != null) && (!schema.isEmpty())) {
- serDDL = new String("struct result { ");
for (int pos = 0; pos < schema.size(); pos++) {
if (pos != 0) {
- serDDL = serDDL.concat(",");
+ namesSb.append(",");
+ typesSb.append(",");
}
- columnTypes.add(schema.get(pos).getType());
columnNames.add(schema.get(pos).getName());
- serDDL = serDDL.concat(schema.get(pos).getType());
- serDDL = serDDL.concat(" ");
- serDDL = serDDL.concat(schema.get(pos).getName());
+ columnTypes.add(schema.get(pos).getType());
+ namesSb.append(schema.get(pos).getName());
+ typesSb.append(schema.get(pos).getType());
}
- serDDL = serDDL.concat("}");
- } else {
- serDDL = new String("struct result { string empty }");
}
+ String names = namesSb.toString();
+ String types = typesSb.toString();
+
+ serde = new LazySimpleSerDe();
+ Properties props = new Properties();
+ if (names.length() > 0) {
+ LOG.info("Column names: " + names);
+ props.setProperty(Constants.LIST_COLUMNS, names);
+ }
+ if (types.length() > 0) {
+ LOG.info("Column types: " + types);
+ props.setProperty(Constants.LIST_COLUMN_TYPES, types);
+ }
+ serde.initialize(new Configuration(), props);
- ds = new DynamicSerDe();
- Properties dsp = new Properties();
- dsp.setProperty(Constants.SERIALIZATION_FORMAT,
- org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol.class
- .getName());
- dsp.setProperty(
- org.apache.hadoop.hive.metastore.api.Constants.META_TABLE_NAME,
- "result");
- dsp.setProperty(Constants.SERIALIZATION_DDL, serDDL);
- dsp.setProperty(Constants.SERIALIZATION_LIB, ds.getClass().toString());
- dsp.setProperty(Constants.FIELD_DELIM, "9");
- ds.initialize(new Configuration(), dsp);
} catch (Exception ex) {
ex.printStackTrace();
throw new SQLException("Could not create ResultSet: " + ex.getMessage());
}
}
+ @Override
public void close() throws SQLException {
client = null;
}
/**
* Moves the cursor down one row from its current position.
- *
+ *
* @see java.sql.ResultSet#next()
* @throws SQLException
* if a database access error occurs.
@@ -122,10 +131,28 @@ public class HiveQueryResultSet extends
try {
rowStr = (String) client.fetchOne();
rowsFetched++;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Fetched row string: " + rowStr);
+ }
+
if (!"".equals(rowStr)) {
- Object o = ds.deserialize(new BytesWritable(rowStr.getBytes()));
- row = (ArrayList<?>) o;
+ StructObjectInspector soi = (StructObjectInspector) serde.getObjectInspector();
+ List<? extends StructField> fieldRefs = soi.getAllStructFieldRefs();
+ Object data = serde.deserialize(new BytesWritable(rowStr.getBytes()));
+
+ assert row.size() == fieldRefs.size() : row.size() + ", " + fieldRefs.size();
+ for (int i = 0; i < fieldRefs.size(); i++) {
+ StructField fieldRef = fieldRefs.get(i);
+ ObjectInspector oi = fieldRef.getFieldObjectInspector();
+ Object obj = soi.getStructFieldData(data, fieldRef);
+ row.set(i, convertLazyToJava(obj, oi));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deserialized row: " + row);
+ }
}
+
} catch (Exception ex) {
ex.printStackTrace();
throw new SQLException("Error retrieving next row");
@@ -134,4 +161,21 @@ public class HiveQueryResultSet extends
return !"".equals(rowStr);
}
+ /**
+ * Convert a LazyObject to a standard Java object in compliance with JDBC 3.0 (see JDBC 3.0
+ * Specification, Table B-3: Mapping from JDBC Types to Java Object Types).
+ *
+ * This method is kept consistent with {@link HiveResultSetMetaData#hiveTypeToSqlType}.
+ */
+ private static Object convertLazyToJava(Object o, ObjectInspector oi) {
+ Object obj = ObjectInspectorUtils.copyToStandardObject(o, oi, ObjectInspectorCopyOption.JAVA);
+
+ // for now, expose non-primitive as a string
+ // TODO: expose non-primitive as a structured object while maintaining JDBC compliance
+ if (obj != null && oi.getCategory() != ObjectInspector.Category.PRIMITIVE) {
+ obj = obj.toString();
+ }
+
+ return obj;
+ }
}
Modified: hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java (original)
+++ hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/HiveResultSetMetaData.java Mon Sep 27 19:17:08 2010
@@ -100,23 +100,31 @@ public class HiveResultSetMetaData imple
* @return Integer java.sql.Types values
* @throws SQLException
*/
- public static Integer hiveTypeToSqlType(String type) throws SQLException {
+ public static int hiveTypeToSqlType(String type) throws SQLException {
if ("string".equalsIgnoreCase(type)) {
return Types.VARCHAR;
- } else if ("bool".equalsIgnoreCase(type) || "boolean".equalsIgnoreCase(type)) {
- return Types.BOOLEAN;
} else if ("float".equalsIgnoreCase(type)) {
return Types.FLOAT;
} else if ("double".equalsIgnoreCase(type)) {
return Types.DOUBLE;
- } else if ("byte".equalsIgnoreCase(type) || "tinyint".equalsIgnoreCase(type)) {
+ } else if ("boolean".equalsIgnoreCase(type)) {
+ return Types.BOOLEAN;
+ } else if ("tinyint".equalsIgnoreCase(type)) {
return Types.TINYINT;
- } else if ("i32".equalsIgnoreCase(type) || "int".equalsIgnoreCase(type)) {
+ } else if ("smallint".equalsIgnoreCase(type)) {
+ return Types.SMALLINT;
+ } else if ("int".equalsIgnoreCase(type)) {
return Types.INTEGER;
- } else if ("i64".equalsIgnoreCase(type) || "bigint".equalsIgnoreCase(type)) {
+ } else if ("bigint".equalsIgnoreCase(type)) {
return Types.BIGINT;
+ } else if (type.startsWith("map<")) {
+ return Types.VARCHAR;
+ } else if (type.startsWith("array<")) {
+ return Types.VARCHAR;
+ } else if (type.startsWith("struct<")) {
+ return Types.VARCHAR;
}
- throw new SQLException("Unrecognized column type: " + type);
+ throw new SQLException("Unrecognized column type: " + type);
}
@@ -130,24 +138,34 @@ public class HiveResultSetMetaData imple
throw new SQLException("Invalid column value: " + column);
}
- // we need to convert the thrift type to the SQL type name
+ // we need to convert the Hive type to the SQL type name
// TODO: this would be better handled in an enum
String type = columnTypes.get(column - 1);
if ("string".equalsIgnoreCase(type)) {
return Constants.STRING_TYPE_NAME;
+ } else if ("float".equalsIgnoreCase(type)) {
+ return Constants.FLOAT_TYPE_NAME;
} else if ("double".equalsIgnoreCase(type)) {
return Constants.DOUBLE_TYPE_NAME;
- } else if ("bool".equalsIgnoreCase(type)) {
+ } else if ("boolean".equalsIgnoreCase(type)) {
return Constants.BOOLEAN_TYPE_NAME;
- } else if ("byte".equalsIgnoreCase(type)) {
+ } else if ("tinyint".equalsIgnoreCase(type)) {
return Constants.TINYINT_TYPE_NAME;
- } else if ("i32".equalsIgnoreCase(type)) {
+ } else if ("smallint".equalsIgnoreCase(type)) {
+ return Constants.SMALLINT_TYPE_NAME;
+ } else if ("int".equalsIgnoreCase(type)) {
return Constants.INT_TYPE_NAME;
- } else if ("i64".equalsIgnoreCase(type)) {
+ } else if ("bigint".equalsIgnoreCase(type)) {
return Constants.BIGINT_TYPE_NAME;
+ } else if (type.startsWith("map<")) {
+ return Constants.STRING_TYPE_NAME;
+ } else if (type.startsWith("array<")) {
+ return Constants.STRING_TYPE_NAME;
+ } else if (type.startsWith("struct<")) {
+ return Constants.STRING_TYPE_NAME;
}
- throw new SQLException("Inrecognized column type: " + type);
+ throw new SQLException("Unrecognized column type: " + type);
}
public int getPrecision(int column) throws SQLException {
Modified: hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java (original)
+++ hadoop/hive/trunk/jdbc/src/java/org/apache/hadoop/hive/jdbc/JdbcColumn.java Mon Sep 27 19:17:08 2010
@@ -24,12 +24,12 @@ import java.sql.SQLException;
* Column metadata.
*/
public class JdbcColumn {
- private String columnName;
- private String tableName;
- private String tableCatalog;
- private String type;
- private String comment;
- private int ordinalPos;
+ private final String columnName;
+ private final String tableName;
+ private final String tableCatalog;
+ private final String type;
+ private final String comment;
+ private final int ordinalPos;
JdbcColumn(String columnName, String tableName, String tableCatalog
, String type, String comment, int ordinalPos) {
Modified: hadoop/hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java (original)
+++ hadoop/hive/trunk/jdbc/src/test/org/apache/hadoop/hive/jdbc/TestJdbcDriver.java Mon Sep 27 19:17:08 2010
@@ -49,8 +49,11 @@ public class TestJdbcDriver extends Test
private static String viewComment = "Simple view";
private static String partitionedTableName = "testHiveJdbcDriverPartitionedTable";
private static String partitionedTableComment = "Partitioned table";
+ private static String dataTypeTableName = "testDataTypeTable";
+ private static String dataTypeTableComment = "Table with many column data types";
private final HiveConf conf;
private final Path dataFilePath;
+ private final Path dataTypeDataFilePath;
private Connection con;
private boolean standAloneServer = false;
@@ -60,6 +63,7 @@ public class TestJdbcDriver extends Test
String dataFileDir = conf.get("test.data.files").replace('\\', '/')
.replace("c:", "");
dataFilePath = new Path(dataFileDir, "kv1.txt");
+ dataTypeDataFilePath = new Path(dataFileDir, "datatypes.txt");
standAloneServer = "true".equals(System
.getProperty("test.service.standalone.server"));
}
@@ -118,6 +122,31 @@ public class TestJdbcDriver extends Test
+ " PARTITION (dt='20090619')");
assertFalse(res.next());
+ // drop table. ignore error.
+ try {
+ stmt.executeQuery("drop table " + dataTypeTableName);
+ } catch (Exception ex) {
+ fail(ex.toString());
+ }
+
+ res = stmt.executeQuery("create table " + dataTypeTableName
+ + " (c1 int, c2 boolean, c3 double, c4 string,"
+ + " c5 array<int>, c6 map<int,string>, c7 map<string,string>,"
+ + " c8 struct<r:string,s:int,t:double>,"
+ + " c9 tinyint, c10 smallint, c11 float, c12 bigint,"
+ + " c13 array<array<string>>,"
+ + " c14 map<int, map<int,int>>,"
+ + " c15 struct<r:int,s:struct<a:int,b:string>>,"
+ + " c16 array<struct<m:map<string,string>,n:int>>) comment '"+dataTypeTableComment
+ +"' partitioned by (dt STRING)");
+ assertFalse(res.next());
+
+ // load data
+ res = stmt.executeQuery("load data local inpath '"
+ + dataTypeDataFilePath.toString() + "' into table " + dataTypeTableName
+ + " PARTITION (dt='20090619')");
+ assertFalse(res.next());
+
// drop view. ignore error.
try {
stmt.executeQuery("drop view " + viewName);
@@ -141,6 +170,8 @@ public class TestJdbcDriver extends Test
assertFalse(res.next());
res = stmt.executeQuery("drop table " + partitionedTableName);
assertFalse(res.next());
+ res = stmt.executeQuery("drop table " + dataTypeTableName);
+ assertFalse(res.next());
con.close();
assertTrue("Connection should be closed", con.isClosed());
@@ -173,6 +204,65 @@ public class TestJdbcDriver extends Test
doTestSelectAll(tableName, 100);
}
+ public void testDataTypes() throws Exception {
+ Statement stmt = con.createStatement();
+
+ ResultSet res = stmt.executeQuery(
+ "select * from " + dataTypeTableName + " order by c1");
+ ResultSetMetaData meta = res.getMetaData();
+
+ // row 1
+ assertTrue(res.next());
+ for (int i = 1; i <= meta.getColumnCount(); i++) {
+ assertNull(res.getObject(i));
+ }
+
+ // row 2
+ assertTrue(res.next());
+ assertEquals(-1, res.getInt(1));
+ assertEquals(false, res.getBoolean(2));
+ assertEquals(-1.1d, res.getDouble(3));
+ assertEquals("", res.getString(4));
+ assertEquals("[]", res.getString(5));
+ assertEquals("{}", res.getString(6));
+ assertEquals("{}", res.getString(7));
+ assertEquals("[null, null, null]", res.getString(8));
+ assertEquals(-1, res.getByte(9));
+ assertEquals(-1, res.getShort(10));
+ assertEquals(-1.0f, res.getFloat(11));
+ assertEquals(-1, res.getLong(12));
+ assertEquals("[]", res.getString(13));
+ assertEquals("{}", res.getString(14));
+ assertEquals("[null, null]", res.getString(15));
+ assertEquals("[]", res.getString(16));
+
+ // row 3
+ assertTrue(res.next());
+ assertEquals(1, res.getInt(1));
+ assertEquals(true, res.getBoolean(2));
+ assertEquals(1.1d, res.getDouble(3));
+ assertEquals("1", res.getString(4));
+ assertEquals("[1, 2]", res.getString(5));
+ assertEquals("{1=x, 2=y}", res.getString(6));
+ assertEquals("{k=v}", res.getString(7));
+ assertEquals("[a, 9, 2.2]", res.getString(8));
+ assertEquals(1, res.getByte(9));
+ assertEquals(1, res.getShort(10));
+ assertEquals(1.0f, res.getFloat(11));
+ assertEquals(1, res.getLong(12));
+ assertEquals("[[a, b], [c, d]]", res.getString(13));
+ assertEquals("{1={11=12, 13=14}, 2={21=22}}", res.getString(14));
+ assertEquals("[1, [2, x]]", res.getString(15));
+ assertEquals("[[{}, 1], [{c=d, a=b}, 2]]", res.getString(16));
+
+ // test getBoolean rules on non-boolean columns
+ assertEquals(true, res.getBoolean(1));
+ assertEquals(true, res.getBoolean(4));
+
+ // no more rows
+ assertFalse(res.next());
+ }
+
private void doTestSelectAll(String tableName, int maxRows) throws Exception {
Statement stmt = con.createStatement();
if (maxRows >= 0) {
@@ -197,6 +287,9 @@ public class TestJdbcDriver extends Test
assertEquals("get update count not as expected", 0, stmt.getUpdateCount());
int i = 0;
+ ResultSetMetaData meta = res.getMetaData();
+ assertEquals("Unexpected column count", 2, meta.getColumnCount());
+
boolean moreRow = res.next();
while (moreRow) {
try {
@@ -506,50 +599,82 @@ public class TestJdbcDriver extends Test
public void testResultSetMetaData() throws SQLException {
Statement stmt = con.createStatement();
- ResultSet res = stmt.executeQuery("drop table " + tableName);
+ ResultSet res = stmt.executeQuery(
+ "select c1, c2, c3, c4, c5 as a, c6, c7, c8, c9, c10, c11, c12, " +
+ "c1*2, sentences(null, null, null) as b from " + dataTypeTableName + " limit 1");
+ ResultSetMetaData meta = res.getMetaData();
- // creating a table with tinyint is failing currently so not including
- res = stmt.executeQuery("create table " + tableName
- + " (a string, b boolean, c bigint, d int, f double)");
- res = stmt.executeQuery(
- "select a,b,c,d,f as e,f*2 from " + tableName + " limit 1");
+ assertEquals(14, meta.getColumnCount());
- ResultSetMetaData meta = res.getMetaData();
- assertEquals("Unexpected column count", 6, meta.getColumnCount());
- assertEquals("Unexpected column name", "a", meta.getColumnName(1));
- assertEquals("Unexpected column name", "b", meta.getColumnName(2));
- assertEquals("Unexpected column name", "c", meta.getColumnName(3));
- assertEquals("Unexpected column name", "d", meta.getColumnName(4));
- assertEquals("Unexpected column name", "e", meta.getColumnName(5));
- assertEquals("Unexpected column name", "_c5", meta.getColumnName(6));
- assertEquals("Unexpected column type", Types.VARCHAR, meta.getColumnType(1));
- assertEquals("Unexpected column type", Types.BOOLEAN, meta.getColumnType(2));
- assertEquals("Unexpected column type", Types.BIGINT, meta.getColumnType(3));
- assertEquals("Unexpected column type", Types.INTEGER, meta.getColumnType(4));
- assertEquals("Unexpected column type", Types.DOUBLE, meta.getColumnType(5));
- assertEquals("Unexpected column type", Types.DOUBLE, meta.getColumnType(6));
- assertEquals("Unexpected column type name", "string", meta.getColumnTypeName(1));
- assertEquals("Unexpected column type name", "boolean", meta.getColumnTypeName(2));
- assertEquals("Unexpected column type name", "bigint", meta.getColumnTypeName(3));
- assertEquals("Unexpected column type name", "int", meta.getColumnTypeName(4));
- assertEquals("Unexpected column type name", "double", meta.getColumnTypeName(5));
- assertEquals("Unexpected column type name", "double", meta.getColumnTypeName(6));
- assertEquals("Unexpected column display size", 32, meta.getColumnDisplaySize(1));
- assertEquals("Unexpected column display size", 8, meta.getColumnDisplaySize(2));
- assertEquals("Unexpected column display size", 32, meta.getColumnDisplaySize(3));
- assertEquals("Unexpected column display size", 16, meta.getColumnDisplaySize(4));
- assertEquals("Unexpected column display size", 16, meta.getColumnDisplaySize(5));
- assertEquals("Unexpected column display size", 16, meta.getColumnDisplaySize(6));
+ assertEquals("c1", meta.getColumnName(1));
+ assertEquals("c2", meta.getColumnName(2));
+ assertEquals("c3", meta.getColumnName(3));
+ assertEquals("c4", meta.getColumnName(4));
+ assertEquals("a", meta.getColumnName(5));
+ assertEquals("c6", meta.getColumnName(6));
+ assertEquals("c7", meta.getColumnName(7));
+ assertEquals("c8", meta.getColumnName(8));
+ assertEquals("c9", meta.getColumnName(9));
+ assertEquals("c10", meta.getColumnName(10));
+ assertEquals("c11", meta.getColumnName(11));
+ assertEquals("c12", meta.getColumnName(12));
+ assertEquals("_c12", meta.getColumnName(13));
+ assertEquals("b", meta.getColumnName(14));
+
+ assertEquals(Types.INTEGER, meta.getColumnType(1));
+ assertEquals(Types.BOOLEAN, meta.getColumnType(2));
+ assertEquals(Types.DOUBLE, meta.getColumnType(3));
+ assertEquals(Types.VARCHAR, meta.getColumnType(4));
+ assertEquals(Types.VARCHAR, meta.getColumnType(5));
+ assertEquals(Types.VARCHAR, meta.getColumnType(6));
+ assertEquals(Types.VARCHAR, meta.getColumnType(7));
+ assertEquals(Types.VARCHAR, meta.getColumnType(8));
+ assertEquals(Types.TINYINT, meta.getColumnType(9));
+ assertEquals(Types.SMALLINT, meta.getColumnType(10));
+ assertEquals(Types.FLOAT, meta.getColumnType(11));
+ assertEquals(Types.BIGINT, meta.getColumnType(12));
+ assertEquals(Types.INTEGER, meta.getColumnType(13));
+ assertEquals(Types.VARCHAR, meta.getColumnType(14));
+
+ assertEquals("int", meta.getColumnTypeName(1));
+ assertEquals("boolean", meta.getColumnTypeName(2));
+ assertEquals("double", meta.getColumnTypeName(3));
+ assertEquals("string", meta.getColumnTypeName(4));
+ assertEquals("string", meta.getColumnTypeName(5));
+ assertEquals("string", meta.getColumnTypeName(6));
+ assertEquals("string", meta.getColumnTypeName(7));
+ assertEquals("string", meta.getColumnTypeName(8));
+ assertEquals("tinyint", meta.getColumnTypeName(9));
+ assertEquals("smallint", meta.getColumnTypeName(10));
+ assertEquals("float", meta.getColumnTypeName(11));
+ assertEquals("bigint", meta.getColumnTypeName(12));
+ assertEquals("int", meta.getColumnTypeName(13));
+ assertEquals("string", meta.getColumnTypeName(14));
+
+ assertEquals(16, meta.getColumnDisplaySize(1));
+ assertEquals(8, meta.getColumnDisplaySize(2));
+ assertEquals(16, meta.getColumnDisplaySize(3));
+ assertEquals(32, meta.getColumnDisplaySize(4));
+ assertEquals(32, meta.getColumnDisplaySize(5));
+ assertEquals(32, meta.getColumnDisplaySize(6));
+ assertEquals(32, meta.getColumnDisplaySize(7));
+ assertEquals(32, meta.getColumnDisplaySize(8));
+ assertEquals(2, meta.getColumnDisplaySize(9));
+ assertEquals(32, meta.getColumnDisplaySize(10));
+ assertEquals(32, meta.getColumnDisplaySize(11));
+ assertEquals(32, meta.getColumnDisplaySize(12));
+ assertEquals(16, meta.getColumnDisplaySize(13));
+ assertEquals(32, meta.getColumnDisplaySize(14));
- for (int i = 1; i <= 6; i++) {
+ for (int i = 1; i <= meta.getColumnCount(); i++) {
assertFalse(meta.isAutoIncrement(i));
assertFalse(meta.isCurrency(i));
assertEquals(ResultSetMetaData.columnNullable, meta.isNullable(i));
- int expectedPrecision = i >= 5 ? -1 : 0;
- int expectedScale = i >= 5 ? -1 : 0;
- assertEquals("Unexpected precision", expectedPrecision, meta.getPrecision(i));
- assertEquals("Unexpected scale", expectedScale, meta.getScale(i));
+ int expectedPrecision = i == 3 ? -1 : 0;
+ int expectedScale = i == 3 ? -1 : 0;
+ assertEquals(expectedPrecision, meta.getPrecision(i));
+ assertEquals(expectedScale, meta.getScale(i));
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchTask.java Mon Sep 27 19:17:08 2010
@@ -23,6 +23,7 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Properties;
+import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.DriverContext;
@@ -31,10 +32,12 @@ import org.apache.hadoop.hive.ql.plan.Fe
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.StageType;
import org.apache.hadoop.hive.serde.Constants;
-import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
+import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
/**
@@ -45,7 +48,7 @@ public class FetchTask extends Task<Fetc
private int maxRows = 100;
private FetchOperator ftOp;
- private LazySimpleSerDe mSerde;
+ private SerDe mSerde;
private int totalRows;
public FetchTask() {
@@ -60,13 +63,21 @@ public class FetchTask extends Task<Fetc
// Create a file system handle
JobConf job = new JobConf(conf, ExecDriver.class);
- mSerde = new LazySimpleSerDe();
- Properties mSerdeProp = new Properties();
- mSerdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
- mSerdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, (work)
- .getSerializationNullFormat());
- mSerde.initialize(job, mSerdeProp);
- mSerde.setUseJSONSerialize(true);
+ String serdeName = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEFETCHOUTPUTSERDE);
+ Class<? extends SerDe> serdeClass = Class.forName(serdeName, true,
+ JavaUtils.getClassLoader()).asSubclass(SerDe.class);
+ // cast only needed for Hadoop 0.17 compatibility
+ mSerde = (SerDe) ReflectionUtils.newInstance(serdeClass, null);
+
+ Properties serdeProp = new Properties();
+
+ // this is the default serialization format
+ if (mSerde instanceof DelimitedJSONSerDe) {
+ serdeProp.put(Constants.SERIALIZATION_FORMAT, "" + Utilities.tabCode);
+ serdeProp.put(Constants.SERIALIZATION_NULL_FORMAT, work.getSerializationNullFormat());
+ }
+
+ mSerde.initialize(job, serdeProp);
ftOp = new FetchOperator(work, job);
} catch (Exception e) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Mon Sep 27 19:17:08 2010
@@ -1418,7 +1418,7 @@ public class SemanticAnalyzer extends Ba
}
TableDesc tblDesc = PlanUtils.getTableDesc(serdeClass, Integer
- .toString(Utilities.tabCode), cols, colTypes, defaultCols, true);
+ .toString(Utilities.tabCode), cols, colTypes, defaultCols);
// copy all the properties
if (child.getChildCount() == 2) {
ASTNode prop = (ASTNode) ((ASTNode) child.getChild(1)).getChild(0);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java Mon Sep 27 19:17:08 2010
@@ -47,6 +47,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.DelimitedJSONSerDe;
import org.apache.hadoop.hive.serde2.Deserializer;
import org.apache.hadoop.hive.serde2.MetadataTypedColumnsetSerDe;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
@@ -151,16 +152,16 @@ public final class PlanUtils {
public static TableDesc getTableDesc(
Class<? extends Deserializer> serdeClass, String separatorCode,
String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine,
- boolean useJSONForLazy) {
+ boolean useDelimitedJSON) {
return getTableDesc(serdeClass, separatorCode, columns, columnTypes,
- lastColumnTakesRestOfTheLine, useJSONForLazy, "TextFile");
+ lastColumnTakesRestOfTheLine, useDelimitedJSON, "TextFile");
}
public static TableDesc getTableDesc(
Class<? extends Deserializer> serdeClass, String separatorCode,
String columns, String columnTypes, boolean lastColumnTakesRestOfTheLine,
- boolean useJSONForLazy, String fileFormat) {
+ boolean useDelimitedJSON, String fileFormat) {
Properties properties = Utilities.makeProperties(
Constants.SERIALIZATION_FORMAT, separatorCode, Constants.LIST_COLUMNS,
@@ -184,8 +185,8 @@ public final class PlanUtils {
// user sees the results as json for custom scripts and has no way for
// specifying that.
// Right now, it is hard-coded in the code
- if (useJSONForLazy) {
- properties.setProperty(Constants.SERIALIZATION_USE_JSON_OBJECTS, "true");
+ if (useDelimitedJSON) {
+ serdeClass = DelimitedJSONSerDe.class;
}
Class inputFormat, outputFormat;
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out Mon Sep 27 19:17:08 2010
@@ -86,7 +86,6 @@ STAGE PLANS:
field.delim 9
serialization.format 9
serialization.last.column.takes.rest true
- serialization.use.json.object true
File Output Operator
compressed: false
GlobalTableId: 1
Modified: hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out_0.17
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out_0.17?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out_0.17 (original)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/binary_output_format.q.out_0.17 Mon Sep 27 19:17:08 2010
@@ -86,13 +86,12 @@ STAGE PLANS:
field.delim 9
serialization.format 9
serialization.last.column.takes.rest true
- serialization.use.json.object true
File Output Operator
compressed: false
GlobalTableId: 1
- directory: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10002
+ directory: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10002
NumFilesPerFileSink: 1
- Stats Publishing Key Prefix: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10000/
+ Stats Publishing Key Prefix: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10000/
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
@@ -102,13 +101,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588394
+ transient_lastDdlTime 1285609050
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
TotalFiles: 1
@@ -116,9 +115,9 @@ STAGE PLANS:
MultiFileSpray: false
Needs Tagging: false
Path -> Alias:
- pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/src [src]
+ pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/src [src]
Path -> Partition:
- pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/src
+ pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/src
Partition
base file name: src
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -129,12 +128,12 @@ STAGE PLANS:
columns.types string:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/src
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/src
name src
serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588338
+ transient_lastDdlTime 1285609044
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -145,12 +144,12 @@ STAGE PLANS:
columns.types string:string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/src
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/src
name src
serialization.ddl struct src { string key, string value}
serialization.format 1
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588338
+ transient_lastDdlTime 1285609044
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: src
name: src
@@ -162,14 +161,14 @@ STAGE PLANS:
Move Operator
files:
hdfs directory: true
- source: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10002
- destination: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10000
+ source: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10002
+ destination: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10000
Stage: Stage-0
Move Operator
tables:
replace: true
- source: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10000
+ source: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10000
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
@@ -179,25 +178,25 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588394
+ transient_lastDdlTime 1285609050
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
- tmp directory: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10001
+ tmp directory: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10001
Stage: Stage-2
Stats-Aggr Operator
- Stats Aggregation Key Prefix: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10000/
+ Stats Aggregation Key Prefix: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10000/
Stage: Stage-3
Map Reduce
Alias -> Map Operator Tree:
- pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10002
+ pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10002
Reduce Output Operator
sort order:
Map-reduce partition columns:
@@ -209,9 +208,9 @@ STAGE PLANS:
type: string
Needs Tagging: false
Path -> Alias:
- pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10002 [pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10002]
+ pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10002 [pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10002]
Path -> Partition:
- pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10002
+ pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10002
Partition
base file name: -ext-10002
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -222,13 +221,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588394
+ transient_lastDdlTime 1285609050
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -239,13 +238,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588394
+ transient_lastDdlTime 1285609050
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
name: dest1
@@ -254,7 +253,7 @@ STAGE PLANS:
File Output Operator
compressed: false
GlobalTableId: 0
- directory: pfile:/data/users/nzhang/work/784/apache-hive/build/ql/scratchdir/hive_2010-09-15_15-06-34_157_3907187975100374432/-ext-10000
+ directory: pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/scratchdir/hive_2010-09-27_10-37-30_669_8810120463896511359/-ext-10000
NumFilesPerFileSink: 1
table:
input format: org.apache.hadoop.mapred.TextInputFormat
@@ -265,13 +264,13 @@ STAGE PLANS:
columns.types string
file.inputformat org.apache.hadoop.mapred.TextInputFormat
file.outputformat org.apache.hadoop.hive.ql.io.HiveBinaryOutputFormat
- location pfile:/data/users/nzhang/work/784/apache-hive/build/ql/test/data/warehouse/dest1
+ location pfile:/data/users/nzhang/reviews/1/apache-hive/build/ql/test/data/warehouse/dest1
name dest1
serialization.ddl struct dest1 { string mydata}
serialization.format 1
serialization.last.column.takes.rest true
serialization.lib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- transient_lastDdlTime 1284588394
+ transient_lastDdlTime 1285609050
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: dest1
TotalFiles: 1
@@ -312,12 +311,12 @@ PREHOOK: query: -- Test the result
SELECT * FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
-PREHOOK: Output: file:/tmp/nzhang/hive_2010-09-15_15-06-40_207_5146800440661796390/-mr-10000
+PREHOOK: Output: file:/tmp/nzhang/hive_2010-09-27_10-37-56_310_4864136950389816911/-mr-10000
POSTHOOK: query: -- Test the result
SELECT * FROM dest1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
-POSTHOOK: Output: file:/tmp/nzhang/hive_2010-09-15_15-06-40_207_5146800440661796390/-mr-10000
+POSTHOOK: Output: file:/tmp/nzhang/hive_2010-09-27_10-37-56_310_4864136950389816911/-mr-10000
POSTHOOK: Lineage: dest1.mydata SCRIPT [(src)src.FieldSchema(name:key, type:string, comment:default), (src)src.FieldSchema(name:value, type:string, comment:default), ]
238 val_238
86 val_86
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/input20.q.xml Mon Sep 27 19:17:08 2010
@@ -375,7 +375,7 @@
<void property="scriptInputInfo">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.DelimitedJSONSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -394,10 +394,6 @@
<string>_col0,_col1</string>
</void>
<void method="put">
- <string>serialization.use.json.object</string>
- <string>true</string>
- </void>
- <void method="put">
<string>serialization.format</string>
<string>9</string>
</void>
@@ -1042,7 +1038,7 @@
<void property="scriptInputInfo">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.DelimitedJSONSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -1061,10 +1057,6 @@
<string>_col0,_col1</string>
</void>
<void method="put">
- <string>serialization.use.json.object</string>
- <string>true</string>
- </void>
- <void method="put">
<string>serialization.format</string>
<string>9</string>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/input4.q.xml Mon Sep 27 19:17:08 2010
@@ -509,7 +509,7 @@
<void property="scriptInputInfo">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.DelimitedJSONSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -528,10 +528,6 @@
<string>_col0,_col1</string>
</void>
<void method="put">
- <string>serialization.use.json.object</string>
- <string>true</string>
- </void>
- <void method="put">
<string>serialization.format</string>
<string>9</string>
</void>
Modified: hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml (original)
+++ hadoop/hive/trunk/ql/src/test/results/compiler/plan/input5.q.xml Mon Sep 27 19:17:08 2010
@@ -513,7 +513,7 @@
<void property="scriptInputInfo">
<object class="org.apache.hadoop.hive.ql.plan.TableDesc">
<void property="deserializerClass">
- <class>org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe</class>
+ <class>org.apache.hadoop.hive.serde2.DelimitedJSONSerDe</class>
</void>
<void property="inputFileFormatClass">
<class>org.apache.hadoop.mapred.TextInputFormat</class>
@@ -532,10 +532,6 @@
<string>_col0,_col1</string>
</void>
<void method="put">
- <string>serialization.use.json.object</string>
- <string>true</string>
- </void>
- <void method="put">
<string>serialization.format</string>
<string>9</string>
</void>
Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java?rev=1001881&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/DelimitedJSONSerDe.java Mon Sep 27 19:17:08 2010
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * DelimitedJSONSerDe.
+ *
+ * This serde can only serialize, because it is just intended for use by the FetchTask class and the
+ * TRANSFORM function.
+ */
+public class DelimitedJSONSerDe extends LazySimpleSerDe {
+
+ public static final Log LOG = LogFactory.getLog(DelimitedJSONSerDe.class.getName());
+
+ public DelimitedJSONSerDe() throws SerDeException {
+ }
+
+ /**
+ * Not implemented.
+ */
+ @Override
+ public Object deserialize(Writable field) throws SerDeException {
+ LOG.error("DelimitedJSONSerDe cannot deserialize.");
+ throw new SerDeException("DelimitedJSONSerDe cannot deserialize.");
+ }
+
+ @Override
+ protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector objInspector,
+ SerDeParameters serdeParams) throws SerDeException {
+ if (!objInspector.getCategory().equals(Category.PRIMITIVE)) {
+ try {
+ serialize(out, SerDeUtils.getJSONString(obj, objInspector),
+ PrimitiveObjectInspectorFactory.javaStringObjectInspector, serdeParams.getSeparators(),
+ 1, serdeParams.getNullSequence(), serdeParams.isEscaped(), serdeParams.getEscapeChar(),
+ serdeParams.getNeedsEscape());
+
+ } catch (IOException e) {
+ throw new SerDeException(e);
+ }
+
+ } else {
+ super.serializeField(out, obj, objInspector, serdeParams);
+ }
+ }
+}
Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=1001881&r1=1001880&r2=1001881&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Mon Sep 27 19:17:08 2010
@@ -32,7 +32,6 @@ import org.apache.hadoop.hive.serde.Cons
import org.apache.hadoop.hive.serde2.ByteStream;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -40,7 +39,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -52,10 +50,10 @@ import org.apache.hadoop.io.Writable;
/**
* LazySimpleSerDe can be used to read the same data format as
* MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol.
- *
+ *
* However, LazySimpleSerDe creates Objects in a lazy way, to provide better
* performance.
- *
+ *
* Also LazySimpleSerDe outputs typed columns instead of treating all columns as
* String like MetadataTypedColumnsetSerDe.
*/
@@ -67,7 +65,6 @@ public class LazySimpleSerDe implements
public static final byte[] DefaultSeparators = {(byte) 1, (byte) 2, (byte) 3};
private ObjectInspector cachedObjectInspector;
- private boolean useJSONSerialize; // use json to serialize
@Override
public String toString() {
@@ -86,7 +83,7 @@ public class LazySimpleSerDe implements
/**
* Return the byte value of the number string.
- *
+ *
* @param altValue
* The string containing a number.
* @param defaultVal
@@ -120,7 +117,6 @@ public class LazySimpleSerDe implements
boolean escaped;
byte escapeChar;
boolean[] needsEscape;
- boolean jsonSerialize;
public List<TypeInfo> getColumnTypes() {
return columnTypes;
@@ -161,22 +157,6 @@ public class LazySimpleSerDe implements
public boolean[] getNeedsEscape() {
return needsEscape;
}
-
- /**
- * @return the jsonSerialize
- */
- public boolean isJsonSerialize() {
- return jsonSerialize;
- }
-
- /**
- * @param jsonSerialize
- * the jsonSerialize to set
- */
- public void setJsonSerialize(boolean jsonSerialize) {
- this.jsonSerialize = jsonSerialize;
- }
-
}
SerDeParameters serdeParams = null;
@@ -186,7 +166,7 @@ public class LazySimpleSerDe implements
* char or byte code (only supports byte-value up to 127) columns:
* ","-separated column names columns.types: ",", ":", or ";"-separated column
* types
- *
+ *
* @see SerDe#initialize(Configuration, Properties)
*/
public void initialize(Configuration job, Properties tbl)
@@ -202,14 +182,10 @@ public class LazySimpleSerDe implements
.isLastColumnTakesRest(), serdeParams.isEscaped(), serdeParams
.getEscapeChar());
- if (serdeParams.isJsonSerialize()) {
- setUseJSONSerialize(true);
- }
-
cachedLazyStruct = (LazyStruct) LazyFactory
.createLazyObject(cachedObjectInspector);
- LOG.debug("LazySimpleSerDe initialized with: columnNames="
+ LOG.debug(getClass().getName() + " initialized with: columnNames="
+ serdeParams.columnNames + " columnTypes=" + serdeParams.columnTypes
+ " separator=" + Arrays.asList(serdeParams.separators)
+ " nullstring=" + serdeParams.nullString + " lastColumnTakesRest="
@@ -242,11 +218,6 @@ public class LazySimpleSerDe implements
serdeParams.lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString
.equalsIgnoreCase("true"));
- String useJsonSerialize = tbl
- .getProperty(Constants.SERIALIZATION_USE_JSON_OBJECTS);
- serdeParams.jsonSerialize = (useJsonSerialize != null && useJsonSerialize
- .equalsIgnoreCase("true"));
-
// Read the configuration parameters
String columnNameProperty = tbl.getProperty(Constants.LIST_COLUMNS);
// NOTE: if "columns.types" is missing, all columns will be of String type
@@ -313,7 +284,7 @@ public class LazySimpleSerDe implements
/**
* Deserialize a row from the Writable to a LazyObject.
- *
+ *
* @param field
* the Writable that contains the data
* @return The deserialized row Object.
@@ -348,7 +319,7 @@ public class LazySimpleSerDe implements
/**
* Returns the Writable Class after serialization.
- *
+ *
* @see SerDe#getSerializedClass()
*/
public Class<? extends Writable> getSerializedClass() {
@@ -360,7 +331,7 @@ public class LazySimpleSerDe implements
/**
* Serialize a row of data.
- *
+ *
* @param obj
* The row object
* @param objInspector
@@ -389,48 +360,27 @@ public class LazySimpleSerDe implements
serializeStream.reset();
- try {
- // Serialize each field
- for (int i = 0; i < fields.size(); i++) {
- // Append the separator if needed.
- if (i > 0) {
- serializeStream.write(serdeParams.separators[0]);
- }
- // Get the field objectInspector and the field object.
- ObjectInspector foi = fields.get(i).getFieldObjectInspector();
- Object f = (list == null ? null : list.get(i));
-
- if (declaredFields != null && i >= declaredFields.size()) {
- throw new SerDeException("Error: expecting " + declaredFields.size()
- + " but asking for field " + i + "\n" + "data=" + obj + "\n"
- + "tableType=" + serdeParams.rowTypeInfo.toString() + "\n"
- + "dataType="
- + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector));
- }
-
- // If the field that is passed in is NOT a primitive, and either the
- // field is not declared (no schema was given at initialization), or
- // the field is declared as a primitive in initialization, serialize
- // the data to JSON string. Otherwise serialize the data in the
- // delimited way.
- if (!foi.getCategory().equals(Category.PRIMITIVE)
- && (declaredFields == null
- || declaredFields.get(i).getFieldObjectInspector()
- .getCategory().equals(Category.PRIMITIVE) || useJSONSerialize)) {
- serialize(serializeStream, SerDeUtils.getJSONString(f, foi),
- PrimitiveObjectInspectorFactory.javaStringObjectInspector,
- serdeParams.separators, 1, serdeParams.nullSequence,
- serdeParams.escaped, serdeParams.escapeChar,
- serdeParams.needsEscape);
- } else {
- serialize(serializeStream, f, foi, serdeParams.separators, 1,
- serdeParams.nullSequence, serdeParams.escaped,
- serdeParams.escapeChar, serdeParams.needsEscape);
- }
+ // Serialize each field
+ for (int i = 0; i < fields.size(); i++) {
+ // Append the separator if needed.
+ if (i > 0) {
+ serializeStream.write(serdeParams.separators[0]);
}
- } catch (IOException e) {
- throw new SerDeException(e);
+ // Get the field objectInspector and the field object.
+ ObjectInspector foi = fields.get(i).getFieldObjectInspector();
+ Object f = (list == null ? null : list.get(i));
+
+ if (declaredFields != null && i >= declaredFields.size()) {
+ throw new SerDeException("Error: expecting " + declaredFields.size()
+ + " but asking for field " + i + "\n" + "data=" + obj + "\n"
+ + "tableType=" + serdeParams.rowTypeInfo.toString() + "\n"
+ + "dataType="
+ + TypeInfoUtils.getTypeInfoFromObjectInspector(objInspector));
+ }
+
+ serializeField(serializeStream, f, foi, serdeParams);
}
+
// TODO: The copy of data is unnecessary, but there is no work-around
// since we cannot directly set the private byte[] field inside Text.
serializeCache
@@ -438,9 +388,19 @@ public class LazySimpleSerDe implements
return serializeCache;
}
+ protected void serializeField(ByteStream.Output out, Object obj, ObjectInspector objInspector,
+ SerDeParameters serdeParams) throws SerDeException {
+ try {
+ serialize(out, obj, objInspector, serdeParams.separators, 1, serdeParams.nullSequence,
+ serdeParams.escaped, serdeParams.escapeChar, serdeParams.needsEscape);
+ } catch (IOException e) {
+ throw new SerDeException(e);
+ }
+ }
+
/**
* Serialize the row into the StringBuilder.
- *
+ *
* @param out
* The StringBuilder to store the serialized data.
* @param obj
@@ -548,19 +508,4 @@ public class LazySimpleSerDe implements
throw new RuntimeException("Unknown category type: "
+ objInspector.getCategory());
}
-
- /**
- * @return the useJSONSerialize
- */
- public boolean isUseJSONSerialize() {
- return useJSONSerialize;
- }
-
- /**
- * @param useJSONSerialize
- * the useJSONSerialize to set
- */
- public void setUseJSONSerialize(boolean useJSONSerialize) {
- this.useJSONSerialize = useJSONSerialize;
- }
}