You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/10/15 06:50:48 UTC
svn commit: r1631939 - in /pig/branches/branch-0.14: CHANGES.txt
src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
test/org/apache/pig/test/TestHBaseStorage.java
Author: daijy
Date: Wed Oct 15 04:50:47 2014
New Revision: 1631939
URL: http://svn.apache.org/r1631939
Log:
PIG-4151: Pig Cannot Write Empty Maps to HBase
Modified:
pig/branches/branch-0.14/CHANGES.txt
pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
pig/branches/branch-0.14/test/org/apache/pig/test/TestHBaseStorage.java
Modified: pig/branches/branch-0.14/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/CHANGES.txt?rev=1631939&r1=1631938&r2=1631939&view=diff
==============================================================================
--- pig/branches/branch-0.14/CHANGES.txt (original)
+++ pig/branches/branch-0.14/CHANGES.txt Wed Oct 15 04:50:47 2014
@@ -84,6 +84,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4151: Pig Cannot Write Empty Maps to HBase (daijy)
+
PIG-4181: Cannot launch tez e2e test on Windows (daijy)
PIG-2834: MultiStorage requires unused constructor argument (daijy)
Modified: pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java?rev=1631939&r1=1631938&r2=1631939&view=diff
==============================================================================
--- pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java (original)
+++ pig/branches/branch-0.14/src/org/apache/pig/backend/hadoop/hbase/HBaseStorage.java Wed Oct 15 04:50:47 2014
@@ -940,21 +940,25 @@ public class HBaseStorage extends LoadFu
DataType.findType(t.get(i)) : fieldSchemas[i].getType()));
} else {
Map<String, Object> cfMap = (Map<String, Object>) t.get(i);
- for (String colName : cfMap.keySet()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("putNext - colName=" + colName +
- ", class: " + colName.getClass());
+ if (cfMap!=null) {
+ for (String colName : cfMap.keySet()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("putNext - colName=" + colName +
+ ", class: " + colName.getClass());
+ }
+ // TODO deal with the fact that maps can have types now. Currently we detect types at
+ // runtime in the case of storing to a cf, which is suboptimal.
+ put.add(columnInfo.getColumnFamily(), Bytes.toBytes(colName.toString()), ts,
+ objToBytes(cfMap.get(colName), DataType.findType(cfMap.get(colName))));
}
- // TODO deal with the fact that maps can have types now. Currently we detect types at
- // runtime in the case of storing to a cf, which is suboptimal.
- put.add(columnInfo.getColumnFamily(), Bytes.toBytes(colName.toString()), ts,
- objToBytes(cfMap.get(colName), DataType.findType(cfMap.get(colName))));
}
}
}
try {
- writer.write(null, put);
+ if (!put.isEmpty()) {
+ writer.write(null, put);
+ }
} catch (InterruptedException e) {
throw new IOException(e);
}
Modified: pig/branches/branch-0.14/test/org/apache/pig/test/TestHBaseStorage.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.14/test/org/apache/pig/test/TestHBaseStorage.java?rev=1631939&r1=1631938&r2=1631939&view=diff
==============================================================================
--- pig/branches/branch-0.14/test/org/apache/pig/test/TestHBaseStorage.java (original)
+++ pig/branches/branch-0.14/test/org/apache/pig/test/TestHBaseStorage.java Wed Oct 15 04:50:47 2014
@@ -16,6 +16,7 @@
*/
package org.apache.pig.test;
+import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
@@ -1101,6 +1102,43 @@ public class TestHBaseStorage {
Assert.assertEquals(index, TEST_ROW_COUNT);
}
+ @Test
+ // See PIG-4151
+ public void testStoreEmptyMap() throws IOException {
+ String tableName = "emptyMapTest";
+ HTable table;
+ try {
+ deleteAllRows(tableName);
+ } catch (Exception e) {
+ // It's ok, table might not exist.
+ }
+ byte[][] cfs = new byte[2][];
+ cfs[0] = Bytes.toBytes("info");
+ cfs[1] = Bytes.toBytes("friends");
+ try {
+ table = util.createTable(Bytes.toBytesBinary(tableName),
+ cfs);
+ } catch (Exception e) {
+ table = new HTable(conf, Bytes.toBytesBinary(tableName));
+ }
+
+ File inputFile = Util.createInputFile("test", "tmp", new String[] {"row1;Homer;Morrison;[1#Silvia,2#Stacy]",
+ "row2;Sheila;Fletcher;[1#Becky,2#Salvador,3#Lois]",
+ "row4;Andre;Morton;[1#Nancy]",
+ "row3;Sonja;Webb;[]"
+ });
+ pig.registerQuery("source = LOAD '" + Util.generateURI(Util.encodeEscape(inputFile.toString()), pig.getPigContext())
+ + "' USING PigStorage(';')"
+ + " AS (row:chararray, first_name:chararray, last_name:chararray, friends:map[]);");
+ pig.registerQuery("STORE source INTO 'hbase://" + tableName + "' USING" +
+ " org.apache.pig.backend.hadoop.hbase.HBaseStorage('info:fname info:lname friends:*');");
+ Get get = new Get(Bytes.toBytes("row3"));
+ Result r = table.get(get);
+ Assert.assertEquals(new String(r.getValue(cfs[0], Bytes.toBytes("fname"))), "Sonja");
+ Assert.assertEquals(new String(r.getValue(cfs[0], Bytes.toBytes("lname"))), "Webb");
+ Assert.assertTrue(r.getFamilyMap(cfs[1]).isEmpty());
+ }
+
private void scanTable1(PigServer pig, DataFormat dataFormat) throws IOException {
scanTable1(pig, dataFormat, "");
}