svn commit: r1565738 - in /pig/trunk: CHANGES.txt src/org/apache/pig/builtin/JsonStorage.java test/org/apache/pig/test/TestJsonLoaderStorage.java

[da...@apache.org]

Author: daijy
Date: Fri Feb  7 17:36:18 2014
New Revision: 1565738

URL: http://svn.apache.org/r1565738
Log:
PIG-3627: Json storage : Doesn't work in cases , where other Store Functions (like PigStorage / AvroStorage) do work

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/JsonStorage.java
    pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1565738&r1=1565737&r2=1565738&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Feb  7 17:36:18 2014
@@ -87,6 +87,9 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-3627: Json storage : Doesn't work in cases , where other Store Functions (like PigStorage / AvroStorage)
+ do work (ssvinarchukhorton via daijy)
+
 PIG-3606: Pig script throws error when searching for hcatalog jars in latest hive (deepesh via daijy)
 
 PIG-3623: HBaseStorage: setting loadKey and noWAL to false doesn't have any affect (nezihyigitbasi via rohini)

Modified: pig/trunk/src/org/apache/pig/builtin/JsonStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/JsonStorage.java?rev=1565738&r1=1565737&r2=1565738&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/JsonStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/JsonStorage.java Fri Feb  7 17:36:18 2014
@@ -113,7 +113,7 @@ public class JsonStorage extends StoreFu
         UDFContext udfc = UDFContext.getUDFContext();
         Properties p =
             udfc.getUDFProperties(this.getClass(), new String[]{udfcSignature});
-        p.setProperty(SCHEMA_SIGNATURE, s.toString());
+        p.setProperty(SCHEMA_SIGNATURE, fixSchema(s).toString());
     }
 
 
@@ -310,4 +310,12 @@ public class JsonStorage extends StoreFu
         metadataWriter.storeSchema(schema, location, job);
     }
 
+    public ResourceSchema fixSchema(ResourceSchema s){
+      for (ResourceFieldSchema filed : s.getFields()) {
+        if(filed.getType() == DataType.NULL)
+          filed.setType(DataType.BYTEARRAY);
+      }
+      return s;
+    }
+
 }

Modified: pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java?rev=1565738&r1=1565737&r2=1565738&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestJsonLoaderStorage.java Fri Feb  7 17:36:18 2014
@@ -113,6 +113,9 @@ public class TestJsonLoaderStorage {
     "\"m\":null" +
     "}";
 
+  private static final String jsonOutput =
+    "{\"f1\":\"18\",\"count\":3}";
+
   private Iterator<Tuple> loadJson(String input) throws IOException {
     File tempFile = File.createTempFile("json", null);
     tempFile.deleteOnExit();
@@ -321,4 +324,53 @@ public class TestJsonLoaderStorage {
 
     br.close();
   }
+
+  @Test
+  public void testSimpleMapSideStreaming() throws Exception {
+    PigServer pigServer = new PigServer(ExecType.LOCAL);
+    File input = Util.createInputFile("tmp", "", new String [] {"1,2,3;4,5,6,7,8",
+        "1,2,3;4,5,6,7,9",
+        "1,2,3;4,5,6,7,18"});
+    File tempJsonFile = File.createTempFile("json", "");
+    tempJsonFile.delete();
+
+    // Pig query to run
+    pigServer.registerQuery("IP = load '"+  Util.generateURI(Util.encodeEscape(input.toString()), pigServer.getPigContext())
+        +"' using PigStorage (';') as (ID:chararray,DETAILS:chararray);");
+    pigServer.registerQuery(
+        "id_details = FOREACH IP GENERATE " +
+            "FLATTEN" +
+            "(STRSPLIT" +
+            "(ID,',',3)) AS (drop, code, transaction) ," +
+            "FLATTEN" +
+            "(STRSPLIT" +
+            "(DETAILS,',',5)) AS (lname, fname, date, price, product);");
+    pigServer.registerQuery(
+        "transactions = FOREACH id_details GENERATE $0 .. ;");
+    pigServer.registerQuery(
+        "transactionsG = group transactions by code;");
+    pigServer.registerQuery(
+        "uniqcnt  = foreach transactionsG {"+
+            "sym = transactions.product ;"+
+            "dsym =  distinct sym ;"+
+            "generate flatten(dsym.product) as f1, COUNT(dsym) as count ;" +
+            "};");
+    pigServer.store("uniqcnt", tempJsonFile.getAbsolutePath(), "JsonStorage");
+
+    BufferedReader br = new BufferedReader(new FileReader(tempJsonFile.getAbsolutePath()+ "/part-r-00000"));
+    String data = br.readLine();
+
+    assertEquals(jsonOutput, data);
+
+    String line = data;
+    int count = 0;
+    while (line != null) {
+      line = br.readLine();
+      count++;
+    }
+    assertEquals(3, count);
+
+    br.close();
+    tempJsonFile.deleteOnExit();
+  }
 }