svn commit: r680885 - in /incubator/pig/branches/types: src/org/apache/pig/builtin/PigStorage.java test/org/apache/pig/test/TestBuiltin.java

[ga...@apache.org]

Author: gates
Date: Tue Jul 29 16:44:39 2008
New Revision: 680885

URL: http://svn.apache.org/viewvc?rev=680885&view=rev
Log:
PIG-340  Fixed issue of delimiter not working with tab or unicode characters.


Modified:
    incubator/pig/branches/types/src/org/apache/pig/builtin/PigStorage.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java

Modified: incubator/pig/branches/types/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/builtin/PigStorage.java?rev=680885&r1=680884&r2=680885&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/builtin/PigStorage.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/builtin/PigStorage.java Tue Jul 29 16:44:39 2008
@@ -24,6 +24,9 @@
 import java.util.ArrayList;
 import java.util.Map;
 
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.Log;
+
 import org.apache.pig.LoadFunc;
 import org.apache.pig.StoreFunc;
 import org.apache.pig.ReversibleLoadStoreFunc;
@@ -43,6 +46,7 @@
 public class PigStorage extends Utf8StorageConverter
         implements ReversibleLoadStoreFunc {
     protected BufferedPositionedInputStream in = null;
+    protected final Log mLog = LogFactory.getLog(getClass());
         
     long                end            = Long.MAX_VALUE;
     private byte recordDel = '\n';
@@ -61,9 +65,28 @@
      *            ("\t" is the default.)
      */
     public PigStorage(String delimiter) {
-        this.fieldDel = (byte)delimiter.charAt(0);
-        //mBuf = new ByteArrayOutputStream(4096);
-        //mProtoTuple = new ArrayList<Object>();
+        if (delimiter.length() == 1) {
+            this.fieldDel = (byte)delimiter.charAt(0);
+        } else if (delimiter.length() > 1 && delimiter.charAt(0) == '\\') {
+            switch (delimiter.charAt(1)) {
+            case 't':
+                this.fieldDel = (byte)'\t';
+                break;
+
+            case 'x':
+            case 'u':
+                this.fieldDel =
+                    Integer.valueOf(delimiter.substring(2)).byteValue();
+                break;
+
+            default:
+                mLog.error("Unknown delimiter " + delimiter);
+                throw new RuntimeException("Unknown delimiter " + delimiter);
+            }
+        } else {
+            mLog.error("PigStorage delimeter must be single character");
+            throw new RuntimeException("PigStorage delimeter must be single character");
+        }
     }
 
     public Tuple getNext() throws IOException {

Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java?rev=680885&r1=680884&r2=680885&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestBuiltin.java Tue Jul 29 16:44:39 2008
@@ -44,7 +44,10 @@
 
 public class TestBuiltin extends TestCase {
     
-    private String initString = "local";
+    private String initString = "mapreduce";
+    //private String initString = "local";
+    MiniCluster cluster = MiniCluster.buildCluster();
+
     
     // some inputs
     private static Integer[] IntInput = { 3, 1, 2, 4, 5, 7, null, 6, 8, 9, 10 };