You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/09/19 22:39:32 UTC
svn commit: r697229 [2/2] - in /incubator/pig/branches/types:
src/org/apache/pig/ src/org/apache/pig/backend/hadoop/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relat...
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java Fri Sep 19 13:39:31 2008
@@ -3,97 +3,25 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.io.BooleanWritable;
/**
*
*/
-public class NullableBooleanWritable extends BooleanWritable {
+public class NullableBooleanWritable extends PigNullableWritable {
- private boolean isNull = false;
-
- public static byte NULL = 0x00;
- public static byte NOTNULL = 0x01;
-
-
-
- /**
- *
- */
public NullableBooleanWritable() {
- super();
+ mValue = new BooleanWritable();
}
/**
* @param value
*/
public NullableBooleanWritable(boolean value) {
- super(value);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableBooleanWritable)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableBooleanWritable)o).isNull())
- return 1;
- else
- return super.compareTo(o);
+ mValue = new BooleanWritable(value);
}
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull()) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
+ public Object getValueAsPigType() {
+ return isNull() ? null : ((BooleanWritable)mValue).get();
}
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
- }
-
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
- }
-
-
-
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java Fri Sep 19 13:39:31 2008
@@ -17,96 +17,28 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.io.BytesWritable;
+import org.apache.pig.data.DataByteArray;
+
/**
*
*/
-public class NullableBytesWritable extends BytesWritable {
+public class NullableBytesWritable extends PigNullableWritable {
- private boolean isNull = false;
-
- public static byte NULL = 0x00;
- public static byte NOTNULL = 0x01;
-
-
- /**
- *
- */
public NullableBytesWritable() {
- super();
+ mValue = new BytesWritable();
}
/**
* @param bytes
*/
public NullableBytesWritable(byte[] bytes) {
- super(bytes);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableBytesWritable)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableBytesWritable)o).isNull())
- return 1;
- else
- return super.compareTo(o);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
+ mValue = new BytesWritable(bytes);
}
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull == true) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
- }
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
+ public Object getValueAsPigType() {
+ BytesWritable bw = (BytesWritable)mValue;
+ return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize());
}
-
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
- }
-
-
-
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java Fri Sep 19 13:39:31 2008
@@ -17,95 +17,25 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.pig.backend.hadoop.DoubleWritable;
/**
*
*/
-public class NullableDoubleWritable extends DoubleWritable {
+public class NullableDoubleWritable extends PigNullableWritable {
- private boolean isNull = false;
-
- public static byte NULL = 0x00;
- public static byte NOTNULL = 0x01;
-
- /**
- *
- */
public NullableDoubleWritable() {
- super();
+ mValue = new DoubleWritable();
}
/**
* @param value
*/
public NullableDoubleWritable(double value) {
- super(value);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableDoubleWritable)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableDoubleWritable)o).isNull())
- return 1;
- else
- return super.compareTo(o);
+ mValue = new DoubleWritable(value);
}
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull()) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
+ public Object getValueAsPigType() {
+ return isNull() ? null : ((DoubleWritable)mValue).get();
}
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
- }
-
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
- }
-
-
-
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java Fri Sep 19 13:39:31 2008
@@ -17,95 +17,25 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.io.FloatWritable;
/**
*
*/
-public class NullableFloatWritable extends FloatWritable {
+public class NullableFloatWritable extends PigNullableWritable {
- private boolean isNull = false;
-
- public static byte NULL = 0x00;
- public static byte NOTNULL = 0x01;
-
- /**
- *
- */
public NullableFloatWritable() {
- super();
+ mValue = new FloatWritable();
}
/**
* @param value
*/
public NullableFloatWritable(float value) {
- super(value);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableFloatWritable)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableFloatWritable)o).isNull())
- return 1;
- else
- return super.compareTo(o);
+ mValue = new FloatWritable(value);
}
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull()) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
+ public Object getValueAsPigType() {
+ return isNull() ? null : ((FloatWritable)mValue).get();
}
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
- }
-
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
- }
-
-
-
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java Fri Sep 19 13:39:31 2008
@@ -17,104 +17,25 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.io.IntWritable;
/**
*
*/
-public class NullableIntWritable extends IntWritable {
+public class NullableIntWritable extends PigNullableWritable {
- private boolean isNull = false;
-
- /**
- * Marker for indicating whether the value this object holds
- * is a null
- */
- public static byte NULL = 0x00;
-
- /**
- * Marker for indicating whether the value this object holds
- * is not a null
- */
- public static byte NOTNULL = 0x01;
-
- /**
- *
- */
public NullableIntWritable() {
- super();
+ mValue = new IntWritable();
}
/**
* @param value
*/
public NullableIntWritable(int value) {
- super(value);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableIntWritable)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableIntWritable)o).isNull())
- return 1;
- else
- return super.compareTo(o);
+ mValue = new IntWritable(value);
}
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
+ public Object getValueAsPigType() {
+ return isNull() ? null : ((IntWritable)mValue).get();
}
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull()) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
- }
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
- }
-
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
- }
-
-
-
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java Fri Sep 19 13:39:31 2008
@@ -17,95 +17,25 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.io.LongWritable;
/**
*
*/
-public class NullableLongWritable extends LongWritable {
+public class NullableLongWritable extends PigNullableWritable {
- private boolean isNull = false;
-
- public static byte NULL = 0x00;
- public static byte NOTNULL = 0x01;
-
- /**
- *
- */
public NullableLongWritable() {
- super();
+ mValue = new LongWritable();
}
/**
* @param value
*/
public NullableLongWritable(long value) {
- super(value);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableLongWritable)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableLongWritable)o).isNull())
- return 1;
- else
- return super.compareTo(o);
+ mValue = new LongWritable(value);
}
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull()) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
+ public Object getValueAsPigType() {
+ return isNull() ? null : ((LongWritable)mValue).get();
}
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
- }
-
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
- }
-
-
-
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java Fri Sep 19 13:39:31 2008
@@ -17,109 +17,32 @@
*/
package org.apache.pig.impl.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.io.Text;
/**
*
*/
-public class NullableText extends Text {
+public class NullableText extends PigNullableWritable {
- private boolean isNull = false;
-
- public static byte NULL = 0x00;
- public static byte NOTNULL = 0x01;
-
- /**
- *
- */
public NullableText() {
- super();
+ mValue = new Text();
}
/**
* @param utf8
*/
public NullableText(byte[] utf8) {
- super(utf8);
+ mValue = new Text(utf8);
}
/**
* @param string
*/
public NullableText(String string) {
- super(string);
- }
-
- /**
- * @param utf8
- */
- public NullableText(Text utf8) {
- super(utf8);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object)
- */
- @Override
- public int compareTo(Object o) {
- // if both are null they are equal only here!
- if(isNull == true && ((NullableText)o).isNull())
- return 0;
- else if(isNull == true)
- return -1;
- else if (((NullableText)o).isNull())
- return 1;
- else
- return super.compareTo(o);
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
- */
- @Override
- public void readFields(DataInput in) throws IOException {
- byte nullMarker = in.readByte();
- if(nullMarker == NULL) {
- isNull = true;
- }
- else {
- isNull = false;
- super.readFields(in);
- }
-
- }
-
- /* (non-Javadoc)
- * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
- */
- @Override
- public void write(DataOutput out) throws IOException {
- if(isNull()) {
- out.writeByte(NULL);
- } else {
- out.writeByte(NOTNULL);
- super.write(out);
- }
- }
-
- /**
- * @return the isNull
- */
- public boolean isNull() {
- return isNull;
+ mValue = new Text(string);
}
- /**
- * @param isNull the isNull to set
- */
- public void setNull(boolean isNull) {
- this.isNull = isNull;
+ public Object getValueAsPigType() {
+ return isNull() ? null : ((Text)mValue).toString();
}
-
-
-
}
Added: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java?rev=697229&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java Fri Sep 19 13:39:31 2008
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.io;
+
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+
+/**
+ *
+ */
+public class NullableTuple extends PigNullableWritable {
+
+ private TupleFactory mFactory = null;
+
+ public NullableTuple() {
+ if (mFactory == null) {
+ mFactory = TupleFactory.getInstance();
+ }
+ mValue = mFactory.newTuple();
+ }
+
+ /**
+ * @param bytes
+ */
+ public NullableTuple(Tuple t) {
+ mValue = t;
+ }
+
+ public Object getValueAsPigType() {
+ return isNull() ? null : (Tuple)mValue;
+ }
+}
Added: incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java?rev=697229&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java Fri Sep 19 13:39:31 2008
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.io;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A base class for all types that pig uses to move data between map and
+ * reduce. It implements WritableComparable so that compareTo etc. can be
+ * called. It also wraps a WritableComparable 'value'. This is set by each
+ * different type to be an object of its specific type.
+ * It also provides a getIndex() and setIndex() calls that are used to get
+ * and set the index. These can be used by LocalRearrange, the partitioner,
+ * and Package to determine the index.
+ *
+ * Index and the null indicator are packed into one byte to save space.
+ */
+public abstract class PigNullableWritable implements WritableComparable {
+
+ private boolean mNull;
+
+ protected WritableComparable mValue;
+
+ private byte mIndex;
+
+ /**
+ * Compare two nullable objects. Step one is to check if either or both
+ * are null. If one is null and the other is not, then the one that is
+ * null is declared to be less. If both are null the indices are
+ * compared. If neither are null the indices are again compared. If
+ * these are equal, finally the values are compared.
+ *
+ * These comparators are used by hadoop as part of the post-map sort, when
+ * the data is still in object format.
+ */
+ public int compareTo(Object o) {
+ PigNullableWritable w = (PigNullableWritable)o;
+ if (!mNull && !w.mNull) {
+ return mValue.compareTo(w.mValue);
+ } else if (mNull && w.mNull) {
+ // If they're both null, compare the indicies
+ if (mIndex < w.mIndex) return -1;
+ else if (mIndex > w.mIndex) return 1;
+ else return 0;
+ }
+ else if (mNull) return -1;
+ else return 1;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput)
+ */
+ public void readFields(DataInput in) throws IOException {
+ mNull = in.readBoolean();
+ if (!mNull) mValue.readFields(in);
+ mIndex = in.readByte();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput)
+ */
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(mNull);
+ if (!mNull) mValue.write(out);
+ out.writeByte(mIndex);
+ }
+
+ /**
+ * @return the isNull
+ */
+ public boolean isNull() {
+ return mNull;
+ }
+
+ /**
+ * @param isNull the isNull to set
+ */
+ public void setNull(boolean isNull) {
+ mNull = isNull;
+ }
+
+ /**
+ * @return the index for this value
+ */
+ public byte getIndex() {
+ return mIndex;
+ }
+
+ /**
+ * @param index for this value.
+ */
+ public void setIndex(byte index) {
+ mIndex = index;
+ }
+
+ /**
+ * @return The wrapped value as a pig type, not as a WritableComparable.
+ */
+ abstract public Object getValueAsPigType();
+
+ @Override
+ public int hashCode() {
+ // For now, always give a null a hash code of 0. It isn't clear this
+ // is what we'll always want. If nulls make a significant but
+ // not overwhelming amount of the data we may want them to get their
+ // own partition. If they make up a big enough percentage of the
+ // data we may want to split them across partitions (though that
+ // would obviously limit how they could be dealt with afterwards).
+ if (mNull) return 0;
+ else return mValue.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return new String("Null: " + mNull + " index: " + mIndex + " " +
+ mValue.toString());
+ }
+}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java Fri Sep 19 13:39:31 2008
@@ -126,8 +126,6 @@
}
- /*
-
@Test
public void testSimpleCount() throws Exception {
File tmpFile = File.createTempFile("test", "txt");
@@ -281,7 +279,6 @@
}
}
}
-*/
private int generateInput(PrintStream ps, boolean withNulls ) {
int numNulls = 0;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java Fri Sep 19 13:39:31 2008
@@ -345,10 +345,11 @@
if (eliminateDuplicates){
pigServer.registerQuery("B = DISTINCT (FOREACH A GENERATE $0) PARALLEL 10;");
}else{
- if(!useUDF)
+ if(!useUDF) {
pigServer.registerQuery("B = ORDER A BY $0 PARALLEL 10;");
- else
+ } else {
pigServer.registerQuery("B = ORDER A BY $0 using " + TupComp.class.getName() + ";");
+ }
}
pigServer.store("B", tmpOutputFile);
@@ -359,12 +360,13 @@
if(!iter.hasNext()) fail("No Results obtained");
while (iter.hasNext()){
Tuple t = iter.next();
- System.out.println(t.get(0).toString());
+ //System.out.println(t.get(0).toString());
if (eliminateDuplicates){
Integer act = Integer.parseInt(t.get(0).toString());
assertFalse(seen.contains(act));
seen.add(act);
}else{
+System.out.println(last + " " + t.get(0));
assertTrue(last.compareTo(t.get(0).toString())<=0);
assertEquals(t.size(), 2);
last = t.get(0).toString();
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java Fri Sep 19 13:39:31 2008
@@ -29,7 +29,6 @@
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultBagFactory;
import org.apache.pig.data.DefaultTuple;
-import org.apache.pig.data.IndexedTuple;
import org.apache.pig.data.Tuple;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java Fri Sep 19 13:39:31 2008
@@ -25,7 +25,6 @@
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultTuple;
-import org.apache.pig.data.IndexedTuple;
import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
@@ -34,6 +33,7 @@
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange;
+import org.apache.pig.impl.io.NullableTuple;
import org.apache.pig.impl.plan.PlanException;
import org.apache.pig.test.utils.GenPhyOp;
import org.apache.pig.test.utils.GenRandomData;
@@ -84,21 +84,21 @@
int size=0;
for(Result res=lr.getNext(t);res.returnStatus!=POStatus.STATUS_EOP;res=lr.getNext(t)){
Tuple t = (Tuple)res.result;
- IndexedTuple it = (IndexedTuple)t.get(1);
+ Tuple val = (Tuple)t.get(2);
//Check if the index is same as input index
- assertEquals((float)0, (float)it.index, 0.01f);
+ assertEquals((byte)0, (byte)(Byte)t.get(0));
- //Check if the input baf contains the value tuple
- assertEquals(true, TestHelper.bagContains(db, it.toTuple()));
+ //Check if the input bag contains the value tuple
+ assertTrue(TestHelper.bagContains(db, val));
//Check if the input key and the output key are same
- String inpKey = (String)it.toTuple().get(0);
- assertEquals(true, inpKey.compareTo((String)t.get(0))==0);
+ String inpKey = (String)val.get(0);
+ assertEquals(0, inpKey.compareTo((String)t.get(1)));
++size;
}
//check if all the tuples in the input are generated
- assertEquals(true, size==db.size());
+ assertEquals(db.size(), size);
}
private void setUp2() throws PlanException, ExecException{
@@ -127,23 +127,23 @@
int size=0;
for(Result res=lr.getNext(t);res.returnStatus!=POStatus.STATUS_EOP;res=lr.getNext(t)){
Tuple t = (Tuple)res.result;
- IndexedTuple it = (IndexedTuple)t.get(1);
+ Tuple val = (Tuple)t.get(2);
//Check if the index is same as input index
- assertEquals((float)0, (float)it.index, 0.01f);
+ assertEquals((byte)0, (byte)(Byte)t.get(0));
//Check if the input baf contains the value tuple
- assertEquals(true, TestHelper.bagContains(db, it.toTuple()));
+ assertTrue(TestHelper.bagContains(db, val));
//Check if the input key and the output key are same
Tuple inpKey = TupleFactory.getInstance().newTuple(2);
- inpKey.set(0,it.toTuple().get(0));
- inpKey.set(1,it.toTuple().get(1));
- assertEquals(true, inpKey.compareTo((Tuple)t.get(0))==0);
+ inpKey.set(0, val.get(0));
+ inpKey.set(1, val.get(1));
+ assertEquals(0, inpKey.compareTo((Tuple)t.get(1)));
++size;
}
//check if all the tuples in the input are generated
- assertEquals(true, size==db.size());
+ assertEquals(db.size(), size);
}
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java?rev=697229&r1=697228&r2=697229&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java Fri Sep 19 13:39:31 2008
@@ -20,19 +20,24 @@
import static org.junit.Assert.assertEquals;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
import java.util.Random;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.hadoop.HDataType;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.data.DataBag;
import org.apache.pig.data.DataType;
import org.apache.pig.data.DefaultBagFactory;
-import org.apache.pig.data.IndexedTuple;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.NullableTuple;
+import org.apache.pig.impl.io.PigNullableWritable;
import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
-import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage;
import org.apache.pig.test.utils.GenRandomData;
import org.apache.pig.test.utils.TestHelper;
import org.junit.After;
@@ -49,124 +54,104 @@
public void tearDown() throws Exception {
}
- static class ITIterator implements Iterator<IndexedTuple>,
- Iterable<IndexedTuple> {
- private Iterator<Tuple> it;
-
- public ITIterator(Iterator<Tuple> it) {
- this.it = it;
- }
-
- public boolean hasNext() {
- return it.hasNext();
- }
-
- public IndexedTuple next() {
- return (IndexedTuple) it.next();
- }
-
- public void remove() {
- // TODO Auto-generated method stub
-
- }
-
- public Iterator<IndexedTuple> iterator() {
- return this;
- }
-
- }
-
- public static boolean test(Object key, boolean inner[]) throws ExecException, IOException {
- boolean ret = false;
+ private void runTest(Object key, boolean inner[]) throws ExecException, IOException {
Random r = new Random();
DataBag db1 = GenRandomData.genRandSmallTupDataBag(r, 10, 100);
DataBag db2 = GenRandomData.genRandSmallTupDataBag(r, 10, 100);
- DataBag db = DefaultBagFactory.getInstance().newDefaultBag();
+ List<NullableTuple> db = new ArrayList<NullableTuple>(200);
Iterator db1Iter = db1.iterator();
if(!inner[0]){
while (db1Iter.hasNext()) {
- IndexedTuple it = new IndexedTuple((Tuple) db1Iter.next(), 0);
+ NullableTuple it = new NullableTuple((Tuple)db1Iter.next());
+ it.setIndex((byte)0);
db.add(it);
}
}
Iterator db2Iter = db2.iterator();
while (db2Iter.hasNext()) {
- IndexedTuple it = new IndexedTuple((Tuple) db2Iter.next(), 1);
+ NullableTuple it = new NullableTuple((Tuple) db2Iter.next());
+ it.setIndex((byte)1);
db.add(it);
}
- ITIterator iti = new TestPackage.ITIterator(db.iterator());
+ //ITIterator iti = new TestPackage.ITIterator(db.iterator());
POPackage pop = new POPackage(new OperatorKey("", r.nextLong()));
pop.setNumInps(2);
pop.setInner(inner);
- pop.attachInput(key, iti);
+ PigNullableWritable k = HDataType.getWritableComparableTypes(key, (byte)0);
+ pop.attachInput(k, db.iterator());
Tuple t = null;
Result res = null;
res = (Result) pop.getNext(t);
- if(res.returnStatus==POStatus.STATUS_NULL && inner[0])
- return true;
- if (res.returnStatus != POStatus.STATUS_OK)
- return false;
+ if(res.returnStatus==POStatus.STATUS_NULL && inner[0]) return;
+ assertEquals(POStatus.STATUS_OK, res.returnStatus);
t = (Tuple) res.result;
Object outKey = t.get(0);
DataBag outDb1 = (DataBag) t.get(1);
DataBag outDb2 = (DataBag) t.get(2);
- if (outKey == key && TestHelper.compareBags(db1, outDb1)
- && TestHelper.compareBags(db2, outDb2))
- return true;
- return ret;
+ assertEquals(key, outKey);
+ assertTrue(TestHelper.compareBags(db1, outDb1));
+ assertTrue(TestHelper.compareBags(db2, outDb2));
}
/**
* To show that it does not have any type specific
* code
*/
- private static boolean test(byte t, boolean[] inner) throws ExecException, IOException {
+ private void pickTest(byte t, boolean[] inner) throws ExecException, IOException {
Random r = new Random();
switch (t) {
case DataType.BAG:
- return test(GenRandomData.genRandSmallTupDataBag(r, 10, 100),inner);
+ runTest(GenRandomData.genRandSmallTupDataBag(r, 10, 100),inner);
+ break;
case DataType.BOOLEAN:
- return test(r.nextBoolean(),inner);
+ runTest(r.nextBoolean(),inner);
+ break;
case DataType.BYTEARRAY:
- return test(GenRandomData.genRandDBA(r),inner);
+ runTest(GenRandomData.genRandDBA(r),inner);
+ break;
case DataType.CHARARRAY:
- return test(GenRandomData.genRandString(r),inner);
+ runTest(GenRandomData.genRandString(r),inner);
+ break;
case DataType.DOUBLE:
- return test(r.nextDouble(),inner);
+ runTest(r.nextDouble(),inner);
+ break;
case DataType.FLOAT:
- return test(r.nextFloat(),inner);
+ runTest(r.nextFloat(),inner);
+ break;
case DataType.INTEGER:
- return test(r.nextLong(),inner);
+ runTest(r.nextLong(),inner);
+ break;
case DataType.LONG:
- return test(r.nextLong(),inner);
+ runTest(r.nextLong(),inner);
+ break;
case DataType.MAP:
- return test(GenRandomData.genRandMap(r, 10),inner);
+ case DataType.BYTE:
+ return; // map not key type
case DataType.TUPLE:
- return test(GenRandomData.genRandSmallBagTuple(r, 10, 100),inner);
+ runTest(GenRandomData.genRandSmallBagTuple(r, 10, 100),inner);
+ break;
+
+ default:
+ fail("No test case for type " + DataType.findTypeName(t));
}
- return false;
}
@Test
public void testOperator() throws ExecException, IOException{
byte[] types = DataType.genAllTypes();
-// Map<Byte, String> map = operatorHelper.genTypeToNameMap();
-// System.out.println("Testing Package:");
for (byte b : types) {
-// System.out.print("\t With " + map.get(b) + ": ");
+ System.out.println("Type " + DataType.findTypeName(b));
boolean succ = true;
int NUM_TRIALS = 10;
boolean[] inner1 = { false , false };
for (int i = 0; i < NUM_TRIALS; i++)
- succ &= test(b, inner1);
- assertEquals(true, succ);
+ pickTest(b, inner1);
boolean[] inner2 = { true , false };
for (int i = 0; i < NUM_TRIALS; i++)
- succ &= test(b, inner2);
- assertEquals(true, succ);
+ pickTest(b, inner2);
/*if (succ)
System.out.println("Success!!");
else