You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/06/02 02:52:24 UTC
orc git commit: ORC-53. Make the complex types Comparable so they can
be put as keys to the OrcMap. (omalley)
Repository: orc
Updated Branches:
refs/heads/master 7a4fe255e -> 545fe3712
ORC-53. Make the complex types Comparable so they can be put as keys to the
OrcMap. (omalley)
Fixes #29
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/545fe371
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/545fe371
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/545fe371
Branch: refs/heads/master
Commit: 545fe37128111a39b24faf40ae0851069317d879
Parents: 7a4fe25
Author: Owen O'Malley <om...@apache.org>
Authored: Thu May 26 08:17:29 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Jun 1 19:51:57 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/orc/TypeDescription.java | 64 ++++++++-
.../org/apache/orc/impl/SchemaEvolution.java | 6 +-
java/core/src/test/resources/log4j.properties | 5 +
.../src/java/org/apache/orc/mapred/OrcList.java | 36 ++++-
.../src/java/org/apache/orc/mapred/OrcMap.java | 60 +++++++-
.../org/apache/orc/mapred/OrcRecordReader.java | 12 +-
.../java/org/apache/orc/mapred/OrcStruct.java | 66 +++++++--
.../org/apache/orc/mapred/OrcTimestamp.java | 6 +-
.../java/org/apache/orc/mapred/OrcUnion.java | 27 +++-
.../test/org/apache/orc/mapred/TestOrcList.java | 45 ++++++
.../test/org/apache/orc/mapred/TestOrcMap.java | 138 ++++++++++++++++++-
.../org/apache/orc/mapred/TestOrcStruct.java | 44 ++++++
.../org/apache/orc/mapred/TestOrcUnion.java | 33 +++++
.../src/test/resources/log4j.properties | 5 +
java/pom.xml | 1 +
java/storage-api/pom.xml | 12 ++
16 files changed, 515 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/core/src/java/org/apache/orc/TypeDescription.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/TypeDescription.java b/java/core/src/java/org/apache/orc/TypeDescription.java
index d4c66d1..da9fe49 100644
--- a/java/core/src/java/org/apache/orc/TypeDescription.java
+++ b/java/core/src/java/org/apache/orc/TypeDescription.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.UnionColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -37,12 +38,61 @@ import java.util.List;
/**
* This is the description of the types in an ORC file.
*/
-public class TypeDescription {
+public class TypeDescription
+ implements Comparable<TypeDescription>, Serializable {
private static final int MAX_PRECISION = 38;
private static final int MAX_SCALE = 38;
private static final int DEFAULT_PRECISION = 38;
private static final int DEFAULT_SCALE = 10;
private static final int DEFAULT_LENGTH = 256;
+
+ @Override
+ public int compareTo(TypeDescription other) {
+ if (this == other) {
+ return 0;
+ } else if (other == null) {
+ return -1;
+ } else {
+ int result = category.compareTo(other.category);
+ if (result == 0) {
+ switch (category) {
+ case CHAR:
+ case VARCHAR:
+ return maxLength - other.maxLength;
+ case DECIMAL:
+ if (precision != other.precision) {
+ return precision - other.precision;
+ }
+ return scale - other.scale;
+ case UNION:
+ case LIST:
+ case MAP:
+ if (children.size() != other.children.size()) {
+ return children.size() - other.children.size();
+ }
+ for(int c=0; result == 0 && c < children.size(); ++c) {
+ result = children.get(c).compareTo(other.children.get(c));
+ }
+ break;
+ case STRUCT:
+ if (children.size() != other.children.size()) {
+ return children.size() - other.children.size();
+ }
+ for(int c=0; result == 0 && c < children.size(); ++c) {
+ result = fieldNames.get(c).compareTo(other.fieldNames.get(c));
+ if (result == 0) {
+ result = children.get(c).compareTo(other.children.get(c));
+ }
+ }
+ break;
+ default:
+ // PASS
+ }
+ }
+ return result;
+ }
+ }
+
public enum Category {
BOOLEAN("boolean", true),
BYTE("tinyint", true),
@@ -463,12 +513,18 @@ public class TypeDescription {
@Override
public int hashCode() {
- return getId();
+ long result = category.ordinal() * 4241 + maxLength + precision * 13 + scale;
+ if (children != null) {
+ for(TypeDescription child: children) {
+ result = result * 6959 + child.hashCode();
+ }
+ }
+ return (int) result;
}
@Override
public boolean equals(Object other) {
- if (other == null || other.getClass() != TypeDescription.class) {
+ if (other == null || !(other instanceof TypeDescription)) {
return false;
}
if (other == this) {
@@ -476,8 +532,6 @@ public class TypeDescription {
}
TypeDescription castOther = (TypeDescription) other;
if (category != castOther.category ||
- getId() != castOther.getId() ||
- getMaximumId() != castOther.getMaximumId() ||
maxLength != castOther.maxLength ||
scale != castOther.scale ||
precision != castOther.precision) {
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
----------------------------------------------------------------------
diff --git a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
index 2c80aaa..3e4e47b 100644
--- a/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
+++ b/java/core/src/java/org/apache/orc/impl/SchemaEvolution.java
@@ -33,7 +33,7 @@ import org.apache.orc.TypeDescription;
* has been schema evolution.
*/
public class SchemaEvolution {
- private final Map<TypeDescription, TypeDescription> readerToFile;
+ private final Map<Integer, TypeDescription> readerToFile;
private final boolean[] included;
private final TypeDescription readerSchema;
private static final Log LOG = LogFactory.getLog(SchemaEvolution.class);
@@ -70,7 +70,7 @@ public class SchemaEvolution {
result = null;
}
} else {
- result = readerToFile.get(readerType);
+ result = readerToFile.get(readerType.getId());
}
return result;
}
@@ -143,7 +143,7 @@ public class SchemaEvolution {
isOk = ConvertTreeReaderFactory.canConvert(fileType, readerType);
}
if (isOk) {
- readerToFile.put(readerType, fileType);
+ readerToFile.put(readerType.getId(), fileType);
} else {
throw new IOException(
String.format(
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/core/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/java/core/src/test/resources/log4j.properties b/java/core/src/test/resources/log4j.properties
new file mode 100644
index 0000000..67e071d
--- /dev/null
+++ b/java/core/src/test/resources/log4j.properties
@@ -0,0 +1,5 @@
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%p\t%d{ISO8601}\t%r\t%c\t[%t]\t%m%n
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java
index 2b94207..b950671 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcList.java
@@ -17,7 +17,7 @@
*/
package org.apache.orc.mapred;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.orc.TypeDescription;
import java.io.DataInput;
@@ -30,8 +30,8 @@ import java.util.Iterator;
* An ArrayList implementation that implements Writable.
* @param <E> the element type, which must be Writable
*/
-public class OrcList<E extends Writable>
- extends ArrayList<E> implements Writable {
+public class OrcList<E extends WritableComparable>
+ extends ArrayList<E> implements WritableComparable<OrcList<E>> {
private final TypeDescription childSchema;
public OrcList(TypeDescription schema) {
@@ -71,4 +71,34 @@ public class OrcList<E extends Writable>
}
}
}
+
+ @Override
+ public int compareTo(OrcList<E> other) {
+ if (other == null) {
+ return -1;
+ }
+ int result = childSchema.compareTo(other.childSchema);
+ if (result != 0) {
+ return result;
+ }
+ int ourSize = size();
+ int otherSize = other.size();
+ for(int e=0; e < ourSize && e < otherSize; ++e) {
+ E ours = get(e);
+ E theirs = other.get(e);
+ if (ours == null) {
+ if (theirs != null) {
+ return 1;
+ }
+ } else if (theirs == null) {
+ return -1;
+ } else {
+ int val = ours.compareTo(theirs);
+ if (val != 0) {
+ return val;
+ }
+ }
+ }
+ return ourSize - otherSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java
index db961fc..cf47827 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcMap.java
@@ -17,24 +17,24 @@
*/
package org.apache.orc.mapred;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.orc.TypeDescription;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
/**
* A TreeMap implementation that implements Writable.
- * @param <K> the key type, which must be Writable
- * @param <V> the value type, which must be Writable
+ * @param <K> the key type, which must be WritableComparable
+ * @param <V> the value type, which must be WritableComparable
*/
-public final class OrcMap<K extends Writable, V extends Writable>
- extends TreeMap<K, V> implements Writable {
+public final class OrcMap<K extends WritableComparable,
+ V extends WritableComparable>
+ extends TreeMap<K, V> implements WritableComparable<OrcMap<K,V>> {
private final TypeDescription keySchema;
private final TypeDescription valueSchema;
@@ -83,4 +83,52 @@ public final class OrcMap<K extends Writable, V extends Writable>
put(key, value);
}
}
+
+ @Override
+ public int compareTo(OrcMap<K,V> other) {
+ if (other == null) {
+ return -1;
+ }
+ int result = keySchema.compareTo(other.keySchema);
+ if (result != 0) {
+ return result;
+ }
+ result = valueSchema.compareTo(other.valueSchema);
+ if (result != 0) {
+ return result;
+ }
+ Iterator<Map.Entry<K,V>> ourItr = entrySet().iterator();
+ Iterator<Map.Entry<K,V>> theirItr = other.entrySet().iterator();
+ while (ourItr.hasNext() && theirItr.hasNext()) {
+ Map.Entry<K,V> ourItem = ourItr.next();
+ Map.Entry<K,V> theirItem = theirItr.next();
+ K ourKey = ourItem.getKey();
+ K theirKey = theirItem.getKey();
+ int val = ourKey.compareTo(theirKey);
+ if (val != 0) {
+ return val;
+ }
+ Comparable<V> ourValue = ourItem.getValue();
+ V theirValue = theirItem.getValue();
+ if (ourValue == null) {
+ if (theirValue != null) {
+ return 1;
+ }
+ } else if (theirValue == null) {
+ return -1;
+ } else {
+ val = ourItem.getValue().compareTo(theirItem.getValue());
+ if (val != 0) {
+ return val;
+ }
+ }
+ }
+ if (ourItr.hasNext()) {
+ return 1;
+ } else if (theirItr.hasNext()) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java
index 0370ae5..f6bf635 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcRecordReader.java
@@ -44,12 +44,12 @@ import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.orc.Reader;
import org.apache.orc.RecordReader;
import org.apache.orc.TypeDescription;
-public class OrcRecordReader<V extends Writable>
+public class OrcRecordReader<V extends WritableComparable>
implements org.apache.hadoop.mapred.RecordReader<NullWritable, V> {
private final TypeDescription schema;
private final RecordReader batchReader;
@@ -501,10 +501,10 @@ public class OrcRecordReader<V extends Writable>
}
}
- static Writable nextValue(ColumnVector vector,
- int row,
- TypeDescription schema,
- Object previous) {
+ static WritableComparable nextValue(ColumnVector vector,
+ int row,
+ TypeDescription schema,
+ Object previous) {
switch (schema.getCategory()) {
case BOOLEAN:
return nextBoolean(vector, row, previous);
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java
index 74b3b28..2dd749b 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcStruct.java
@@ -28,28 +28,28 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.orc.TypeDescription;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-public final class OrcStruct implements Writable {
+public final class OrcStruct implements WritableComparable<OrcStruct> {
- private Writable[] fields;
+ private WritableComparable[] fields;
private final TypeDescription schema;
public OrcStruct(TypeDescription schema) {
this.schema = schema;
- fields = new Writable[schema.getChildren().size()];
+ fields = new WritableComparable[schema.getChildren().size()];
}
- public Writable getFieldValue(int fieldIndex) {
+ public WritableComparable getFieldValue(int fieldIndex) {
return fields[fieldIndex];
}
- public void setFieldValue(int fieldIndex, Writable value) {
+ public void setFieldValue(int fieldIndex, WritableComparable value) {
fields[fieldIndex] = value;
}
@@ -59,10 +59,10 @@ public final class OrcStruct implements Writable {
@Override
public void write(DataOutput output) throws IOException {
- for(int f=0; f < fields.length; ++f) {
- output.writeBoolean(fields[f] != null);
- if (fields[f] != null) {
- fields[f].write(output);
+ for(WritableComparable field: fields) {
+ output.writeBoolean(field != null);
+ if (field != null) {
+ field.write(output);
}
}
}
@@ -81,7 +81,21 @@ public final class OrcStruct implements Writable {
}
}
- public void setFieldValue(String fieldName, Writable value) {
+ /**
+ * Set all of the fields in the struct
+ * @param values the list of values for each of the fields.
+ */
+ public void setAllFields(WritableComparable... values) {
+ if (fields.length != values.length) {
+ throw new IllegalArgumentException("Wrong number (" + values.length +
+ ") of fields for " + schema);
+ }
+ for (int col = 0; col < fields.length && col < values.length; ++col) {
+ fields[col] = values[col];
+ }
+ }
+
+ public void setFieldValue(String fieldName, WritableComparable value) {
int fieldIdx = schema.getFieldNames().indexOf(fieldName);
if (fieldIdx == -1) {
throw new IllegalArgumentException("Field " + fieldName +
@@ -90,7 +104,7 @@ public final class OrcStruct implements Writable {
fields[fieldIdx] = value;
}
- public Writable getFieldValue(String fieldName) {
+ public WritableComparable getFieldValue(String fieldName) {
int fieldIdx = schema.getFieldNames().indexOf(fieldName);
if (fieldIdx == -1) {
throw new IllegalArgumentException("Field " + fieldName +
@@ -150,7 +164,7 @@ public final class OrcStruct implements Writable {
/* Routines for stubbing into Writables */
- public static Writable createValue(TypeDescription type) {
+ public static WritableComparable createValue(TypeDescription type) {
switch (type.getCategory()) {
case BOOLEAN: return new BooleanWritable();
case BYTE: return new ByteWritable();
@@ -185,4 +199,30 @@ public final class OrcStruct implements Writable {
throw new IllegalArgumentException("Unknown type " + type);
}
}
+
+ @Override
+ public int compareTo(OrcStruct other) {
+ if (other == null) {
+ return -1;
+ }
+ int result = schema.compareTo(other.schema);
+ if (result != 0) {
+ return result;
+ }
+ for(int c = 0; c < fields.length && c < other.fields.length; ++c) {
+ if (fields[c] == null) {
+ if (other.fields[c] != null) {
+ return 1;
+ }
+ } else if (other.fields[c] == null) {
+ return -1;
+ } else {
+ int val = fields[c].compareTo(other.fields[c]);
+ if (val != 0) {
+ return val;
+ }
+ }
+ }
+ return fields.length - other.fields.length;
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java
index 200a966..ee97b9f 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcTimestamp.java
@@ -17,17 +17,19 @@
*/
package org.apache.orc.mapred;
-import org.apache.hadoop.io.Writable;
+
+import org.apache.hadoop.io.WritableComparable;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.sql.Timestamp;
+import java.util.Date;
/**
* A Timestamp implementation that implements Writable.
*/
-public class OrcTimestamp extends Timestamp implements Writable {
+public class OrcTimestamp extends Timestamp implements WritableComparable<Date> {
public OrcTimestamp() {
super(0);
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java b/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java
index 3e7c909..6eb0e6a 100644
--- a/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java
+++ b/java/mapreduce/src/java/org/apache/orc/mapred/OrcUnion.java
@@ -18,6 +18,7 @@
package org.apache.orc.mapred;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableComparable;
import org.apache.orc.TypeDescription;
import java.io.DataInput;
@@ -27,17 +28,17 @@ import java.io.IOException;
/**
* An in-memory representation of a union type.
*/
-public final class OrcUnion implements Writable {
+public final class OrcUnion implements WritableComparable<OrcUnion> {
private byte tag;
- private Writable object;
+ private WritableComparable object;
private final TypeDescription schema;
public OrcUnion(TypeDescription schema) {
this.schema = schema;
}
- public void set(byte tag, Writable object) {
- this.tag = tag;
+ public void set(int tag, WritableComparable object) {
+ this.tag = (byte) tag;
this.object = object;
}
@@ -100,4 +101,22 @@ public final class OrcUnion implements Writable {
object = null;
}
}
+
+ @Override
+ public int compareTo(OrcUnion other) {
+ if (other == null) {
+ return -1;
+ }
+ int result = schema.compareTo(other.schema);
+ if (result != 0) {
+ return result;
+ }
+ if (tag != other.tag) {
+ return tag - other.tag;
+ }
+ if (object == null) {
+ return other.object == null ? 0 : 1;
+ }
+ return object.compareTo(other.object);
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java
index 4ac7ca9..b0cdfb7 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcList.java
@@ -21,6 +21,7 @@ package org.apache.orc.mapred;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.orc.TypeDescription;
import org.junit.Test;
@@ -62,4 +63,48 @@ public class TestOrcList {
cloneWritable(expected, actual);
assertEquals(expected, actual);
}
+
+ @Test
+ public void testCompare() {
+ TypeDescription schema = TypeDescription.fromString("array<string>");
+ OrcList<Text> left = new OrcList<>(schema);
+ assertEquals(-1 ,left.compareTo(null));
+ OrcList<Text> right = new OrcList<>(schema);
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+ right.add(new Text("aa"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ left.add(new Text("aa"));
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+ left.add(new Text("bb"));
+ right.add(new Text("cc"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ left.clear();
+ right.clear();
+ left.add(null);
+ right.add(null);
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+ right.clear();
+ right.add(new Text("ddd"));
+ assertEquals(1, left.compareTo(right));
+ assertEquals(-1, right.compareTo(left));
+ }
+
+ @Test
+ public void testSchemaInCompare() {
+ TypeDescription leftType = TypeDescription.fromString("array<int>");
+ TypeDescription rightType = TypeDescription.fromString("array<string>");
+ OrcList leftList = new OrcList(leftType);
+ OrcList rightList = new OrcList(rightType);
+ assertEquals(-4, leftList.compareTo(rightList));
+ assertEquals(4, rightList.compareTo(leftList));
+ leftList.add(new IntWritable(123));
+ rightList.add(new Text("123"));
+ assertEquals(-4, leftList.compareTo(rightList));
+ assertEquals(4, rightList.compareTo(leftList));
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java
index 34e1feb..d34b72e 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcMap.java
@@ -18,11 +18,9 @@
package org.apache.orc.mapred;
-import org.apache.hadoop.io.DataInputBuffer;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.Text;
import org.apache.orc.TypeDescription;
import org.junit.Test;
@@ -54,4 +52,138 @@ public class TestOrcMap {
TestOrcList.cloneWritable(expected, actual);
assertEquals(expected, actual);
}
+
+ @Test
+ public void testCompare() {
+ TypeDescription schema = TypeDescription.fromString("map<string,string>");
+ OrcMap<Text,Text> left = new OrcMap<>(schema);
+ assertEquals(-1 ,left.compareTo(null));
+ OrcMap<Text,Text> right = new OrcMap<>(schema);
+
+ // empty maps
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+
+ // {} vs {"aa" -> null}
+ right.put(new Text("aa"), null);
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+
+ // {"aa" -> null} vs {"aa" -> null}
+ left.put(new Text("aa"), null);
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+
+ // {"aa" -> "bb"} vs {"aa" -> "bb"}
+ left.put(new Text("aa"), new Text("bb"));
+ right.put(new Text("aa"), new Text("bb"));
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+
+ // {"aa" -> "bb"} vs {"aa" -> "cc"}
+ right.put(new Text("aa"), new Text("cc"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+
+ // {"aa" -> "bb"} vs {"a" -> "zzz", "aa" -> "cc"}
+ right.put(new Text("a"), new Text("zzz"));
+ assertEquals(1, left.compareTo(right));
+ assertEquals(-1, right.compareTo(left));
+
+ // {"aa" -> null} vs {"aa" -> "bb"}
+ left.put(new Text("aa"), null);
+ right.remove(new Text("a"));
+ right.put(new Text("aa"), new Text("cc"));
+ assertEquals(1, left.compareTo(right));
+ assertEquals(-1, right.compareTo(left));
+
+ // {"aa" -> null, "bb" -> "cc"} vs {"aa" -> null, "bb" -> "dd"}
+ left.put(new Text("aa"), null);
+ left.put(new Text("bb"), new Text("cc"));
+ right.put(new Text("aa"), null);
+ right.put(new Text("bb"), new Text("dd"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ }
+
+ @Test
+ public void testStructKeys() {
+ TypeDescription schema = TypeDescription.fromString("map<struct<i:int>,string>");
+ OrcMap<OrcStruct, Text> map = new OrcMap<>(schema);
+ OrcStruct struct = new OrcStruct(schema.getChildren().get(0));
+ struct.setFieldValue(0, new IntWritable(12));
+ map.put(struct, new Text("a"));
+ assertEquals("a", map.get(struct).toString());
+ struct = new OrcStruct(schema.getChildren().get(0));
+ struct.setFieldValue(0, new IntWritable(14));
+ map.put(struct, new Text("b"));
+ assertEquals(2, map.size());
+ }
+
+ @Test
+ public void testListKeys() {
+ TypeDescription schema = TypeDescription.fromString("map<array<int>,string>");
+ OrcMap<OrcList, Text> map = new OrcMap<>(schema);
+ OrcList<IntWritable> list = new OrcList<>(schema.getChildren().get(0));
+ list.add(new IntWritable(123));
+ map.put(list, new Text("a"));
+ assertEquals("a", map.get(list).toString());
+ list = new OrcList<>(schema.getChildren().get(0));
+ list.add(new IntWritable(333));
+ map.put(list, new Text("b"));
+ assertEquals(2, map.size());
+ assertEquals("b", map.get(list).toString());
+ }
+
+ @Test
+ public void testUnionKeys() {
+ TypeDescription schema = TypeDescription.fromString("map<uniontype<int,string>,string>");
+ OrcMap<OrcUnion, Text> map = new OrcMap<>(schema);
+ OrcUnion un = new OrcUnion(schema.getChildren().get(0));
+ un.set(0, new IntWritable(123));
+ map.put(un, new Text("hi"));
+ un = new OrcUnion(schema.getChildren().get(0));
+ un.set(1, new Text("aaaa"));
+ map.put(un, new Text("google"));
+ assertEquals(2, map.size());
+ assertEquals("google", map.get(un).toString());
+ }
+
+ @Test
+ public void testMapKeys() {
+ TypeDescription schema = TypeDescription.fromString("map<map<string,string>,string>");
+ OrcMap<OrcMap<Text,Text>, Text> left = new OrcMap<>(schema);
+
+ assertEquals(-1, left.compareTo(null));
+
+ OrcMap<OrcMap<Text,Text>, Text> right = new OrcMap<>(schema);
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+
+ OrcMap<Text,Text> item = new OrcMap<>(schema.getChildren().get(0));
+ item.put(new Text("aa"), new Text("bb"));
+ left.put(item, new Text("cc"));
+ assertEquals(1, left.compareTo(right));
+ assertEquals(-1, right.compareTo(left));
+
+ item = new OrcMap<>(schema.getChildren().get(0));
+ item.put(new Text("aa"), new Text("dd"));
+ right.put(item, new Text("bb"));
+ assertEquals(-2, left.compareTo(right));
+ assertEquals(2, right.compareTo(left));
+ }
+
+ @Test
+ public void testSchemaInCompare() {
+ TypeDescription leftType = TypeDescription.fromString("map<string,int>");
+ TypeDescription rightType = TypeDescription.fromString("map<string,string>");
+ OrcMap left = new OrcMap(leftType);
+ OrcMap right = new OrcMap(rightType);
+ assertEquals(-4, left.compareTo(right));
+ assertEquals(4, right.compareTo(left));
+ left.put(new Text("123"), new IntWritable(123));
+ right.put(new Text("123"), new Text("123"));
+ assertEquals(-4, left.compareTo(right));
+ assertEquals(4, right.compareTo(left));
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java
index 4af6a1c..d32ce94 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcStruct.java
@@ -93,4 +93,48 @@ public class TestOrcStruct {
thrown.expect(IllegalArgumentException.class);
struct.setFieldValue("bad", new Text("foobar"));
}
+
+ @Test
+ public void testCompare() {
+ OrcStruct left = new OrcStruct(TypeDescription.fromString
+ ("struct<i:int,j:string>"));
+ assertEquals(-1 ,left.compareTo(null));
+ OrcStruct right = new OrcStruct(TypeDescription.fromString
+ ("struct<i:int,j:string,k:int>"));
+ left.setFieldValue(0, new IntWritable(10));
+ right.setFieldValue(0, new IntWritable(12));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ right.setFieldValue(0, new IntWritable(10));
+ left.setFieldValue(1, new Text("a"));
+ right.setFieldValue(1, new Text("b"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ right.setFieldValue(1, new Text("a"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ right = new OrcStruct(TypeDescription.fromString
+ ("struct<i:int,j:string>"));
+ left.setFieldValue(0, null);
+ left.setFieldValue(1, null);
+ assertEquals(0, left.compareTo(right));
+ assertEquals(0, right.compareTo(left));
+ right.setFieldValue(0, new IntWritable(12));
+ assertEquals(1 , left.compareTo(right));
+ assertEquals(-1, right.compareTo(left));
+ }
+
+ @Test
+ public void testSchemaInCompare() {
+ TypeDescription leftType = TypeDescription.fromString("struct<s:string,i:int>");
+ TypeDescription rightType = TypeDescription.fromString("struct<s:string,j:bigint>");
+ OrcStruct left = new OrcStruct(leftType);
+ OrcStruct right = new OrcStruct(rightType);
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ left.setAllFields(new Text("123"), new IntWritable(123));
+ right.setAllFields(new Text("123"), new LongWritable(456));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java
index 82fd94f..7c8d523 100644
--- a/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java
+++ b/java/mapreduce/src/test/org/apache/orc/mapred/TestOrcUnion.java
@@ -55,4 +55,37 @@ public class TestOrcUnion {
TestOrcList.cloneWritable(expected, actual);
assertEquals(expected, actual);
}
+
+ @Test
+ public void testCompare() {
+ TypeDescription schema =
+ TypeDescription.fromString("uniontype<int,string,bigint>");
+ OrcUnion left = new OrcUnion(schema);
+ OrcUnion right = new OrcUnion(schema);
+ assertEquals(-1 ,left.compareTo(null));
+ assertEquals(0, left.compareTo(right));
+
+ left.set(1, new IntWritable(10));
+ right.set(1, new IntWritable(12));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+
+ right.set(2, new Text("a"));
+ assertEquals(-1, left.compareTo(right));
+ assertEquals(1, right.compareTo(left));
+ }
+
+ @Test
+ public void testSchemaInCompare() {
+ TypeDescription leftType = TypeDescription.fromString("uniontype<string,tinyint>");
+ TypeDescription rightType = TypeDescription.fromString("uniontype<string,bigint>");
+ OrcUnion left = new OrcUnion(leftType);
+ OrcUnion right = new OrcUnion(rightType);
+ assertEquals(-3, left.compareTo(right));
+ assertEquals(3, right.compareTo(left));
+ left.set(0, new Text("123"));
+ right.set(0, new Text("1"));
+ assertEquals(-3, left.compareTo(right));
+ assertEquals(3, right.compareTo(left));
+ }
}
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/mapreduce/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/java/mapreduce/src/test/resources/log4j.properties b/java/mapreduce/src/test/resources/log4j.properties
new file mode 100644
index 0000000..67e071d
--- /dev/null
+++ b/java/mapreduce/src/test/resources/log4j.properties
@@ -0,0 +1,5 @@
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%p\t%d{ISO8601}\t%r\t%c\t[%t]\t%m%n
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/pom.xml
----------------------------------------------------------------------
diff --git a/java/pom.xml b/java/pom.xml
index 008c9bf..9941dee 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -92,6 +92,7 @@
<TZ>US/Pacific</TZ>
<LANG>en_US.UTF-8</LANG>
</environmentVariables>
+ <failIfNoTests>false</failIfNoTests>
<systemPropertyVariables>
<test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
</systemPropertyVariables>
http://git-wip-us.apache.org/repos/asf/orc/blob/545fe371/java/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/java/storage-api/pom.xml b/java/storage-api/pom.xml
index 46f534f..46e9c9e 100644
--- a/java/storage-api/pom.xml
+++ b/java/storage-api/pom.xml
@@ -73,6 +73,18 @@
<target>1.7</target>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <reuseForks>false</reuseForks>
+ <argLine>-Xmx2048m -XX:MaxPermSize=512m</argLine>
+ <failIfNoTests>false</failIfNoTests>
+ <systemPropertyVariables>
+ <test.tmp.dir>${test.tmp.dir}</test.tmp.dir>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
</plugins>
</build>