You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by he...@apache.org on 2011/07/21 05:45:25 UTC
svn commit: r1149027 - in /hive/trunk/serde/src:
java/org/apache/hadoop/hive/serde2/objectinspector/
test/org/apache/hadoop/hive/serde2/objectinspector/
Author: heyongqiang
Date: Thu Jul 21 03:45:23 2011
New Revision: 1149027
URL: http://svn.apache.org/viewvc?rev=1149027&view=rev
Log:
HIVE-2209:add support for map comparision in serde layer (Krishna Kumar via He Yongqiang)
Added:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/CrossMapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+/*
+ * The equality is implemented fully, the greater-than/less-than
+ * values do not implement a transitive relation.
+ */
+
+public class CrossMapEqualComparer implements MapEqualComparer {
+ @Override
+ public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) {
+ int mapsize1 = moi1.getMapSize(o1);
+ int mapsize2 = moi2.getMapSize(o2);
+ if (mapsize1 != mapsize2) {
+ return mapsize1 - mapsize2;
+ }
+ ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector();
+ ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector();
+
+ ObjectInspector mvoi1 = moi1.getMapValueObjectInspector();
+ ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
+
+ Map<?, ?> map1 = moi1.getMap(o1);
+ Map<?, ?> map2 = moi2.getMap(o2);
+ for (Object mk1 : map1.keySet()) {
+ boolean notFound = true;
+ for (Object mk2 : map2.keySet()) {
+ int rc = ObjectInspectorUtils.compare(mk1, mkoi1, mk2, mkoi2, this);
+ if (rc != 0) {
+ continue;
+ }
+ notFound = false;
+ Object mv1 = map1.get(mk1);
+ Object mv2 = map2.get(mk2);
+ rc = ObjectInspectorUtils.compare(mv1, mvoi1, mv2, mvoi2, this);
+ if (rc != 0) {
+ return rc;
+ } else {
+ break;
+ }
+ }
+ if (notFound) {
+ return 1;
+ }
+ }
+ return 0;
+ }
+
+}
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/FullMapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.Map;
+import java.util.TreeMap;
+
+/*
+ * The equality is implemented fully, the implementation sorts the maps
+ * by their keys to provide a transitive compare.
+ */
+
+public class FullMapEqualComparer implements MapEqualComparer {
+
+ private static class MapKeyComparator implements Comparator<Object> {
+
+ private ObjectInspector oi;
+
+ MapKeyComparator(ObjectInspector oi) {
+ this.oi = oi;
+ }
+
+ @Override
+ public int compare(Object o1, Object o2) {
+ return ObjectInspectorUtils.compare(o1, oi, o2, oi);
+ }
+ }
+
+ @Override
+ public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) {
+ int mapsize1 = moi1.getMapSize(o1);
+ int mapsize2 = moi2.getMapSize(o2);
+ if (mapsize1 != mapsize2) {
+ return mapsize1 - mapsize2;
+ }
+ ObjectInspector mkoi1 = moi1.getMapKeyObjectInspector();
+ ObjectInspector mkoi2 = moi2.getMapKeyObjectInspector();
+
+ ObjectInspector mvoi1 = moi1.getMapValueObjectInspector();
+ ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
+
+ Map<?, ?> map1 = moi1.getMap(o1);
+ Map<?, ?> map2 = moi2.getMap(o2);
+
+ Object[] sortedMapKeys1 = map1.keySet().toArray();
+ Arrays.sort(sortedMapKeys1, new MapKeyComparator(mkoi1));
+
+ Object[] sortedMapKeys2 = map2.keySet().toArray();
+ Arrays.sort(sortedMapKeys2, new MapKeyComparator(mkoi2));
+
+ for (int i = 0; i < mapsize1; ++i) {
+ Object mk1 = sortedMapKeys1[i];
+ Object mk2 = sortedMapKeys2[i];
+ int rc = ObjectInspectorUtils.compare(mk1, mkoi1, mk2, mkoi2, this);
+ if (rc != 0) {
+ return rc;
+ }
+ Object mv1 = map1.get(mk1);
+ Object mv2 = map2.get(mk2);
+ rc = ObjectInspectorUtils.compare(mv1, mvoi1, mv2, mvoi2, this);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+ return 0;
+ }
+
+}
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+public interface MapEqualComparer {
+ /*
+ * Compare the two map objects for equality.
+ */
+ public int compare(Object o1, MapObjectInspector moi1,
+ Object o2, MapObjectInspector moi2);
+}
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java?rev=1149027&r1=1149026&r2=1149027&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java Thu Jul 21 03:45:23 2011
@@ -515,6 +515,14 @@ public final class ObjectInspectorUtils
*/
public static int compare(Object o1, ObjectInspector oi1, Object o2,
ObjectInspector oi2) {
+ return compare(o1, oi1, o2, oi2, null);
+ }
+
+ /**
+ * Compare two objects with their respective ObjectInspectors.
+ */
+ public static int compare(Object o1, ObjectInspector oi1, Object o2,
+ ObjectInspector oi2, MapEqualComparer mapEqualComparer) {
if (oi1.getCategory() != oi2.getCategory()) {
return oi1.getCategory().compareTo(oi2.getCategory());
}
@@ -599,7 +607,8 @@ public final class ObjectInspectorUtils
for (int i = 0; i < minimum; i++) {
int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1
.get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2,
- fields2.get(i)), fields2.get(i).getFieldObjectInspector());
+ fields2.get(i)), fields2.get(i).getFieldObjectInspector(),
+ mapEqualComparer);
if (r != 0) {
return r;
}
@@ -613,7 +622,8 @@ public final class ObjectInspectorUtils
for (int i = 0; i < minimum; i++) {
int r = compare(loi1.getListElement(o1, i), loi1
.getListElementObjectInspector(), loi2.getListElement(o2, i), loi2
- .getListElementObjectInspector());
+ .getListElementObjectInspector(),
+ mapEqualComparer);
if (r != 0) {
return r;
}
@@ -621,7 +631,11 @@ public final class ObjectInspectorUtils
return loi1.getListLength(o1) - loi2.getListLength(o2);
}
case MAP: {
- throw new RuntimeException("Compare on map type not supported!");
+ if (mapEqualComparer == null) {
+ throw new RuntimeException("Compare on map type not supported!");
+ } else {
+ return mapEqualComparer.compare(o1, (MapObjectInspector)oi1, o2, (MapObjectInspector)oi2);
+ }
}
case UNION: {
UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
@@ -633,7 +647,8 @@ public final class ObjectInspectorUtils
}
return compare(uoi1.getField(o1),
uoi1.getObjectInspectors().get(tag1),
- uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2));
+ uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2),
+ mapEqualComparer);
}
default:
throw new RuntimeException("Compare on unknown type: "
Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/SimpleMapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+
+/*
+ * Assumes that a getMapValueElement on object2 will work with a key from
+ * object1. The equality is implemented fully, the greater-than/less-than
+ * values do not implement a transitive relation.
+ */
+public class SimpleMapEqualComparer implements MapEqualComparer {
+
+ @Override
+ public int compare(Object o1, MapObjectInspector moi1, Object o2, MapObjectInspector moi2) {
+ int mapsize1 = moi1.getMapSize(o1);
+ int mapsize2 = moi2.getMapSize(o2);
+ if (mapsize1 != mapsize2) {
+ return mapsize1 - mapsize2;
+ }
+ ObjectInspector mvoi1 = moi1.getMapValueObjectInspector();
+ ObjectInspector mvoi2 = moi2.getMapValueObjectInspector();
+ Map<?, ?> map1 = moi1.getMap(o1);
+ for (Object mk1: map1.keySet()) {
+ int rc = ObjectInspectorUtils.compare(moi1.getMapValueElement(o1, mk1), mvoi1,
+ moi2.getMapValueElement(o2, mk1), mvoi2, this);
+ if (rc != 0) {
+ return rc;
+ }
+ }
+ return 0;
+ }
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestCrossMapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class TestCrossMapEqualComparer extends TestCase {
+
+ public static class IntegerStringMapHolder {
+ Map<Integer, String> mMap;
+
+ public IntegerStringMapHolder() {
+ mMap = new TreeMap<Integer, String>();
+ }
+ }
+
+ public void testSameType() {
+ // empty maps
+ IntegerStringMapHolder o1 = new IntegerStringMapHolder();
+ IntegerStringMapHolder o2 = new IntegerStringMapHolder();
+ ObjectInspector oi1 = ObjectInspectorFactory
+ .getReflectionObjectInspector(IntegerStringMapHolder.class, ObjectInspectorOptions.JAVA);
+ int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer());
+ assertEquals(0, rc);
+
+ // equal maps
+ o1.mMap.put(42, "The answer to Life, Universe And Everything");
+ o2.mMap.put(42, "The answer to Life, Universe And Everything");
+
+ o1.mMap.put(1729, "A taxi cab number");
+ o2.mMap.put(1729, "A taxi cab number");
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer());
+ assertEquals(0, rc);
+
+ // unequal maps
+ o2.mMap.put(1729, "Hardy-Ramanujan Number");
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new CrossMapEqualComparer());
+ assertFalse(0 == rc);
+ }
+
+ public static class TextStringMapHolder {
+ Map<Text, String> mMap;
+
+ public TextStringMapHolder() {
+ mMap = new TreeMap<Text, String>();
+ }
+ }
+
+ Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1,
+ LazySimpleSerDe serde,
+ SerDeParameters serdeParams) throws IOException, SerDeException {
+ ByteStream.Output serializeStream = new ByteStream.Output();
+ LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+ .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+ .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+ .getNeedsEscape());
+ Text t = new Text(serializeStream.toByteArray());
+ return serde.deserialize(t);
+ }
+
+ public void testCompatibleType() throws SerDeException, IOException {
+ // empty maps
+ TextStringMapHolder o1 = new TextStringMapHolder();
+ StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);
+
+ LazySimpleSerDe serde = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+ Properties tbl = new Properties();
+ tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+ tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+ SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+ LazySimpleSerDe.class.getName());
+ serde.initialize(conf, tbl);
+ ObjectInspector oi2 = serde.getObjectInspector();
+
+ Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+ int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+ assertEquals(0, rc);
+
+ // equal maps
+ o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything");
+ o1.mMap.put(new Text("1729"), "A taxi cab number");
+ o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+ assertEquals(0, rc);
+
+ // unequal maps
+ o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number");
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+ assertFalse(0 == rc);
+ }
+
+ public static class StringTextMapHolder {
+ Map<String, Text> mMap;
+
+ public StringTextMapHolder() {
+ mMap = new TreeMap<String, Text>();
+ }
+ }
+
+ Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1,
+ LazySimpleSerDe serde,
+ SerDeParameters serdeParams) throws IOException, SerDeException {
+ ByteStream.Output serializeStream = new ByteStream.Output();
+ LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+ .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+ .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+ .getNeedsEscape());
+ Text t = new Text(serializeStream.toByteArray());
+ return serde.deserialize(t);
+ }
+
+ public void testIncompatibleType() throws SerDeException, IOException {
+ // empty maps
+ StringTextMapHolder o1 = new StringTextMapHolder();
+ StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);
+
+ LazySimpleSerDe serde = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+ Properties tbl = new Properties();
+ tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+ tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+ SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+ LazySimpleSerDe.class.getName());
+ serde.initialize(conf, tbl);
+ ObjectInspector oi2 = serde.getObjectInspector();
+
+ Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+ int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+ assertEquals(0, rc);
+
+ // equal maps
+ o1.mMap.put("42", new Text("The answer to Life, Universe And Everything"));
+ o1.mMap.put("1729", new Text("A taxi cab number"));
+ o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+ assertEquals(0, rc);
+
+ // unequal maps
+ o1.mMap.put("1729", new Text("Hardy-Ramanujan Number"));
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
+ assertFalse(0 == rc);
+
+ }
+
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestFullMapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.util.Map;
+import java.util.TreeMap;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+
+public class TestFullMapEqualComparer extends TestCase {
+
+ public static class IntegerIntegerMapHolder {
+ Map<Integer, Integer> mMap;
+
+ public IntegerIntegerMapHolder() {
+ mMap = new TreeMap<Integer, Integer>();
+ }
+ }
+
+ public void testAntiSymmetry() {
+ IntegerIntegerMapHolder o1 = new IntegerIntegerMapHolder();
+ IntegerIntegerMapHolder o2 = new IntegerIntegerMapHolder();
+
+ ObjectInspector oi = ObjectInspectorFactory
+ .getReflectionObjectInspector(IntegerIntegerMapHolder.class, ObjectInspectorOptions.JAVA);
+
+ o1.mMap.put(1, 1);
+
+ o2.mMap.put(0, 99);
+
+ { // not anti-symmetric
+ int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new SimpleMapEqualComparer());
+ assertTrue(rc12 > 0);
+ int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new SimpleMapEqualComparer());
+ assertTrue(rc21 > 0);
+ }
+ { // not anti-symmetric
+ int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new CrossMapEqualComparer());
+ assertTrue(rc12 > 0);
+ int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new CrossMapEqualComparer());
+ assertTrue(rc21 > 0);
+ }
+ {// anti-symmetric
+ int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new FullMapEqualComparer());
+ assertTrue(rc12 > 0);
+ int rc21 = ObjectInspectorUtils.compare(o2, oi, o1, oi, new FullMapEqualComparer());
+ assertTrue(rc21 < 0);
+ }
+
+ }
+
+ public void testTransitivity() {
+ IntegerIntegerMapHolder o1 = new IntegerIntegerMapHolder();
+ IntegerIntegerMapHolder o2 = new IntegerIntegerMapHolder();
+ IntegerIntegerMapHolder o3 = new IntegerIntegerMapHolder();
+
+ ObjectInspector oi = ObjectInspectorFactory
+ .getReflectionObjectInspector(IntegerIntegerMapHolder.class, ObjectInspectorOptions.JAVA);
+
+ o1.mMap.put(1, 1);
+ o1.mMap.put(99, 99);
+
+ o2.mMap.put(0, 99);
+ o2.mMap.put(99, 99);
+
+ o3.mMap.put(0, 1);
+ o3.mMap.put(1, 99);
+
+ { // non-transitive
+ int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new SimpleMapEqualComparer());
+ assertTrue(rc12 > 0);
+ int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new SimpleMapEqualComparer());
+ assertTrue(rc23 > 0);
+ int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new SimpleMapEqualComparer());
+ assertTrue(rc13 < 0);
+ }
+ { // non-transitive
+ int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new CrossMapEqualComparer());
+ assertTrue(rc12 > 0);
+ int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new CrossMapEqualComparer());
+ assertTrue(rc23 > 0);
+ int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new CrossMapEqualComparer());
+ assertTrue(rc13 < 0);
+ }
+ {// transitive
+ int rc12 = ObjectInspectorUtils.compare(o1, oi, o2, oi, new FullMapEqualComparer());
+ assertTrue(rc12 > 0);
+ int rc23 = ObjectInspectorUtils.compare(o2, oi, o3, oi, new FullMapEqualComparer());
+ assertTrue(rc23 > 0);
+ int rc13 = ObjectInspectorUtils.compare(o1, oi, o3, oi, new FullMapEqualComparer());
+ assertTrue(rc13 > 0);
+ }
+ }
+
+}
Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java?rev=1149027&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestSimpleMapEqualComparer.java Thu Jul 21 03:45:23 2011
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.io.Text;
+
+import junit.framework.TestCase;
+
+public class TestSimpleMapEqualComparer extends TestCase {
+
+ public static class IntegerStringMapHolder {
+ Map<Integer, String> mMap;
+
+ public IntegerStringMapHolder() {
+ mMap = new TreeMap<Integer, String>();
+ }
+ }
+
+ public void testSameType() {
+ // empty maps
+ IntegerStringMapHolder o1 = new IntegerStringMapHolder();
+ IntegerStringMapHolder o2 = new IntegerStringMapHolder();
+ ObjectInspector oi1 = ObjectInspectorFactory
+ .getReflectionObjectInspector(IntegerStringMapHolder.class, ObjectInspectorOptions.JAVA);
+ int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer());
+ assertEquals(0, rc);
+
+ // equal maps
+ o1.mMap.put(42, "The answer to Life, Universe And Everything");
+ o2.mMap.put(42, "The answer to Life, Universe And Everything");
+
+ o1.mMap.put(1729, "A taxi cab number");
+ o2.mMap.put(1729, "A taxi cab number");
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer());
+ assertEquals(0, rc);
+
+ // unequal maps
+ o2.mMap.put(1729, "Hardy-Ramanujan Number");
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi1, new SimpleMapEqualComparer());
+ assertFalse(0 == rc);
+ }
+
+ public static class TextStringMapHolder {
+ Map<Text, String> mMap;
+
+ public TextStringMapHolder() {
+ mMap = new TreeMap<Text, String>();
+ }
+ }
+
+ Object serializeAndDeserialize(TextStringMapHolder o1, StructObjectInspector oi1,
+ LazySimpleSerDe serde,
+ SerDeParameters serdeParams) throws IOException, SerDeException {
+ ByteStream.Output serializeStream = new ByteStream.Output();
+ LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+ .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+ .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+ .getNeedsEscape());
+ Text t = new Text(serializeStream.toByteArray());
+ return serde.deserialize(t);
+ }
+
+ public void testCompatibleType() throws SerDeException, IOException {
+ // empty maps
+ TextStringMapHolder o1 = new TextStringMapHolder();
+ StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);
+
+ LazySimpleSerDe serde = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+ Properties tbl = new Properties();
+ tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+ tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+ SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+ LazySimpleSerDe.class.getName());
+ serde.initialize(conf, tbl);
+ ObjectInspector oi2 = serde.getObjectInspector();
+
+ Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+ int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+ assertEquals(0, rc);
+
+ // equal maps
+ o1.mMap.put(new Text("42"), "The answer to Life, Universe And Everything");
+ o1.mMap.put(new Text("1729"), "A taxi cab number");
+ o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+ assertEquals(0, rc);
+
+ // unequal maps
+ o1.mMap.put(new Text("1729"), "Hardy-Ramanujan Number");
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+ assertFalse(0 == rc);
+ }
+
+ public static class StringTextMapHolder {
+ Map<String, Text> mMap;
+
+ public StringTextMapHolder() {
+ mMap = new TreeMap<String, Text>();
+ }
+ }
+
+ Object serializeAndDeserialize(StringTextMapHolder o1, StructObjectInspector oi1,
+ LazySimpleSerDe serde,
+ SerDeParameters serdeParams) throws IOException, SerDeException {
+ ByteStream.Output serializeStream = new ByteStream.Output();
+ LazySimpleSerDe.serialize(serializeStream, o1, oi1, serdeParams
+ .getSeparators(), 0, serdeParams.getNullSequence(), serdeParams
+ .isEscaped(), serdeParams.getEscapeChar(), serdeParams
+ .getNeedsEscape());
+ Text t = new Text(serializeStream.toByteArray());
+ return serde.deserialize(t);
+ }
+
+ public void testIncompatibleType() throws SerDeException, IOException {
+ // empty maps
+ StringTextMapHolder o1 = new StringTextMapHolder();
+ StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
+ .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);
+
+ LazySimpleSerDe serde = new LazySimpleSerDe();
+ Configuration conf = new Configuration();
+ Properties tbl = new Properties();
+ tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
+ tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
+ SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
+ LazySimpleSerDe.class.getName());
+ serde.initialize(conf, tbl);
+ ObjectInspector oi2 = serde.getObjectInspector();
+
+ Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+
+ int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+ assertEquals(0, rc);
+
+ // equal maps
+ o1.mMap.put("42", new Text("The answer to Life, Universe And Everything"));
+ o1.mMap.put("1729", new Text("A taxi cab number"));
+ o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);
+ rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
+ assertFalse(0 == rc);
+ }
+
+}