You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by to...@apache.org on 2014/11/03 15:12:17 UTC
git commit: PARQUET-121: Allow Parquet to build with Java 8
Repository: incubator-parquet-mr
Updated Branches:
refs/heads/master a29815abf -> f1da5e927
PARQUET-121: Allow Parquet to build with Java 8
There are test failures running with Java 8 due to http://openjdk.java.net/jeps/180 which changed retrieval order for HashMap.
Here's how I tested this:
```bash
use-java8
mvn clean install -DskipTests -Dmaven.javadoc.skip=true
mvn test
mvn test -P hadoop-2
```
I also compiled the main code with Java 7 (target=1.6 bytecode), and compiled the tests with Java 8, and ran them with Java 8. The idea here is to simulate users who want to run Parquet with JRE 8.
```bash
use-java7
mvn clean install -DskipTests -Dmaven.javadoc.skip=true
use-java8
find . -name test-classes | grep target/test-classes | grep -v 'parquet-scrooge' | xargs rm -rf
mvn test -DtargetJavaVersion=1.8 -Dmaven.main.skip=true -Dscala.maven.test.skip=true
```
A couple of notes about this:
* The targetJavaVersion property is used since other Hadoop projects use the same name.
* I couldn’t get parquet-scrooge to compile with target=1.8, which is why I introduced scala.maven.test.skip (and updated scala-maven-plugin to the latest version which supports the property). Compiling with target=1.8 should be fixed in another JIRA as it looks pretty involved.
Author: Tom White <to...@cloudera.com>
Closes #77 from tomwhite/PARQUET-121-java8 and squashes the following commits:
8717e13 [Tom White] Fix tests to run under Java 8.
35ea670 [Tom White] PARQUET-121. Allow Parquet to build with Java 8.
Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/f1da5e92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/f1da5e92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/f1da5e92
Branch: refs/heads/master
Commit: f1da5e927ed18aeec1610bec67f88facd6a470e1
Parents: a29815a
Author: Tom White <to...@cloudera.com>
Authored: Mon Nov 3 14:11:03 2014 +0000
Committer: Tom White <to...@cloudera.com>
Committed: Mon Nov 3 14:11:03 2014 +0000
----------------------------------------------------------------------
.../java/parquet/pig/convert/MapConverter.java | 3 +-
.../java/parquet/pig/TestParquetStorer.java | 20 ++++++++--
.../parquet/pig/TestPigSchemaConverter.java | 7 +++-
parquet-scrooge/pom.xml | 5 ++-
.../thrift/TestParquetWriteProtocol.java | 39 +++++++++++++++++++-
pom.xml | 6 ++-
6 files changed, 69 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java b/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
index 2827239..81220e1 100644
--- a/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
+++ b/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
@@ -20,6 +20,7 @@ import java.util.AbstractSet;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -78,7 +79,7 @@ final class MapConverter extends GroupConverter {
@Override
public void end() {
- parent.add(new HashMap<String, Object>(buffer));
+ parent.add(new LinkedHashMap<String, Object>(buffer));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java b/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
index 99cd6a2..9d32226 100644
--- a/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
+++ b/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
@@ -121,16 +121,28 @@ public class TestParquetStorer {
final Schema schema = data.getSchema("out");
assertEquals(2, schema.size());
- assertEquals("a", schema.getField(0).alias);
- assertEquals("b", schema.getField(1).alias);
+ // union could be in either order
+ int ai;
+ int bi;
+ if ("a".equals(schema.getField(0).alias)) {
+ ai = 0;
+ bi = 1;
+ assertEquals("a", schema.getField(0).alias);
+ assertEquals("b", schema.getField(1).alias);
+ } else {
+ ai = 1;
+ bi = 0;
+ assertEquals("b", schema.getField(0).alias);
+ assertEquals("a", schema.getField(1).alias);
+ }
assertEquals(rows * 2, result.size());
int a = 0;
int b = 0;
for (Tuple tuple : result) {
- String fa = (String) tuple.get(0);
- String fb = (String) tuple.get(1);
+ String fa = (String) tuple.get(ai);
+ String fb = (String) tuple.get(bi);
if (fa != null) {
assertEquals("a" + a, fa);
++a;
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java b/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
index e95150f..8615602 100644
--- a/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
+++ b/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
@@ -22,11 +22,14 @@ import static parquet.pig.TupleReadSupport.getPigSchemaFromMultipleFiles;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.util.Utils;
+import org.junit.Ignore;
import org.junit.Test;
import parquet.schema.MessageType;
@@ -214,8 +217,8 @@ public class TestPigSchemaConverter {
@Test
public void testSchemaEvolution() {
- Map<String, Set<String>> map = new HashMap<String, Set<String>>();
- map.put("pig.schema", new HashSet<String>(Arrays.asList(
+ Map<String, Set<String>> map = new LinkedHashMap<String, Set<String>>();
+ map.put("pig.schema", new LinkedHashSet<String>(Arrays.asList(
"a:int, b:int, c:int, d:int, e:int, f:int",
"aa:int, aaa:int, b:int, c:int, ee:int")));
Schema result = getPigSchemaFromMultipleFiles(new MessageType("empty"), map);
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-scrooge/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml
index 341a804..035a0ac 100644
--- a/parquet-scrooge/pom.xml
+++ b/parquet-scrooge/pom.xml
@@ -128,7 +128,7 @@
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
- <version>3.0.1</version>
+ <version>3.1.6</version>
<executions>
<execution>
<id>scala-compile-first</id>
@@ -146,6 +146,9 @@
</goals>
</execution>
</executions>
+ <configuration>
+ <skip>${scala.maven.test.skip}</skip>
+ </configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java b/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
index 4c211cd..bea5baf 100644
--- a/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
+++ b/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
+import org.junit.ComparisonFailure;
import thrift.test.OneOfEach;
import org.apache.pig.data.Tuple;
@@ -95,12 +96,48 @@ public class TestParquetWriteProtocol {
"endField(names, 1)",
"endMessage()"
};
+ String[] expectationsAlt = {
+ "startMessage()",
+ "startField(name, 0)",
+ "addBinary(map_name)",
+ "endField(name, 0)",
+ "startField(names, 1)",
+ "startGroup()",
+ "startField(map, 0)",
+ "startGroup()",
+ "startField(key, 0)",
+ "addBinary(foo2)",
+ "endField(key, 0)",
+ "startField(value, 1)",
+ "addBinary(bar2)",
+ "endField(value, 1)",
+ "endGroup()",
+ "startGroup()",
+ "startField(key, 0)",
+ "addBinary(foo)",
+ "endField(key, 0)",
+ "startField(value, 1)",
+ "addBinary(bar)",
+ "endField(value, 1)",
+ "endGroup()",
+ "endField(map, 0)",
+ "endGroup()",
+ "endField(names, 1)",
+ "endMessage()"
+ };
final Map<String, String> map = new TreeMap<String, String>();
map.put("foo", "bar");
map.put("foo2", "bar2");
TestMap testMap = new TestMap("map_name", map);
- validatePig(expectations, testMap);
+ try {
+ validatePig(expectations, testMap);
+ } catch (ComparisonFailure e) {
+ // This can happen despite using a stable TreeMap, since ThriftToPig#toPigMap
+ // in com.twitter.elephantbird.pig.util creates a HashMap.
+ // So we test with the map elements in reverse order
+ validatePig(expectationsAlt, testMap);
+ }
validateThrift(expectations, testMap);
}
http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1293a5d..95bacea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,8 +75,9 @@
</pluginRepositories>
<properties>
+ <targetJavaVersion>1.6</targetJavaVersion>
<maven.compiler.source>1.6</maven.compiler.source>
- <maven.compiler.target>1.6</maven.compiler.target>
+ <maven.compiler.target>${targetJavaVersion}</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<github.global.server>github</github.global.server>
<maven-jar-plugin.version>2.4</maven-jar-plugin.version>
@@ -93,6 +94,7 @@
<scala.version>2.10.4</scala.version>
<!-- scala.binary.version is used for projects that fetch dependencies that are in scala -->
<scala.binary.version>2.10</scala.binary.version>
+ <scala.maven.test.skip>false</scala.maven.test.skip>
<pig.version>0.11.1</pig.version>
<pig.classifier></pig.classifier>
</properties>
@@ -318,7 +320,7 @@
<configuration>
<argLine>-XX:MaxPermSize=256m</argLine>
<source>1.6</source>
- <target>1.6</target>
+ <target>${targetJavaVersion}</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>