You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by al...@apache.org on 2015/07/28 23:55:17 UTC
parquet-mr git commit: PARQUET-342: Updates to be Java 6 compatible
Repository: parquet-mr
Updated Branches:
refs/heads/master 83406b73e -> 454fc3655
PARQUET-342: Updates to be Java 6 compatible
Author: Nezih Yigitbasi <ny...@netflix.com>
Closes #248 from nezihyigitbasi/java6-fixes and squashes the following commits:
2ab2598 [Nezih Yigitbasi] Updates to be Java 6 compatible
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/454fc365
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/454fc365
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/454fc365
Branch: refs/heads/master
Commit: 454fc3655509f1f4f47ce44acaff7c1566ede108
Parents: 83406b7
Author: Nezih Yigitbasi <ny...@netflix.com>
Authored: Tue Jul 28 14:55:14 2015 -0700
Committer: Alex Levenson <al...@twitter.com>
Committed: Tue Jul 28 14:55:14 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/parquet/Files.java | 51 ++++++++++++++++++++
.../org/apache/parquet/SemanticVersion.java | 16 ++++--
.../TestInputOutputFormatWithPadding.java | 11 ++---
.../hadoop/example/TestInputOutputFormat.java | 9 ++--
4 files changed, 73 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-common/src/main/java/org/apache/parquet/Files.java
----------------------------------------------------------------------
diff --git a/parquet-common/src/main/java/org/apache/parquet/Files.java b/parquet-common/src/main/java/org/apache/parquet/Files.java
new file mode 100644
index 0000000..1d2b506
--- /dev/null
+++ b/parquet-common/src/main/java/org/apache/parquet/Files.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.parquet;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+
+//TODO: Use java.nio.file.Files when Parquet is updated to Java 7
+public final class Files {
+ private Files() { }
+
+ public static List<String> readAllLines(File file, Charset charset) throws IOException {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), charset));
+ try {
+ List<String> result = new ArrayList<String>();
+ for (;;) {
+ String line = reader.readLine();
+ if (line == null)
+ break;
+ result.add(line);
+ }
+ return result;
+ }
+ finally {
+ reader.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java
----------------------------------------------------------------------
diff --git a/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java b/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java
index d573a81..c6cb406 100644
--- a/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java
+++ b/parquet-common/src/main/java/org/apache/parquet/SemanticVersion.java
@@ -96,22 +96,30 @@ public final class SemanticVersion implements Comparable<SemanticVersion> {
public int compareTo(SemanticVersion o) {
int cmp;
- cmp = Integer.compare(major, o.major);
+ cmp = compareIntegers(major, o.major);
if (cmp != 0) {
return cmp;
}
- cmp = Integer.compare(minor, o.minor);
+ cmp = compareIntegers(minor, o.minor);
if (cmp != 0) {
return cmp;
}
- cmp = Integer.compare(patch, o.patch);
+ cmp = compareIntegers(patch, o.patch);
if (cmp != 0) {
return cmp;
}
- return Boolean.compare(o.prerelease, prerelease);
+ return compareBooleans(o.prerelease, prerelease);
+ }
+
+ int compareIntegers(int x, int y) {
+ return (x < y) ? -1 : ((x == y) ? 0 : 1);
+ }
+
+ int compareBooleans(boolean x, boolean y) {
+ return (x == y) ? 0 : (x ? 1 : -1);
}
@Override
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java
index dcb0c59..0ac9c0f 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestInputOutputFormatWithPadding.java
@@ -19,7 +19,6 @@
package org.apache.parquet.hadoop;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
@@ -28,6 +27,7 @@ import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.parquet.Files;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.format.converter.ParquetMetadataConverter;
@@ -42,13 +42,11 @@ import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.net.URI;
import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Paths;
import java.util.UUID;
import static java.lang.Thread.sleep;
@@ -66,6 +64,8 @@ public class TestInputOutputFormatWithPadding {
.required(BINARY).as(UTF8).named("char")
.named("FormatTestObject");
+ private static final Charset UTF_8 = Charset.forName("UTF-8");
+
/**
* ParquetInputFormat that will not split the input file (easier validation)
*/
@@ -179,8 +179,7 @@ public class TestInputOutputFormatWithPadding {
Assert.assertNotNull("Should find a data file", dataFile);
StringBuilder contentBuilder = new StringBuilder();
- for (String line : Files.readAllLines(
- Paths.get(dataFile.toURI()), Charset.forName("UTF-8"))) {
+ for (String line : Files.readAllLines(dataFile, UTF_8)) {
contentBuilder.append(line);
}
String reconstructed = contentBuilder.toString();
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/454fc365/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java
index 987554e..5ca041b 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java
@@ -29,7 +29,6 @@ import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.Method;
import java.nio.charset.Charset;
-import java.nio.file.Files;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -48,6 +47,7 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.parquet.Files;
import org.apache.parquet.Strings;
import org.apache.parquet.filter2.predicate.FilterApi;
import org.junit.Before;
@@ -68,6 +68,7 @@ import org.apache.parquet.schema.MessageTypeParser;
public class TestInputOutputFormat {
private static final Log LOG = Log.getLog(TestInputOutputFormat.class);
+ private static final Charset UTF_8 = Charset.forName("UTF-8");
final Path parquetPath = new Path("target/test/example/TestInputOutputFormat/parquet");
final Path inputPath = new Path("src/test/java/org/apache/parquet/hadoop/example/TestInputOutputFormat.java");
final Path outputPath = new Path("target/test/example/TestInputOutputFormat/out");
@@ -258,7 +259,7 @@ public class TestInputOutputFormat {
put(ParquetInputFormat.FILTER_PREDICATE, fpString);
}});
- List<String> lines = Files.readAllLines(new File(outputPath.toString(), "part-m-00000").toPath(), Charset.forName("UTF-8"));
+ List<String> lines = Files.readAllLines(new File(outputPath.toString(), "part-m-00000"), UTF_8);
assertTrue(lines.isEmpty());
}
@@ -276,7 +277,7 @@ public class TestInputOutputFormat {
put(ParquetInputFormat.FILTER_PREDICATE, fpString);
}});
- List<String> expected = Files.readAllLines(new File(inputPath.toString()).toPath(), Charset.forName("UTF-8"));
+ List<String> expected = Files.readAllLines(new File(inputPath.toString()), UTF_8);
// grab the lines that contain the first 500 characters (including the rest of the line past 500 characters)
int size = 0;
@@ -293,7 +294,7 @@ public class TestInputOutputFormat {
}
// put the output back into it's original format (remove the character counts / tabs)
- List<String> found = Files.readAllLines(new File(outputPath.toString(), "part-m-00000").toPath(), Charset.forName("UTF-8"));
+ List<String> found = Files.readAllLines(new File(outputPath.toString(), "part-m-00000"), UTF_8);
StringBuilder sbFound = new StringBuilder();
for (String line : found) {
sbFound.append(line.split("\t", -1)[1]);