You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ga...@apache.org on 2021/04/26 13:28:45 UTC
[parquet-mr] branch parquet-1.12.x created (now 76f3594)
This is an automated email from the ASF dual-hosted git repository.
gabor pushed a change to branch parquet-1.12.x
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git.
at 76f3594 PARQUET-2027: Fix calculating directory offset for merge (#896)
This branch includes the following new commits:
new 4cc5baf Prepare next development iteration
new 76f3594 PARQUET-2027: Fix calculating directory offset for merge (#896)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[parquet-mr] 01/02: Prepare next development iteration
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gabor pushed a commit to branch parquet-1.12.x
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
commit 4cc5baf57420d628f017fb9dd0fd4d3c2b5e2749
Author: Gabor Szadovszky <ga...@apache.org>
AuthorDate: Wed Jan 22 11:22:42 2020 +0100
Prepare next development iteration
---
parquet-arrow/pom.xml | 2 +-
parquet-avro/pom.xml | 2 +-
parquet-benchmarks/pom.xml | 2 +-
parquet-cascading-deprecated/pom.xml | 2 +-
parquet-cascading3-deprecated/pom.xml | 2 +-
parquet-cli/pom.xml | 2 +-
parquet-column/pom.xml | 2 +-
parquet-common/pom.xml | 2 +-
parquet-encoding/pom.xml | 2 +-
parquet-format-structures/pom.xml | 2 +-
parquet-generator/pom.xml | 2 +-
parquet-hadoop-bundle/pom.xml | 2 +-
parquet-hadoop/pom.xml | 2 +-
parquet-jackson/pom.xml | 2 +-
parquet-pig-bundle/pom.xml | 2 +-
parquet-pig/pom.xml | 2 +-
parquet-protobuf/pom.xml | 2 +-
parquet-scala/pom.xml | 2 +-
parquet-scrooge-deprecated/pom.xml | 2 +-
parquet-thrift/pom.xml | 2 +-
parquet-tools-deprecated/pom.xml | 2 +-
pom.xml | 4 ++--
22 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/parquet-arrow/pom.xml b/parquet-arrow/pom.xml
index dd20e9b..b7a9481 100644
--- a/parquet-arrow/pom.xml
+++ b/parquet-arrow/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-avro/pom.xml b/parquet-avro/pom.xml
index c53071d..e7b8e6b 100644
--- a/parquet-avro/pom.xml
+++ b/parquet-avro/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-benchmarks/pom.xml b/parquet-benchmarks/pom.xml
index 904e4e9..ae16bfb 100644
--- a/parquet-benchmarks/pom.xml
+++ b/parquet-benchmarks/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-cascading-deprecated/pom.xml b/parquet-cascading-deprecated/pom.xml
index 08f9d47..1ceb23a 100644
--- a/parquet-cascading-deprecated/pom.xml
+++ b/parquet-cascading-deprecated/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-cascading3-deprecated/pom.xml b/parquet-cascading3-deprecated/pom.xml
index 9c9d638..e67f0fa 100644
--- a/parquet-cascading3-deprecated/pom.xml
+++ b/parquet-cascading3-deprecated/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-cli/pom.xml b/parquet-cli/pom.xml
index 379e81b..d886951 100644
--- a/parquet-cli/pom.xml
+++ b/parquet-cli/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-column/pom.xml b/parquet-column/pom.xml
index 30bd635..cb0c185 100644
--- a/parquet-column/pom.xml
+++ b/parquet-column/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-common/pom.xml b/parquet-common/pom.xml
index c4b944b..a368446 100644
--- a/parquet-common/pom.xml
+++ b/parquet-common/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-encoding/pom.xml b/parquet-encoding/pom.xml
index ac53aa6..f1f7d9a 100644
--- a/parquet-encoding/pom.xml
+++ b/parquet-encoding/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-format-structures/pom.xml b/parquet-format-structures/pom.xml
index d0aa0ee..4fb990b 100644
--- a/parquet-format-structures/pom.xml
+++ b/parquet-format-structures/pom.xml
@@ -24,7 +24,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<artifactId>parquet-format-structures</artifactId>
diff --git a/parquet-generator/pom.xml b/parquet-generator/pom.xml
index 694fbf5..1126c8e 100644
--- a/parquet-generator/pom.xml
+++ b/parquet-generator/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-hadoop-bundle/pom.xml b/parquet-hadoop-bundle/pom.xml
index 6e1950a..4212ae0 100644
--- a/parquet-hadoop-bundle/pom.xml
+++ b/parquet-hadoop-bundle/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-hadoop/pom.xml b/parquet-hadoop/pom.xml
index 5e61b31..c11c560 100644
--- a/parquet-hadoop/pom.xml
+++ b/parquet-hadoop/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-jackson/pom.xml b/parquet-jackson/pom.xml
index 3810886..c12e247 100644
--- a/parquet-jackson/pom.xml
+++ b/parquet-jackson/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-pig-bundle/pom.xml b/parquet-pig-bundle/pom.xml
index a86f6ee..eede369 100644
--- a/parquet-pig-bundle/pom.xml
+++ b/parquet-pig-bundle/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml
index ac109b5..a725184 100644
--- a/parquet-pig/pom.xml
+++ b/parquet-pig/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-protobuf/pom.xml b/parquet-protobuf/pom.xml
index e333ff7..990d5e9 100644
--- a/parquet-protobuf/pom.xml
+++ b/parquet-protobuf/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-scala/pom.xml b/parquet-scala/pom.xml
index 482dc07..5f5f759 100644
--- a/parquet-scala/pom.xml
+++ b/parquet-scala/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-scrooge-deprecated/pom.xml b/parquet-scrooge-deprecated/pom.xml
index 9c762e0..96f8ecd 100644
--- a/parquet-scrooge-deprecated/pom.xml
+++ b/parquet-scrooge-deprecated/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml
index a9e216d..51e4aa3 100644
--- a/parquet-thrift/pom.xml
+++ b/parquet-thrift/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/parquet-tools-deprecated/pom.xml b/parquet-tools-deprecated/pom.xml
index 3290b11..5343d25 100644
--- a/parquet-tools-deprecated/pom.xml
+++ b/parquet-tools-deprecated/pom.xml
@@ -21,7 +21,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
<relativePath>../pom.xml</relativePath>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index 7445e1c..30b65ab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -9,7 +9,7 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet</artifactId>
- <version>1.12.0</version>
+ <version>1.12.1-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Parquet MR</name>
@@ -20,7 +20,7 @@
<connection>scm:git:git@github.com:apache/parquet-mr.git</connection>
<url>scm:git:git@github.com:apache/parquet-mr.git</url>
<developerConnection>scm:git:git@github.com:apache/parquet-mr.git</developerConnection>
- <tag>apache-parquet-1.12.0-rc4</tag>
+ <tag>HEAD</tag>
</scm>
<licenses>
[parquet-mr] 02/02: PARQUET-2027: Fix calculating directory offset
for merge (#896)
Posted by ga...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gabor pushed a commit to branch parquet-1.12.x
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git
commit 76f35944d5994817a1f60fac5af791e7f515d3cf
Author: Gabor Szadovszky <ga...@apache.org>
AuthorDate: Fri Apr 23 19:05:34 2021 +0200
PARQUET-2027: Fix calculating directory offset for merge (#896)
(cherry picked from commit 2ce35c73746cf091ed223da150daefd323a9ad3a)
---
.../java/org/apache/parquet/hadoop/Offsets.java | 8 ++-
.../hadoop/TestParquetWriterAppendBlocks.java | 63 ++++++++++++++++++++-
.../src/test/resources/test-append_1.parquet | Bin 0 -> 7375 bytes
.../src/test/resources/test-append_2.parquet | Bin 0 -> 7374 bytes
4 files changed, 65 insertions(+), 6 deletions(-)
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/Offsets.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/Offsets.java
index fa25943..2edc585 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/Offsets.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/Offsets.java
@@ -55,7 +55,7 @@ class Offsets {
* (0 cannot be a valid offset because of the MAGIC bytes)
* - The firstDataPageOffset might point to the dictionary page
*/
- dictionaryPageSize = readDictionaryPageSize(input, newChunkStart);
+ dictionaryPageSize = readDictionaryPageSize(input, chunk);
} else {
dictionaryPageSize = chunk.getFirstDataPageOffset() - chunk.getDictionaryPageOffset();
}
@@ -68,12 +68,14 @@ class Offsets {
return new Offsets(firstDataPageOffset, dictionaryPageOffset);
}
- private static long readDictionaryPageSize(SeekableInputStream in, long pos) throws IOException {
+ private static long readDictionaryPageSize(SeekableInputStream in, ColumnChunkMetaData chunk) throws IOException {
long origPos = -1;
try {
origPos = in.getPos();
+ in.seek(chunk.getStartingPos());
+ long headerStart = in.getPos();
PageHeader header = Util.readPageHeader(in);
- long headerSize = in.getPos() - origPos;
+ long headerSize = in.getPos() - headerStart;
return headerSize + header.getCompressed_page_size();
} finally {
if (origPos != -1) {
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetWriterAppendBlocks.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetWriterAppendBlocks.java
index bda5333..82d48f4 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetWriterAppendBlocks.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestParquetWriterAppendBlocks.java
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -38,6 +38,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
@@ -68,6 +69,17 @@ public class TestParquetWriterAppendBlocks {
public static final SimpleGroupFactory GROUP_FACTORY =
new SimpleGroupFactory(FILE_SCHEMA);
+ private static final Path STATIC_FILE_1 = createPathFromCP("/test-append_1.parquet");
+ private static final Path STATIC_FILE_2 = createPathFromCP("/test-append_2.parquet");
+
+ private static Path createPathFromCP(String path) {
+ try {
+ return new Path(TestParquetWriterAppendBlocks.class.getResource(path).toURI());
+ } catch (URISyntaxException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public Path file1;
public List<Group> file1content = new ArrayList<Group>();
public Path file2;
@@ -134,6 +146,51 @@ public class TestParquetWriterAppendBlocks {
Assert.assertEquals("All records should be present", 0, expected.size());
}
+ /**
+ * This test is similar to {@link #testBasicBehavior()} only that it uses static files generated by a previous release
+ * (1.11.1). This test is to validate the fix of PARQUET-2027.
+ */
+ @Test
+ public void testBasicBehaviorWithStaticFiles() throws IOException {
+ List<Group> expected = new ArrayList<>();
+ readAll(STATIC_FILE_1, expected);
+ readAll(STATIC_FILE_2, expected);
+
+ Path combinedFile = newTemp();
+ ParquetFileWriter writer = new ParquetFileWriter(
+ CONF, FILE_SCHEMA, combinedFile);
+ writer.start();
+ writer.appendFile(CONF, STATIC_FILE_1);
+ writer.appendFile(CONF, STATIC_FILE_2);
+ writer.end(EMPTY_METADATA);
+
+ try (ParquetReader<Group> reader = ParquetReader
+ .builder(new GroupReadSupport(), combinedFile)
+ .build()) {
+
+ for (Group expectedNext : expected) {
+ Group next = reader.read();
+ // check each value; equals is not supported for simple records
+ Assert.assertEquals("Each id should match",
+ expectedNext.getInteger("id", 0), next.getInteger("id", 0));
+ Assert.assertEquals("Each string should match",
+ expectedNext.getString("string", 0), next.getString("string", 0));
+ }
+ Assert.assertNull("No extra records should be present", reader.read());
+ }
+
+ }
+
+ private void readAll(Path file, List<Group> values) throws IOException {
+ try (ParquetReader<Group> reader = ParquetReader
+ .builder(new GroupReadSupport(), file)
+ .build()) {
+ for (Group g = reader.read(); g != null; g = reader.read()) {
+ values.add(g);
+ }
+ }
+ }
+
@Test
public void testMergedMetadata() throws IOException {
Path combinedFile = newTemp();
diff --git a/parquet-hadoop/src/test/resources/test-append_1.parquet b/parquet-hadoop/src/test/resources/test-append_1.parquet
new file mode 100644
index 0000000..a255f86
Binary files /dev/null and b/parquet-hadoop/src/test/resources/test-append_1.parquet differ
diff --git a/parquet-hadoop/src/test/resources/test-append_2.parquet b/parquet-hadoop/src/test/resources/test-append_2.parquet
new file mode 100644
index 0000000..3081f89
Binary files /dev/null and b/parquet-hadoop/src/test/resources/test-append_2.parquet differ