You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by bl...@apache.org on 2015/07/02 01:33:46 UTC
parquet-mr git commit: PARQUET-320: Fix semver problems for
parquet-hadoop.
Repository: parquet-mr
Updated Branches:
refs/heads/master e3b95020f -> 9fde65345
PARQUET-320: Fix semver problems for parquet-hadoop.
Re-enables semver checks for Parquet packages by removing the parquet/** exclusion that was matching unexpected classes. This also fixes all of the semver problems that have been committed since the check started excluding all Parquet classes.
Author: Ryan Blue <bl...@apache.org>
Closes #230 from rdblue/PARQUET-320-fix-semver-issues and squashes the following commits:
a0e730d [Ryan Blue] PARQUET-320: Fix Thrift incompatibilities from ded56ffd.
ba71f3f [Ryan Blue] PARQUET-320: Fix semver problems for parquet-hadoop.
Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/9fde6534
Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/9fde6534
Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/9fde6534
Branch: refs/heads/master
Commit: 9fde65345e677256975bcecdc027649f31450a57
Parents: e3b9502
Author: Ryan Blue <bl...@apache.org>
Authored: Wed Jul 1 16:33:39 2015 -0700
Committer: Ryan Blue <bl...@apache.org>
Committed: Wed Jul 1 16:33:39 2015 -0700
----------------------------------------------------------------------
.../converter/ParquetMetadataConverter.java | 51 +++---
.../hadoop/ColumnChunkPageWriteStore.java | 6 +-
.../parquet/hadoop/ParquetFileReader.java | 16 +-
.../parquet/hadoop/ParquetFileWriter.java | 10 +-
.../converter/TestParquetMetadataConverter.java | 33 ++--
.../thrift/ThriftSchemaConvertVisitor.java | 2 +-
.../projection/amend/DefaultEventsVisitor.java | 2 +-
.../thrift/struct/CompatibilityChecker.java | 48 ++----
.../parquet/thrift/struct/ThriftType.java | 169 +++++++++++++++++--
.../thrift/projection/TestFieldsPath.java | 2 +-
pom.xml | 5 +-
11 files changed, 243 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
index 420d97e..e8b6d73 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java
@@ -72,7 +72,6 @@ import org.apache.parquet.schema.Types;
// TODO: This file has become too long!
// TODO: Lets split it up: https://issues.apache.org/jira/browse/PARQUET-310
public class ParquetMetadataConverter {
- private ParquetMetadataConverter() { }
public static final MetadataFilter NO_FILTER = new NoFilter();
public static final MetadataFilter SKIP_ROW_GROUPS = new SkipMetadataFilter();
@@ -87,7 +86,7 @@ public class ParquetMetadataConverter {
private static final ConcurrentHashMap<Set<org.apache.parquet.column.Encoding>, Set<org.apache.parquet.column.Encoding>>
cachedEncodingSets = new ConcurrentHashMap<Set<org.apache.parquet.column.Encoding>, Set<org.apache.parquet.column.Encoding>>();
- public static FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata) {
+ public FileMetaData toParquetMetadata(int currentVersion, ParquetMetadata parquetMetadata) {
List<BlockMetaData> blocks = parquetMetadata.getBlocks();
List<RowGroup> rowGroups = new ArrayList<RowGroup>();
int numRows = 0;
@@ -111,13 +110,13 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static List<SchemaElement> toParquetSchema(MessageType schema) {
+ List<SchemaElement> toParquetSchema(MessageType schema) {
List<SchemaElement> result = new ArrayList<SchemaElement>();
addToList(result, schema);
return result;
}
- private static void addToList(final List<SchemaElement> result, org.apache.parquet.schema.Type field) {
+ private void addToList(final List<SchemaElement> result, org.apache.parquet.schema.Type field) {
field.accept(new TypeVisitor() {
@Override
public void visit(PrimitiveType primitiveType) {
@@ -164,7 +163,7 @@ public class ParquetMetadataConverter {
});
}
- private static void addRowGroup(ParquetMetadata parquetMetadata, List<RowGroup> rowGroups, BlockMetaData block) {
+ private void addRowGroup(ParquetMetadata parquetMetadata, List<RowGroup> rowGroups, BlockMetaData block) {
//rowGroup.total_byte_size = ;
List<ColumnChunkMetaData> columns = block.getColumns();
List<ColumnChunk> parquetColumns = new ArrayList<ColumnChunk>();
@@ -193,7 +192,7 @@ public class ParquetMetadataConverter {
rowGroups.add(rowGroup);
}
- private static List<Encoding> toFormatEncodings(Set<org.apache.parquet.column.Encoding> encodings) {
+ private List<Encoding> toFormatEncodings(Set<org.apache.parquet.column.Encoding> encodings) {
List<Encoding> converted = new ArrayList<Encoding>(encodings.size());
for (org.apache.parquet.column.Encoding encoding : encodings) {
converted.add(getEncoding(encoding));
@@ -202,7 +201,7 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static Set<org.apache.parquet.column.Encoding> fromFormatEncodings(List<Encoding> encodings) {
+ Set<org.apache.parquet.column.Encoding> fromFormatEncodings(List<Encoding> encodings) {
Set<org.apache.parquet.column.Encoding> converted = new HashSet<org.apache.parquet.column.Encoding>();
for (Encoding encoding : encodings) {
@@ -225,11 +224,11 @@ public class ParquetMetadataConverter {
return cached;
}
- public static org.apache.parquet.column.Encoding getEncoding(Encoding encoding) {
+ public org.apache.parquet.column.Encoding getEncoding(Encoding encoding) {
return org.apache.parquet.column.Encoding.valueOf(encoding.name());
}
- public static Encoding getEncoding(org.apache.parquet.column.Encoding encoding) {
+ public Encoding getEncoding(org.apache.parquet.column.Encoding encoding) {
return Encoding.valueOf(encoding.name());
}
@@ -270,7 +269,7 @@ public class ParquetMetadataConverter {
return stats;
}
- public static PrimitiveTypeName getPrimitive(Type type) {
+ public PrimitiveTypeName getPrimitive(Type type) {
switch (type) {
case BYTE_ARRAY: // TODO: rename BINARY and remove this switch
return PrimitiveTypeName.BINARY;
@@ -294,7 +293,7 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static Type getType(PrimitiveTypeName type) {
+ Type getType(PrimitiveTypeName type) {
switch (type) {
case INT64:
return Type.INT64;
@@ -318,7 +317,7 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static OriginalType getOriginalType(ConvertedType type) {
+ OriginalType getOriginalType(ConvertedType type) {
switch (type) {
case UTF8:
return OriginalType.UTF8;
@@ -366,7 +365,7 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static ConvertedType getConvertedType(OriginalType type) {
+ ConvertedType getConvertedType(OriginalType type) {
switch (type) {
case UTF8:
return ConvertedType.UTF8;
@@ -487,7 +486,7 @@ public class ParquetMetadataConverter {
}
@Deprecated
- public static ParquetMetadata readParquetMetadata(InputStream from) throws IOException {
+ public ParquetMetadata readParquetMetadata(InputStream from) throws IOException {
return readParquetMetadata(from, NO_FILTER);
}
@@ -524,7 +523,7 @@ public class ParquetMetadataConverter {
return offset;
}
- public static ParquetMetadata readParquetMetadata(final InputStream from, MetadataFilter filter) throws IOException {
+ public ParquetMetadata readParquetMetadata(final InputStream from, MetadataFilter filter) throws IOException {
FileMetaData fileMetaData = filter.accept(new MetadataFilterVisitor<FileMetaData, IOException>() {
@Override
public FileMetaData visit(NoFilter filter) throws IOException {
@@ -547,7 +546,7 @@ public class ParquetMetadataConverter {
return parquetMetadata;
}
- public static ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata) throws IOException {
+ public ParquetMetadata fromParquetMetadata(FileMetaData parquetMetadata) throws IOException {
MessageType messageType = fromParquetSchema(parquetMetadata.getSchema());
List<BlockMetaData> blocks = new ArrayList<BlockMetaData>();
List<RowGroup> row_groups = parquetMetadata.getRow_groups();
@@ -606,7 +605,7 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static MessageType fromParquetSchema(List<SchemaElement> schema) {
+ MessageType fromParquetSchema(List<SchemaElement> schema) {
Iterator<SchemaElement> iterator = schema.iterator();
SchemaElement root = iterator.next();
Types.MessageTypeBuilder builder = Types.buildMessage();
@@ -614,7 +613,7 @@ public class ParquetMetadataConverter {
return builder.named(root.name);
}
- private static void buildChildren(Types.GroupBuilder builder,
+ private void buildChildren(Types.GroupBuilder builder,
Iterator<SchemaElement> schema,
int childrenCount) {
for (int i = 0; i < childrenCount; i++) {
@@ -654,17 +653,17 @@ public class ParquetMetadataConverter {
}
// Visible for testing
- static FieldRepetitionType toParquetRepetition(Repetition repetition) {
+ FieldRepetitionType toParquetRepetition(Repetition repetition) {
return FieldRepetitionType.valueOf(repetition.name());
}
// Visible for testing
- static Repetition fromParquetRepetition(FieldRepetitionType repetition) {
+ Repetition fromParquetRepetition(FieldRepetitionType repetition) {
return Repetition.valueOf(repetition.name());
}
@Deprecated
- public static void writeDataPageHeader(
+ public void writeDataPageHeader(
int uncompressedSize,
int compressedSize,
int valueCount,
@@ -681,7 +680,7 @@ public class ParquetMetadataConverter {
valuesEncoding), to);
}
- public static void writeDataPageHeader(
+ public void writeDataPageHeader(
int uncompressedSize,
int compressedSize,
int valueCount,
@@ -696,7 +695,7 @@ public class ParquetMetadataConverter {
to);
}
- private static PageHeader newDataPageHeader(
+ private PageHeader newDataPageHeader(
int uncompressedSize, int compressedSize,
int valueCount,
org.apache.parquet.column.statistics.Statistics statistics,
@@ -717,7 +716,7 @@ public class ParquetMetadataConverter {
return pageHeader;
}
- public static void writeDataPageV2Header(
+ public void writeDataPageV2Header(
int uncompressedSize, int compressedSize,
int valueCount, int nullCount, int rowCount,
org.apache.parquet.column.statistics.Statistics statistics,
@@ -733,7 +732,7 @@ public class ParquetMetadataConverter {
rlByteLength, dlByteLength), to);
}
- private static PageHeader newDataPageV2Header(
+ private PageHeader newDataPageV2Header(
int uncompressedSize, int compressedSize,
int valueCount, int nullCount, int rowCount,
org.apache.parquet.column.statistics.Statistics<?> statistics,
@@ -753,7 +752,7 @@ public class ParquetMetadataConverter {
return pageHeader;
}
- public static void writeDictionaryPageHeader(
+ public void writeDictionaryPageHeader(
int uncompressedSize, int compressedSize, int valueCount,
org.apache.parquet.column.Encoding valuesEncoding, OutputStream to) throws IOException {
PageHeader pageHeader = new PageHeader(PageType.DICTIONARY_PAGE, uncompressedSize, compressedSize);
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
index c90ba8a..0a0b316 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ColumnChunkPageWriteStore.java
@@ -46,6 +46,8 @@ import org.apache.parquet.schema.MessageType;
class ColumnChunkPageWriteStore implements PageWriteStore {
private static final Log LOG = Log.getLog(ColumnChunkPageWriteStore.class);
+ private static ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
+
private static final class ColumnChunkPageWriter implements PageWriter {
private final ColumnDescriptor path;
@@ -92,7 +94,7 @@ class ColumnChunkPageWriteStore implements PageWriteStore {
+ compressedSize);
}
tempOutputStream.reset();
- ParquetMetadataConverter.writeDataPageHeader(
+ parquetMetadataConverter.writeDataPageHeader(
(int)uncompressedSize,
(int)compressedSize,
valueCount,
@@ -131,7 +133,7 @@ class ColumnChunkPageWriteStore implements PageWriteStore {
compressedData.size() + repetitionLevels.size() + definitionLevels.size()
);
tempOutputStream.reset();
- ParquetMetadataConverter.writeDataPageV2Header(
+ parquetMetadataConverter.writeDataPageV2Header(
uncompressedSize, compressedSize,
valueCount, nullCount, rowCount,
statistics,
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
index 19370cd..ea7a672 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileReader.java
@@ -89,7 +89,9 @@ public class ParquetFileReader implements Closeable {
private static final Log LOG = Log.getLog(ParquetFileReader.class);
- public static final String PARQUET_READ_PARALLELISM = "parquet.metadata.read.parallelism";
+ public static String PARQUET_READ_PARALLELISM = "parquet.metadata.read.parallelism";
+
+ private static ParquetMetadataConverter converter = new ParquetMetadataConverter();
/**
* for files provided, check if there's a summary file.
@@ -426,7 +428,7 @@ public class ParquetFileReader implements Closeable {
throw new RuntimeException("corrupted file: the footer index is not within the file");
}
f.seek(footerIndex);
- return ParquetMetadataConverter.readParquetMetadata(f, filter);
+ return converter.readParquetMetadata(f, filter);
} finally {
f.close();
}
@@ -573,7 +575,7 @@ public class ParquetFileReader implements Closeable {
this.readAsBytesInput(compressedPageSize),
uncompressedPageSize,
dicHeader.getNum_values(),
- ParquetMetadataConverter.getEncoding(dicHeader.getEncoding())
+ converter.getEncoding(dicHeader.getEncoding())
);
break;
case DATA_PAGE:
@@ -587,9 +589,9 @@ public class ParquetFileReader implements Closeable {
createdBy,
dataHeaderV1.getStatistics(),
descriptor.col.getType()),
- ParquetMetadataConverter.getEncoding(dataHeaderV1.getRepetition_level_encoding()),
- ParquetMetadataConverter.getEncoding(dataHeaderV1.getDefinition_level_encoding()),
- ParquetMetadataConverter.getEncoding(dataHeaderV1.getEncoding())
+ converter.getEncoding(dataHeaderV1.getRepetition_level_encoding()),
+ converter.getEncoding(dataHeaderV1.getDefinition_level_encoding()),
+ converter.getEncoding(dataHeaderV1.getEncoding())
));
valuesCountReadSoFar += dataHeaderV1.getNum_values();
break;
@@ -603,7 +605,7 @@ public class ParquetFileReader implements Closeable {
dataHeaderV2.getNum_values(),
this.readAsBytesInput(dataHeaderV2.getRepetition_levels_byte_length()),
this.readAsBytesInput(dataHeaderV2.getDefinition_levels_byte_length()),
- ParquetMetadataConverter.getEncoding(dataHeaderV2.getEncoding()),
+ converter.getEncoding(dataHeaderV2.getEncoding()),
this.readAsBytesInput(dataSize),
uncompressedPageSize,
fromParquetStatistics(
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
index 1f87240..e285376 100644
--- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
+++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetFileWriter.java
@@ -67,6 +67,8 @@ import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
public class ParquetFileWriter {
private static final Log LOG = Log.getLog(ParquetFileWriter.class);
+ private static ParquetMetadataConverter metadataConverter = new ParquetMetadataConverter();
+
public static final String PARQUET_METADATA_FILE = "_metadata";
public static final String PARQUET_COMMON_METADATA_FILE = "_common_metadata";
public static final byte[] MAGIC = "PAR1".getBytes(Charset.forName("ASCII"));
@@ -302,7 +304,7 @@ public class ParquetFileWriter {
currentChunkDictionaryPageOffset = out.getPos();
int uncompressedSize = dictionaryPage.getUncompressedSize();
int compressedPageSize = (int)dictionaryPage.getBytes().size(); // TODO: fix casts
- ParquetMetadataConverter.writeDictionaryPageHeader(
+ metadataConverter.writeDictionaryPageHeader(
uncompressedSize,
compressedPageSize,
dictionaryPage.getDictionarySize(),
@@ -337,7 +339,7 @@ public class ParquetFileWriter {
long beforeHeader = out.getPos();
if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values");
int compressedPageSize = (int)bytes.size();
- ParquetMetadataConverter.writeDataPageHeader(
+ metadataConverter.writeDataPageHeader(
uncompressedPageSize, compressedPageSize,
valueCount,
rlEncoding,
@@ -374,7 +376,7 @@ public class ParquetFileWriter {
long beforeHeader = out.getPos();
if (DEBUG) LOG.debug(beforeHeader + ": write data page: " + valueCount + " values");
int compressedPageSize = (int)bytes.size();
- ParquetMetadataConverter.writeDataPageHeader(
+ metadataConverter.writeDataPageHeader(
uncompressedPageSize, compressedPageSize,
valueCount,
statistics,
@@ -467,7 +469,7 @@ public class ParquetFileWriter {
private static void serializeFooter(ParquetMetadata footer, FSDataOutputStream out) throws IOException {
long footerIndex = out.getPos();
- org.apache.parquet.format.FileMetaData parquetMetadata = ParquetMetadataConverter.toParquetMetadata(CURRENT_VERSION, footer);
+ org.apache.parquet.format.FileMetaData parquetMetadata = metadataConverter.toParquetMetadata(CURRENT_VERSION, footer);
writeFileMetaData(parquetMetadata, out);
if (DEBUG) LOG.debug(out.getPos() + ": footer length = " + (out.getPos() - footerIndex));
BytesUtils.writeIntLittleEndian(out, (int)(out.getPos() - footerIndex));
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
index dd4aba9..e44d140 100644
--- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
+++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java
@@ -87,14 +87,16 @@ public class TestParquetMetadataConverter {
@Test
public void testSchemaConverter() {
- List<SchemaElement> parquetSchema = ParquetMetadataConverter.toParquetSchema(Paper.schema);
- MessageType schema = ParquetMetadataConverter.fromParquetSchema(parquetSchema);
+ ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
+ List<SchemaElement> parquetSchema = parquetMetadataConverter.toParquetSchema(Paper.schema);
+ MessageType schema = parquetMetadataConverter.fromParquetSchema(parquetSchema);
assertEquals(Paper.schema, schema);
}
@Test
public void testSchemaConverterDecimal() {
- List<SchemaElement> schemaElements = ParquetMetadataConverter.toParquetSchema(
+ ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
+ List<SchemaElement> schemaElements = parquetMetadataConverter.toParquetSchema(
Types.buildMessage()
.required(PrimitiveTypeName.BINARY)
.as(OriginalType.DECIMAL).precision(9).scale(2)
@@ -123,29 +125,30 @@ public class TestParquetMetadataConverter {
@Test
public void testEnumEquivalence() {
+ ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
for (org.apache.parquet.column.Encoding encoding : org.apache.parquet.column.Encoding.values()) {
- assertEquals(encoding, ParquetMetadataConverter.getEncoding(ParquetMetadataConverter.getEncoding(encoding)));
+ assertEquals(encoding, parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding)));
}
for (org.apache.parquet.format.Encoding encoding : org.apache.parquet.format.Encoding.values()) {
- assertEquals(encoding, ParquetMetadataConverter.getEncoding(ParquetMetadataConverter.getEncoding(encoding)));
+ assertEquals(encoding, parquetMetadataConverter.getEncoding(parquetMetadataConverter.getEncoding(encoding)));
}
for (Repetition repetition : Repetition.values()) {
- assertEquals(repetition, ParquetMetadataConverter.fromParquetRepetition(ParquetMetadataConverter.toParquetRepetition(repetition)));
+ assertEquals(repetition, parquetMetadataConverter.fromParquetRepetition(parquetMetadataConverter.toParquetRepetition(repetition)));
}
for (FieldRepetitionType repetition : FieldRepetitionType.values()) {
- assertEquals(repetition, ParquetMetadataConverter.toParquetRepetition(ParquetMetadataConverter.fromParquetRepetition(repetition)));
+ assertEquals(repetition, parquetMetadataConverter.toParquetRepetition(parquetMetadataConverter.fromParquetRepetition(repetition)));
}
for (PrimitiveTypeName primitiveTypeName : PrimitiveTypeName.values()) {
- assertEquals(primitiveTypeName, ParquetMetadataConverter.getPrimitive(ParquetMetadataConverter.getType(primitiveTypeName)));
+ assertEquals(primitiveTypeName, parquetMetadataConverter.getPrimitive(parquetMetadataConverter.getType(primitiveTypeName)));
}
for (Type type : Type.values()) {
- assertEquals(type, ParquetMetadataConverter.getType(ParquetMetadataConverter.getPrimitive(type)));
+ assertEquals(type, parquetMetadataConverter.getType(parquetMetadataConverter.getPrimitive(type)));
}
for (OriginalType original : OriginalType.values()) {
- assertEquals(original, ParquetMetadataConverter.getOriginalType(ParquetMetadataConverter.getConvertedType(original)));
+ assertEquals(original, parquetMetadataConverter.getOriginalType(parquetMetadataConverter.getConvertedType(original)));
}
for (ConvertedType converted : ConvertedType.values()) {
- assertEquals(converted, ParquetMetadataConverter.getConvertedType(ParquetMetadataConverter.getOriginalType(converted)));
+ assertEquals(converted, parquetMetadataConverter.getConvertedType(parquetMetadataConverter.getOriginalType(converted)));
}
}
@@ -283,6 +286,8 @@ public class TestParquetMetadataConverter {
@Test
public void testEncodingsCache() {
+ ParquetMetadataConverter parquetMetadataConverter = new ParquetMetadataConverter();
+
List<org.apache.parquet.format.Encoding> formatEncodingsCopy1 =
Arrays.asList(org.apache.parquet.format.Encoding.BIT_PACKED,
org.apache.parquet.format.Encoding.RLE_DICTIONARY,
@@ -298,9 +303,9 @@ public class TestParquetMetadataConverter {
expected.add(org.apache.parquet.column.Encoding.RLE_DICTIONARY);
expected.add(org.apache.parquet.column.Encoding.DELTA_LENGTH_BYTE_ARRAY);
- Set<org.apache.parquet.column.Encoding> res1 = ParquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1);
- Set<org.apache.parquet.column.Encoding> res2 = ParquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1);
- Set<org.apache.parquet.column.Encoding> res3 = ParquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy2);
+ Set<org.apache.parquet.column.Encoding> res1 = parquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1);
+ Set<org.apache.parquet.column.Encoding> res2 = parquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy1);
+ Set<org.apache.parquet.column.Encoding> res3 = parquetMetadataConverter.fromFormatEncodings(formatEncodingsCopy2);
// make sure they are all semantically equal
assertEquals(expected, res1);
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java
index 97d90e4..2c05c30 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/ThriftSchemaConvertVisitor.java
@@ -73,7 +73,7 @@ import static org.apache.parquet.schema.Types.primitive;
*
* @author Tianshuo Deng
*/
-public class ThriftSchemaConvertVisitor implements ThriftType.TypeVisitor<ConvertedField, ThriftSchemaConvertVisitor.State> {
+class ThriftSchemaConvertVisitor implements ThriftType.StateVisitor<ConvertedField, ThriftSchemaConvertVisitor.State> {
private final FieldProjectionFilter fieldProjectionFilter;
private final boolean doProjection;
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java
index ec63d85..e8e5617 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/projection/amend/DefaultEventsVisitor.java
@@ -34,7 +34,7 @@ import java.util.List;
/**
* Create a dummy events for all required fields according to thrift definition
*/
-class DefaultEventsVisitor implements ThriftType.TypeVisitor<Void, Void> {
+class DefaultEventsVisitor implements ThriftType.StateVisitor<Void, Void> {
List<ParquetProtocol> dummyEvents= new ArrayList<ParquetProtocol>();
@Override
public Void visit(ThriftType.MapType mapType, Void v) {
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java
index 95a6d27..46c26a5 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/CompatibilityChecker.java
@@ -44,7 +44,7 @@ public class CompatibilityChecker {
public CompatibilityReport checkCompatibility(ThriftType.StructType oldStruct, ThriftType.StructType newStruct) {
CompatibleCheckerVisitor visitor = new CompatibleCheckerVisitor(oldStruct);
- newStruct.accept(visitor, null);
+ newStruct.accept(visitor);
return visitor.getReport();
}
@@ -68,7 +68,7 @@ class CompatibilityReport {
}
}
-class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
+class CompatibleCheckerVisitor implements ThriftType.TypeVisitor {
ThriftType oldType;
CompatibilityReport report = new CompatibilityReport();
@@ -81,7 +81,7 @@ class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
}
@Override
- public Void visit(ThriftType.MapType mapType, Void v) {
+ public void visit(ThriftType.MapType mapType) {
ThriftType.MapType currentOldType = ((ThriftType.MapType) oldType);
ThriftField oldKeyField = currentOldType.getKey();
ThriftField newKeyField = mapType.getKey();
@@ -93,27 +93,24 @@ class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
checkField(oldValueField, newValueField);
oldType = currentOldType;
- return null;
}
@Override
- public Void visit(ThriftType.SetType setType, Void v) {
+ public void visit(ThriftType.SetType setType) {
ThriftType.SetType currentOldType = ((ThriftType.SetType) oldType);
ThriftField oldField = currentOldType.getValues();
ThriftField newField = setType.getValues();
checkField(oldField, newField);
oldType = currentOldType;
- return null;
}
@Override
- public Void visit(ThriftType.ListType listType, Void v) {
+ public void visit(ThriftType.ListType listType) {
ThriftType.ListType currentOldType = ((ThriftType.ListType) oldType);
ThriftField oldField = currentOldType.getValues();
ThriftField newField = listType.getValues();
checkField(oldField, newField);
oldType = currentOldType;
- return null;
}
public void fail(String message) {
@@ -138,7 +135,7 @@ class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
}
oldType = oldField.getType();
- newField.getType().accept(this, null);
+ newField.getType().accept(this);
}
private boolean firstIsMoreRestirctive(ThriftField.Requirement firstReq, ThriftField.Requirement secReq) {
@@ -151,7 +148,7 @@ class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
}
@Override
- public Void visit(ThriftType.StructType newStruct, Void v) {
+ public void visit(ThriftType.StructType newStruct) {
ThriftType.StructType currentOldType = ((ThriftType.StructType) oldType);
short oldMaxId = 0;
for (ThriftField oldField : currentOldType.getChildren()) {
@@ -162,7 +159,7 @@ class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
ThriftField newField = newStruct.getChildById(fieldId);
if (newField == null) {
fail("can not find index in new Struct: " + fieldId +" in " + newStruct);
- return null;
+ return;
}
checkField(oldField, newField);
}
@@ -176,58 +173,49 @@ class CompatibleCheckerVisitor implements ThriftType.TypeVisitor<Void, Void> {
short newFieldId = newField.getFieldId();
if (newFieldId > oldMaxId) {
fail("new required field " + newField.getName() + " is added");
- return null;
+ return;
}
if (newFieldId < oldMaxId && currentOldType.getChildById(newFieldId) == null) {
fail("new required field " + newField.getName() + " is added");
- return null;
+ return;
}
}
//restore
oldType = currentOldType;
- return null;
}
@Override
- public Void visit(EnumType enumType, Void v) {
- return null;
+ public void visit(EnumType enumType) {
}
@Override
- public Void visit(BoolType boolType, Void v) {
- return null;
+ public void visit(BoolType boolType) {
}
@Override
- public Void visit(ByteType byteType, Void v) {
- return null;
+ public void visit(ByteType byteType) {
}
@Override
- public Void visit(DoubleType doubleType, Void v) {
- return null;
+ public void visit(DoubleType doubleType) {
}
@Override
- public Void visit(I16Type i16Type, Void v) {
- return null;
+ public void visit(I16Type i16Type) {
}
@Override
- public Void visit(I32Type i32Type, Void v) {
- return null;
+ public void visit(I32Type i32Type) {
}
@Override
- public Void visit(I64Type i64Type, Void v) {
- return null;
+ public void visit(I64Type i64Type) {
}
@Override
- public Void visit(StringType stringType, Void v) {
- return null;
+ public void visit(StringType stringType) {
}
}
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
index 92d12b4..19c7c9f 100644
--- a/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
+++ b/parquet-thrift/src/main/java/org/apache/parquet/thrift/struct/ThriftType.java
@@ -98,7 +98,7 @@ public abstract class ThriftType {
return toJSON();
}
- public interface TypeVisitor<R, S> {
+ public interface StateVisitor<R, S> {
R visit(MapType mapType, S state);
@@ -126,6 +126,85 @@ public abstract class ThriftType {
}
+ /**
+ * @deprecated will be removed in 2.0.0; use StateVisitor instead.
+ */
+ public interface TypeVisitor {
+
+ void visit(MapType mapType);
+
+ void visit(SetType setType);
+
+ void visit(ListType listType);
+
+ void visit(StructType structType);
+
+ void visit(EnumType enumType);
+
+ void visit(BoolType boolType);
+
+ void visit(ByteType byteType);
+
+ void visit(DoubleType doubleType);
+
+ void visit(I16Type i16Type);
+
+ void visit(I32Type i32Type);
+
+ void visit(I64Type i64Type);
+
+ void visit(StringType stringType);
+
+ }
+
+ /**
+ * @deprecated will be removed in 2.0.0.
+ */
+ @Deprecated
+ public static abstract class ComplexTypeVisitor implements TypeVisitor {
+
+ @Override
+ final public void visit(EnumType enumType) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(BoolType boolType) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(ByteType byteType) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(DoubleType doubleType) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(I16Type i16Type) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(I32Type i32Type) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(I64Type i64Type) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ @Override
+ final public void visit(StringType stringType) {
+ throw new IllegalArgumentException("Expected complex type");
+ }
+
+ }
+
public static class StructType extends ThriftType {
private final List<ThriftField> children;
@@ -189,11 +268,16 @@ public abstract class ThriftType {
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
@Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
@@ -232,11 +316,16 @@ public abstract class ThriftType {
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
@Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof MapType)) return false;
@@ -273,11 +362,16 @@ public abstract class ThriftType {
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
@Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof SetType)) return false;
@@ -312,11 +406,16 @@ public abstract class ThriftType {
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
@Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof ListType)) return false;
@@ -408,11 +507,16 @@ public abstract class ThriftType {
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
@Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
+
+ @Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof EnumType)) return false;
@@ -440,9 +544,14 @@ public abstract class ThriftType {
super(BOOL);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
public static class ByteType extends ThriftType {
@@ -452,9 +561,14 @@ public abstract class ThriftType {
super(BYTE);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
public static class DoubleType extends ThriftType {
@@ -464,9 +578,14 @@ public abstract class ThriftType {
super(DOUBLE);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
public static class I16Type extends ThriftType {
@@ -476,9 +595,14 @@ public abstract class ThriftType {
super(I16);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
public static class I32Type extends ThriftType {
@@ -488,9 +612,14 @@ public abstract class ThriftType {
super(I32);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
public static class I64Type extends ThriftType {
@@ -500,10 +629,15 @@ public abstract class ThriftType {
super(I64);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
public static class StringType extends ThriftType {
@@ -513,9 +647,14 @@ public abstract class ThriftType {
super(STRING);
}
@Override
- public <R, S> R accept(TypeVisitor<R, S> visitor, S state) {
+ public <R, S> R accept(StateVisitor<R, S> visitor, S state) {
return visitor.visit(this, state);
}
+
+ @Override
+ public void accept(TypeVisitor visitor) {
+ visitor.visit(this);
+ }
}
private final ThriftTypeID type;
@@ -525,7 +664,9 @@ public abstract class ThriftType {
this.type = type;
}
- public abstract <R, S> R accept(TypeVisitor<R, S> visitor, S state);
+ public abstract void accept(TypeVisitor visitor);
+
+ public abstract <R, S> R accept(StateVisitor<R, S> visitor, S state);
@JsonIgnore
public ThriftTypeID getType() {
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java b/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java
index ee545db..a00718a 100644
--- a/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java
+++ b/parquet-thrift/src/test/java/org/apache/parquet/thrift/projection/TestFieldsPath.java
@@ -69,7 +69,7 @@ public class TestFieldsPath {
}
- private static class PrimitivePathVisitor implements ThriftType.TypeVisitor<List<String>, FieldsPath> {
+ private static class PrimitivePathVisitor implements ThriftType.StateVisitor<List<String>, FieldsPath> {
private String delim;
private PrimitivePathVisitor(String delim) {
http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/9fde6534/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c171660..d48021c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -226,7 +226,10 @@
<exclude>org/apache/parquet/column/**</exclude>
<exclude>org/apache/parquet/hadoop/ParquetInputSplit</exclude>
<exclude>shaded/**</exclude> <!-- shaded by parquet -->
- <exclude>parquet/**</exclude> <!-- shaded by parquet-format -->
+ <!-- temporary exclusions for false-positives -->
+ <exclude>org/apache/parquet/Version</exclude>
+ <exclude>org/apache/parquet/schema/**</exclude> <!-- methods moved to new superclass -->
+ <exclude>org/apache/parquet/thrift/ThriftSchemaConvertVisitor</exclude> <!-- not public -->
</excludes>
</requireBackwardCompatibility>
</rules>