You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by ga...@apache.org on 2020/02/26 13:18:47 UTC
[parquet-mr] branch master updated (7469e87 -> 806037c)
This is an automated email from the ASF dual-hosted git repository.
gabor pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/parquet-mr.git.
from 7469e87 PARQUET-1784: Column-wise configuration (#754)
add 806037c PARQUET-41: Add bloom filter (#757)
No new revisions were added by this update.
Summary of changes:
parquet-column/pom.xml | 5 +
.../apache/parquet/column/ParquetProperties.java | 100 +++++-
.../parquet/column/impl/ColumnWriteStoreBase.java | 39 ++-
.../parquet/column/impl/ColumnWriteStoreV1.java | 13 +-
.../parquet/column/impl/ColumnWriteStoreV2.java | 13 +-
.../parquet/column/impl/ColumnWriterBase.java | 80 +++++
.../apache/parquet/column/impl/ColumnWriterV1.java | 6 +
.../apache/parquet/column/impl/ColumnWriterV2.java | 12 +-
.../values/bloomfilter/BlockSplitBloomFilter.java | 382 +++++++++++++++++++++
.../column/values/bloomfilter/BloomFilter.java | 171 +++++++++
.../bloomfilter/BloomFilterWriteStore.java} | 20 +-
.../values/bloomfilter/BloomFilterWriter.java | 19 +-
.../column/values/bloomfilter/HashFunction.java | 25 +-
.../parquet/column/values/bloomfilter/XxHash.java | 36 +-
.../bloomfilter/TestBlockSplitBloomFilter.java | 229 ++++++++++++
.../main/java/org/apache/parquet/format/Util.java | 8 +
.../java/org/apache/parquet/HadoopReadOptions.java | 9 +-
.../org/apache/parquet/ParquetReadOptions.java | 21 +-
.../filter2/bloomfilterlevel/BloomFilterImpl.java | 150 ++++++++
.../parquet/filter2/compat/RowGroupFilter.java | 16 +-
.../format/converter/ParquetMetadataConverter.java | 39 ++-
.../apache/parquet/hadoop/BloomFilterReader.java | 70 ++++
.../parquet/hadoop/ColumnChunkPageWriteStore.java | 18 +-
.../hadoop/InternalParquetRecordWriter.java | 11 +-
.../apache/parquet/hadoop/ParquetFileReader.java | 50 +++
.../apache/parquet/hadoop/ParquetFileWriter.java | 66 +++-
.../apache/parquet/hadoop/ParquetInputFormat.java | 11 +-
.../apache/parquet/hadoop/ParquetOutputFormat.java | 53 ++-
.../org/apache/parquet/hadoop/ParquetReader.java | 16 +-
.../org/apache/parquet/hadoop/ParquetWriter.java | 36 +-
.../hadoop/metadata/ColumnChunkMetaData.java | 25 +-
.../apache/parquet/hadoop/TestBloomFiltering.java | 257 ++++++++++++++
.../hadoop/TestColumnChunkPageWriteStore.java | 1 +
.../parquet/hadoop/TestParquetFileWriter.java | 42 ++-
.../apache/parquet/hadoop/TestParquetWriter.java | 54 ++-
pom.xml | 6 +
36 files changed, 1997 insertions(+), 112 deletions(-)
create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BlockSplitBloomFilter.java
create mode 100644 parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BloomFilter.java
copy parquet-column/src/main/java/org/apache/parquet/column/{page/PageWriteStore.java => values/bloomfilter/BloomFilterWriteStore.java} (74%)
copy parquet-generator/src/main/java/org/apache/parquet/version/Generator.java => parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/BloomFilterWriter.java (77%)
copy parquet-common/src/main/java/org/apache/parquet/bytes/ByteBufferAllocator.java => parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/HashFunction.java (62%)
copy parquet-common/src/main/java/org/apache/parquet/bytes/HeapByteBufferAllocator.java => parquet-column/src/main/java/org/apache/parquet/column/values/bloomfilter/XxHash.java (64%)
create mode 100644 parquet-column/src/test/java/org/apache/parquet/column/values/bloomfilter/TestBlockSplitBloomFilter.java
create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/filter2/bloomfilterlevel/BloomFilterImpl.java
create mode 100644 parquet-hadoop/src/main/java/org/apache/parquet/hadoop/BloomFilterReader.java
create mode 100644 parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestBloomFiltering.java