You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2022/02/03 12:43:21 UTC
[hudi] branch master updated (d681824 -> 5927bdd)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git.
from d681824 [HUDI-3337] Fixing Parquet Column Range metadata extraction (#4705)
add 5927bdd [HUDI-1295] Metadata Index - Bloom filter and Column stats index to speed up index lookups (#4352)
No new revisions were added by this update.
Summary of changes:
.../org/apache/hudi/config/HoodieWriteConfig.java | 8 +
.../org/apache/hudi/index/HoodieIndexUtils.java | 45 ++
.../bloom/HoodieBaseBloomIndexCheckFunction.java | 14 +-
.../apache/hudi/index/bloom/HoodieBloomIndex.java | 104 +++-
.../hudi/index/bloom/HoodieGlobalBloomIndex.java | 6 +-
.../bloom/ListBasedHoodieBloomIndexHelper.java | 9 +-
.../java/org/apache/hudi/io/HoodieIOHandle.java | 6 +-
.../hudi/io/HoodieKeyLocationFetchHandle.java | 2 +-
.../org/apache/hudi/io/HoodieKeyLookupHandle.java | 130 ++---
.../org/apache/hudi/io/HoodieKeyLookupResult.java | 39 +-
.../org/apache/hudi/io/HoodieRangeInfoHandle.java | 7 +-
.../java/org/apache/hudi/io/HoodieReadHandle.java | 24 +-
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 2 +-
.../metadata/HoodieBackedTableMetadataWriter.java | 167 +++++-
.../java/org/apache/hudi/table/HoodieTable.java | 3 +
.../FlinkHoodieBackedTableMetadataWriter.java | 39 +-
.../index/bloom/TestFlinkHoodieBloomIndex.java | 11 +-
.../index/bloom/HoodieBloomIndexCheckFunction.java | 16 +-
.../HoodieMetadataBloomIndexCheckFunction.java | 161 ++++++
.../index/bloom/SparkHoodieBloomIndexHelper.java | 37 +-
.../index/columnstats/ColumnStatsIndexHelper.java | 2 +-
.../SparkHoodieBackedTableMetadataWriter.java | 29 +-
.../functional/TestHoodieBackedMetadata.java | 71 ++-
.../functional/TestHoodieBackedTableMetadata.java | 6 +-
.../client/functional/TestHoodieMetadataBase.java | 32 +-
.../hudi/index/bloom/TestHoodieBloomIndex.java | 31 +-
.../index/bloom/TestHoodieGlobalBloomIndex.java | 4 +-
.../hudi/testutils/HoodieClientTestHarness.java | 3 +-
hudi-common/src/main/avro/HoodieMetadata.avsc | 135 ++++-
.../bloom/HoodieDynamicBoundedBloomFilter.java | 2 +-
.../hudi/common/config/HoodieMetadataConfig.java | 86 +++
.../common/model/HoodieColumnRangeMetadata.java | 42 +-
.../table/log/AbstractHoodieLogRecordReader.java | 4 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 41 +-
.../hash/{ColumnID.java => ColumnIndexID.java} | 15 +-
.../util/hash/{FileID.java => FileIndexID.java} | 13 +-
.../hash/{HoodieID.java => HoodieIndexID.java} | 14 +-
.../{PartitionID.java => PartitionIndexID.java} | 13 +-
.../apache/hudi/io/storage/HoodieFileReader.java | 6 +
.../apache/hudi/io/storage/HoodieHFileReader.java | 79 ++-
.../apache/hudi/metadata/BaseTableMetadata.java | 149 ++++-
.../metadata/FileSystemBackedTableMetadata.java | 20 +
.../hudi/metadata/HoodieBackedTableMetadata.java | 158 ++++--
.../HoodieMetadataMergedLogRecordReader.java | 2 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 4 +-
.../hudi/metadata/HoodieMetadataPayload.java | 306 ++++++++--
.../apache/hudi/metadata/HoodieTableMetadata.java | 36 ++
.../hudi/metadata/HoodieTableMetadataUtil.java | 628 +++++++++++++++++++--
.../hudi/metadata/MetadataPartitionType.java | 39 +-
49 files changed, 2267 insertions(+), 533 deletions(-)
copy hudi-common/src/main/java/org/apache/hudi/metadata/MetadataPartitionType.java => hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieKeyLookupResult.java (55%)
create mode 100644 hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/bloom/HoodieMetadataBloomIndexCheckFunction.java
rename hudi-common/src/main/java/org/apache/hudi/common/util/hash/{ColumnID.java => ColumnIndexID.java} (80%)
rename hudi-common/src/main/java/org/apache/hudi/common/util/hash/{FileID.java => FileIndexID.java} (84%)
rename hudi-common/src/main/java/org/apache/hudi/common/util/hash/{HoodieID.java => HoodieIndexID.java} (89%)
rename hudi-common/src/main/java/org/apache/hudi/common/util/hash/{PartitionID.java => PartitionIndexID.java} (83%)