You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yu...@apache.org on 2022/10/09 03:57:17 UTC
[hudi] branch release-0.12.1 updated (28cb191df7 -> baeff4331d)
This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-0.12.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 28cb191df7 [MINOR] Update release version to reflect published version 0.12.1
new 51105621af [HUDI-4992] Fixing invalid min/max record key stats in Parquet metadata (#6883)
new baeff4331d Bumping release candidate number 2
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 4 +-
hudi-cli/pom.xml | 2 +-
hudi-client/hudi-client-common/pom.xml | 4 +-
.../apache/hudi/io/storage/HoodieOrcWriter.java | 10 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 118 +++++++++++++++++++++
.../hudi/io/storage/TestHoodieOrcReaderWriter.java | 7 +-
hudi-client/hudi-flink-client/pom.xml | 4 +-
.../row/HoodieRowDataParquetWriteSupport.java | 55 ++++------
hudi-client/hudi-java-client/pom.xml | 4 +-
hudi-client/hudi-spark-client/pom.xml | 4 +-
.../storage/row/HoodieRowParquetWriteSupport.java | 61 +++++------
.../row/TestHoodieInternalRowParquetWriter.java | 95 ++++++++++-------
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 2 +-
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 60 +++++------
.../hudi/avro/HoodieBloomFilterWriteSupport.java | 96 +++++++++++++++++
.../org/apache/hudi/common/util/BaseFileUtils.java | 13 +--
.../hudi/avro/TestHoodieAvroWriteSupport.java | 67 ------------
hudi-examples/hudi-examples-common/pom.xml | 2 +-
hudi-examples/hudi-examples-flink/pom.xml | 2 +-
hudi-examples/hudi-examples-java/pom.xml | 2 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 4 +-
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 4 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 4 +-
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 4 +-
hudi-flink-datasource/pom.xml | 4 +-
hudi-gcp/pom.xml | 2 +-
hudi-hadoop-mr/pom.xml | 2 +-
hudi-integ-test/pom.xml | 2 +-
hudi-kafka-connect/pom.xml | 4 +-
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 +-
hudi-spark-datasource/hudi-spark/pom.xml | 4 +-
hudi-spark-datasource/hudi-spark2-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 4 +-
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 4 +-
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 4 +-
.../hudi-spark3.2plus-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 4 +-
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
hudi-sync/hudi-sync-common/pom.xml | 2 +-
hudi-sync/pom.xml | 2 +-
hudi-tests-common/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 2 +-
hudi-utilities/pom.xml | 2 +-
packaging/hudi-aws-bundle/pom.xml | 2 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 2 +-
packaging/hudi-flink-bundle/pom.xml | 2 +-
packaging/hudi-gcp-bundle/pom.xml | 2 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 2 +-
packaging/hudi-hive-sync-bundle/pom.xml | 2 +-
packaging/hudi-integ-test-bundle/pom.xml | 2 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 2 +-
packaging/hudi-presto-bundle/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
packaging/hudi-trino-bundle/pom.xml | 2 +-
packaging/hudi-utilities-bundle/pom.xml | 2 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 2 +-
pom.xml | 2 +-
80 files changed, 448 insertions(+), 308 deletions(-)
create mode 100644 hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
create mode 100644 hudi-common/src/main/java/org/apache/hudi/avro/HoodieBloomFilterWriteSupport.java
delete mode 100644 hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java
[hudi] 01/02: [HUDI-4992] Fixing invalid min/max record key stats in Parquet metadata (#6883)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a commit to branch release-0.12.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 51105621af08eb77a8007ac965a8e5d0882c5d97
Author: Alexey Kudinkin <al...@infinilake.com>
AuthorDate: Fri Oct 7 03:37:26 2022 -0700
[HUDI-4992] Fixing invalid min/max record key stats in Parquet metadata (#6883)
---
.../apache/hudi/io/storage/HoodieOrcWriter.java | 10 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 118 +++++++++++++++++++++
.../hudi/io/storage/TestHoodieOrcReaderWriter.java | 7 +-
.../row/HoodieRowDataParquetWriteSupport.java | 55 ++++------
.../storage/row/HoodieRowParquetWriteSupport.java | 61 +++++------
.../row/TestHoodieInternalRowParquetWriter.java | 95 ++++++++++-------
.../apache/hudi/avro/HoodieAvroWriteSupport.java | 60 +++++------
.../hudi/avro/HoodieBloomFilterWriteSupport.java | 96 +++++++++++++++++
.../org/apache/hudi/common/util/BaseFileUtils.java | 13 +--
.../hudi/avro/TestHoodieAvroWriteSupport.java | 67 ------------
10 files changed, 361 insertions(+), 221 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcWriter.java
index a532ac66c9..4bcab2cec8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/storage/HoodieOrcWriter.java
@@ -23,6 +23,7 @@ import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter;
import org.apache.hudi.common.engine.TaskContextSupplier;
@@ -44,9 +45,6 @@ import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER;
public class HoodieOrcWriter<T extends HoodieRecordPayload, R extends IndexedRecord>
implements HoodieFileWriter<R>, Closeable {
@@ -155,11 +153,11 @@ public class HoodieOrcWriter<T extends HoodieRecordPayload, R extends IndexedRec
final BloomFilter bloomFilter = orcConfig.getBloomFilter();
writer.addUserMetadata(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY, ByteBuffer.wrap(bloomFilter.serializeToString().getBytes()));
if (minRecordKey != null && maxRecordKey != null) {
- writer.addUserMetadata(HOODIE_MIN_RECORD_KEY_FOOTER, ByteBuffer.wrap(minRecordKey.getBytes()));
- writer.addUserMetadata(HOODIE_MAX_RECORD_KEY_FOOTER, ByteBuffer.wrap(maxRecordKey.getBytes()));
+ writer.addUserMetadata(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER, ByteBuffer.wrap(minRecordKey.getBytes()));
+ writer.addUserMetadata(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER, ByteBuffer.wrap(maxRecordKey.getBytes()));
}
if (bloomFilter.getBloomFilterTypeCode().name().contains(HoodieDynamicBoundedBloomFilter.TYPE_CODE_PREFIX)) {
- writer.addUserMetadata(HOODIE_BLOOM_FILTER_TYPE_CODE, ByteBuffer.wrap(bloomFilter.getBloomFilterTypeCode().name().getBytes()));
+ writer.addUserMetadata(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE, ByteBuffer.wrap(bloomFilter.getBloomFilterTypeCode().name().getBytes()));
}
}
writer.addUserMetadata(HoodieOrcConfig.AVRO_SCHEMA_METADATA_KEY, ByteBuffer.wrap(avroSchema.toString().getBytes()));
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
new file mode 100644
index 0000000000..df879dc816
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.DummyTaskContextSupplier;
+import org.apache.hudi.common.bloom.BloomFilter;
+import org.apache.hudi.common.bloom.BloomFilterFactory;
+import org.apache.hudi.common.bloom.BloomFilterTypeCode;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ParquetUtils;
+import org.apache.hudi.io.storage.HoodieAvroParquetWriter;
+import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.parquet.avro.AvroSchemaConverter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestHoodieAvroParquetWriter {
+
+ @TempDir java.nio.file.Path tmpDir;
+
+ @Test
+ public void testProperWriting() throws IOException {
+ Configuration hadoopConf = new Configuration();
+
+ HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(0xDEED);
+ List<GenericRecord> records = dataGen.generateGenericRecords(10);
+
+ Schema schema = records.get(0).getSchema();
+
+ BloomFilter filter = BloomFilterFactory.createBloomFilter(1000, 0.0001, 10000,
+ BloomFilterTypeCode.DYNAMIC_V0.name());
+ HoodieAvroWriteSupport writeSupport = new HoodieAvroWriteSupport(new AvroSchemaConverter().convert(schema),
+ schema, Option.of(filter));
+
+ HoodieParquetConfig<HoodieAvroWriteSupport> parquetConfig =
+ new HoodieParquetConfig(writeSupport, CompressionCodecName.GZIP, ParquetWriter.DEFAULT_BLOCK_SIZE,
+ ParquetWriter.DEFAULT_PAGE_SIZE, 1024 * 1024 * 1024, hadoopConf, 0.1);
+
+ Path filePath = new Path(tmpDir.resolve("test.parquet").toAbsolutePath().toString());
+
+ try (HoodieAvroParquetWriter<GenericRecord> writer =
+ new HoodieAvroParquetWriter<>(filePath, parquetConfig, "001", new DummyTaskContextSupplier(), true)) {
+ for (GenericRecord record : records) {
+ writer.writeAvro((String) record.get("_row_key"), record);
+ }
+ }
+
+ ParquetUtils utils = new ParquetUtils();
+
+ // Step 1: Make sure records are written appropriately
+ List<GenericRecord> readRecords = utils.readAvroRecords(hadoopConf, filePath);
+
+ assertEquals(toJson(records), toJson(readRecords));
+
+ // Step 2: Assert Parquet metadata was written appropriately
+ List<String> recordKeys = records.stream().map(r -> (String) r.get("_row_key")).collect(Collectors.toList());
+
+ String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
+ String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
+
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(hadoopConf, filePath).getFileMetaData();
+
+ Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
+
+ assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER), minKey);
+ assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER), maxKey);
+ assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE), BloomFilterTypeCode.DYNAMIC_V0.name());
+
+ // Step 3: Make sure Bloom Filter contains all the record keys
+ BloomFilter bloomFilter = utils.readBloomFilterFromMetadata(hadoopConf, filePath);
+ recordKeys.forEach(recordKey -> {
+ assertTrue(bloomFilter.mightContain(recordKey));
+ });
+ }
+
+ private static List<String> toJson(List<GenericRecord> records) {
+ return records.stream().map(r -> {
+ try {
+ return new String(HoodieAvroUtils.avroToJson(r, true));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }).collect(Collectors.toList());
+ }
+}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
index 817fc25a5d..373fc31a56 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/io/storage/TestHoodieOrcReaderWriter.java
@@ -18,6 +18,7 @@
package org.apache.hudi.io.storage;
+import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
@@ -37,8 +38,6 @@ import java.io.IOException;
import java.util.function.Supplier;
import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER;
import static org.apache.hudi.io.storage.HoodieOrcConfig.AVRO_SCHEMA_METADATA_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -78,8 +77,8 @@ public class TestHoodieOrcReaderWriter extends TestHoodieReaderWriterBase {
protected void verifyMetadata(Configuration conf) throws IOException {
Reader orcReader = OrcFile.createReader(getFilePath(), OrcFile.readerOptions(conf));
assertEquals(4, orcReader.getMetadataKeys().size());
- assertTrue(orcReader.getMetadataKeys().contains(HOODIE_MIN_RECORD_KEY_FOOTER));
- assertTrue(orcReader.getMetadataKeys().contains(HOODIE_MAX_RECORD_KEY_FOOTER));
+ assertTrue(orcReader.getMetadataKeys().contains(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER));
+ assertTrue(orcReader.getMetadataKeys().contains(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER));
assertTrue(orcReader.getMetadataKeys().contains(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY));
assertTrue(orcReader.getMetadataKeys().contains(AVRO_SCHEMA_METADATA_KEY));
assertEquals(CompressionKind.ZLIB.name(), orcReader.getCompressionKind().toString());
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java
index 035cb2eab9..b939498c3e 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriteSupport.java
@@ -18,20 +18,18 @@
package org.apache.hudi.io.storage.row;
+import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
-import org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.logical.RowType;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hudi.common.util.Option;
import org.apache.parquet.hadoop.api.WriteSupport;
-import java.util.HashMap;
-
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Map;
/**
* Hoodie Write Support for directly writing {@link RowData} to Parquet.
@@ -39,14 +37,13 @@ import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_
public class HoodieRowDataParquetWriteSupport extends RowDataParquetWriteSupport {
private final Configuration hadoopConf;
- private final BloomFilter bloomFilter;
- private String minRecordKey;
- private String maxRecordKey;
+ private final Option<HoodieBloomFilterWriteSupport<String>> bloomFilterWriteSupportOpt;
public HoodieRowDataParquetWriteSupport(Configuration conf, RowType rowType, BloomFilter bloomFilter) {
super(rowType);
this.hadoopConf = new Configuration(conf);
- this.bloomFilter = bloomFilter;
+ this.bloomFilterWriteSupportOpt = Option.ofNullable(bloomFilter)
+ .map(HoodieBloomFilterRowDataWriteSupport::new);
}
public Configuration getHadoopConf() {
@@ -55,32 +52,26 @@ public class HoodieRowDataParquetWriteSupport extends RowDataParquetWriteSupport
@Override
public WriteSupport.FinalizedWriteContext finalizeWrite() {
- HashMap<String, String> extraMetaData = new HashMap<>();
- if (bloomFilter != null) {
- extraMetaData.put(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY, bloomFilter.serializeToString());
- if (minRecordKey != null && maxRecordKey != null) {
- extraMetaData.put(HOODIE_MIN_RECORD_KEY_FOOTER, minRecordKey);
- extraMetaData.put(HOODIE_MAX_RECORD_KEY_FOOTER, maxRecordKey);
- }
- if (bloomFilter.getBloomFilterTypeCode().name().contains(HoodieDynamicBoundedBloomFilter.TYPE_CODE_PREFIX)) {
- extraMetaData.put(HOODIE_BLOOM_FILTER_TYPE_CODE, bloomFilter.getBloomFilterTypeCode().name());
- }
- }
- return new WriteSupport.FinalizedWriteContext(extraMetaData);
+ Map<String, String> extraMetadata =
+ bloomFilterWriteSupportOpt.map(HoodieBloomFilterWriteSupport::finalizeMetadata)
+ .orElse(Collections.emptyMap());
+
+ return new WriteSupport.FinalizedWriteContext(extraMetadata);
}
public void add(String recordKey) {
- this.bloomFilter.add(recordKey);
- if (minRecordKey != null) {
- minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey;
- } else {
- minRecordKey = recordKey;
+ this.bloomFilterWriteSupportOpt.ifPresent(bloomFilterWriteSupport ->
+ bloomFilterWriteSupport.addKey(recordKey));
+ }
+
+ private static class HoodieBloomFilterRowDataWriteSupport extends HoodieBloomFilterWriteSupport<String> {
+ public HoodieBloomFilterRowDataWriteSupport(BloomFilter bloomFilter) {
+ super(bloomFilter);
}
- if (maxRecordKey != null) {
- maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey;
- } else {
- maxRecordKey = recordKey;
+ @Override
+ protected byte[] getUTF8Bytes(String key) {
+ return key.getBytes(StandardCharsets.UTF_8);
}
}
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
index 28964ecc3f..bb4dd9c619 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetWriteSupport.java
@@ -19,8 +19,8 @@
package org.apache.hudi.io.storage.row;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
-import org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.parquet.hadoop.api.WriteSupport;
@@ -28,12 +28,8 @@ import org.apache.spark.sql.execution.datasources.parquet.ParquetWriteSupport;
import org.apache.spark.sql.types.StructType;
import org.apache.spark.unsafe.types.UTF8String;
-import java.util.HashMap;
-
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER;
-import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER;
+import java.util.Collections;
+import java.util.Map;
/**
* Hoodie Write Support for directly writing Row to Parquet.
@@ -41,19 +37,17 @@ import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_
public class HoodieRowParquetWriteSupport extends ParquetWriteSupport {
private final Configuration hadoopConf;
- private final BloomFilter bloomFilter;
-
- private UTF8String minRecordKey;
- private UTF8String maxRecordKey;
+ private final Option<HoodieBloomFilterWriteSupport<UTF8String>> bloomFilterWriteSupportOpt;
public HoodieRowParquetWriteSupport(Configuration conf, StructType structType, Option<BloomFilter> bloomFilterOpt, HoodieWriteConfig writeConfig) {
Configuration hadoopConf = new Configuration(conf);
hadoopConf.set("spark.sql.parquet.writeLegacyFormat", writeConfig.parquetWriteLegacyFormatEnabled());
hadoopConf.set("spark.sql.parquet.outputTimestampType", writeConfig.parquetOutputTimestampType());
hadoopConf.set("spark.sql.parquet.fieldId.write.enabled", writeConfig.parquetFieldIdWriteEnabled());
- this.hadoopConf = hadoopConf;
setSchema(structType, hadoopConf);
- this.bloomFilter = bloomFilterOpt.orElse(null);
+
+ this.hadoopConf = hadoopConf;
+ this.bloomFilterWriteSupportOpt = bloomFilterOpt.map(HoodieBloomFilterRowWriteSupport::new);
}
public Configuration getHadoopConf() {
@@ -62,32 +56,35 @@ public class HoodieRowParquetWriteSupport extends ParquetWriteSupport {
@Override
public WriteSupport.FinalizedWriteContext finalizeWrite() {
- HashMap<String, String> extraMetaData = new HashMap<>();
- if (bloomFilter != null) {
- extraMetaData.put(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY, bloomFilter.serializeToString());
- if (minRecordKey != null && maxRecordKey != null) {
- extraMetaData.put(HOODIE_MIN_RECORD_KEY_FOOTER, minRecordKey.toString());
- extraMetaData.put(HOODIE_MAX_RECORD_KEY_FOOTER, maxRecordKey.toString());
- }
- if (bloomFilter.getBloomFilterTypeCode().name().contains(HoodieDynamicBoundedBloomFilter.TYPE_CODE_PREFIX)) {
- extraMetaData.put(HOODIE_BLOOM_FILTER_TYPE_CODE, bloomFilter.getBloomFilterTypeCode().name());
- }
- }
- return new WriteSupport.FinalizedWriteContext(extraMetaData);
+ Map<String, String> extraMetadata =
+ bloomFilterWriteSupportOpt.map(HoodieBloomFilterWriteSupport::finalizeMetadata)
+ .orElse(Collections.emptyMap());
+
+ return new WriteSupport.FinalizedWriteContext(extraMetadata);
}
public void add(UTF8String recordKey) {
- this.bloomFilter.add(recordKey.getBytes());
+ this.bloomFilterWriteSupportOpt.ifPresent(bloomFilterWriteSupport ->
+ bloomFilterWriteSupport.addKey(recordKey));
+ }
- if (minRecordKey == null || minRecordKey.compareTo(recordKey) < 0) {
+ private static class HoodieBloomFilterRowWriteSupport extends HoodieBloomFilterWriteSupport<UTF8String> {
+ public HoodieBloomFilterRowWriteSupport(BloomFilter bloomFilter) {
+ super(bloomFilter);
+ }
+
+ @Override
+ protected byte[] getUTF8Bytes(UTF8String key) {
+ return key.getBytes();
+ }
+
+ @Override
+ protected UTF8String dereference(UTF8String key) {
// NOTE: [[clone]] is performed here (rather than [[copy]]) to only copy underlying buffer in
// cases when [[UTF8String]] is pointing into a buffer storing the whole containing record,
// and simply do a pass over when it holds a (immutable) buffer holding just the string
- minRecordKey = recordKey.clone();
- }
-
- if (maxRecordKey == null || maxRecordKey.compareTo(recordKey) > 0) {
- maxRecordKey = recordKey.clone();
+ return key.clone();
}
}
+
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
index 481cda00d6..dce0e2fad5 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
@@ -18,40 +18,44 @@
package org.apache.hudi.io.storage.row;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
+import org.apache.hudi.common.bloom.BloomFilterTypeCode;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.testutils.HoodieClientTestHarness;
import org.apache.hudi.testutils.SparkDatasetTestUtils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.hadoop.metadata.FileMetaData;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.catalyst.InternalRow;
+import org.apache.spark.sql.types.StructType;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import java.util.Comparator;
import java.util.List;
-import java.util.Random;
-import java.util.UUID;
+import java.util.Map;
+import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Unit tests {@link HoodieInternalRowParquetWriter}.
*/
public class TestHoodieInternalRowParquetWriter extends HoodieClientTestHarness {
- private static final Random RANDOM = new Random();
-
@BeforeEach
public void setUp() throws Exception {
initSparkContexts("TestHoodieInternalRowParquetWriter");
@@ -68,44 +72,55 @@ public class TestHoodieInternalRowParquetWriter extends HoodieClientTestHarness
@ParameterizedTest
@ValueSource(booleans = {true, false})
- public void endToEndTest(boolean parquetWriteLegacyFormatEnabled) throws Exception {
+ public void testProperWriting(boolean parquetWriteLegacyFormatEnabled) throws Exception {
+ // Generate inputs
+ Dataset<Row> inputRows = SparkDatasetTestUtils.getRandomRows(sqlContext, 100,
+ HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, false);
+ StructType schema = inputRows.schema();
+
+ List<InternalRow> rows = SparkDatasetTestUtils.toInternalRows(inputRows, SparkDatasetTestUtils.ENCODER);
+
HoodieWriteConfig.Builder writeConfigBuilder =
SparkDatasetTestUtils.getConfigBuilder(basePath, timelineServicePort);
- for (int i = 0; i < 5; i++) {
- // init write support and parquet config
- HoodieRowParquetWriteSupport writeSupport = getWriteSupport(writeConfigBuilder, hadoopConf, parquetWriteLegacyFormatEnabled);
- HoodieWriteConfig cfg = writeConfigBuilder.build();
- HoodieParquetConfig<HoodieRowParquetWriteSupport> parquetConfig = new HoodieParquetConfig<>(writeSupport,
- CompressionCodecName.SNAPPY, cfg.getParquetBlockSize(), cfg.getParquetPageSize(), cfg.getParquetMaxFileSize(),
- writeSupport.getHadoopConf(), cfg.getParquetCompressionRatio(), cfg.parquetDictionaryEnabled());
-
- // prepare path
- String fileId = UUID.randomUUID().toString();
- Path filePath = new Path(basePath + "/" + fileId);
- String partitionPath = HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH;
- metaClient.getFs().mkdirs(new Path(basePath));
-
- // init writer
- HoodieInternalRowParquetWriter writer = new HoodieInternalRowParquetWriter(filePath, parquetConfig);
-
- // generate input
- int size = 10 + RANDOM.nextInt(100);
- // Generate inputs
- Dataset<Row> inputRows = SparkDatasetTestUtils.getRandomRows(sqlContext, size, partitionPath, false);
- List<InternalRow> internalRows = SparkDatasetTestUtils.toInternalRows(inputRows, SparkDatasetTestUtils.ENCODER);
-
- // issue writes
- for (InternalRow internalRow : internalRows) {
- writer.write(internalRow);
- }
- // close the writer
- writer.close();
+ HoodieRowParquetWriteSupport writeSupport = getWriteSupport(writeConfigBuilder, hadoopConf, parquetWriteLegacyFormatEnabled);
+ HoodieWriteConfig cfg = writeConfigBuilder.build();
+ HoodieParquetConfig<HoodieRowParquetWriteSupport> parquetConfig = new HoodieParquetConfig<>(writeSupport,
+ CompressionCodecName.SNAPPY, cfg.getParquetBlockSize(), cfg.getParquetPageSize(), cfg.getParquetMaxFileSize(),
+ writeSupport.getHadoopConf(), cfg.getParquetCompressionRatio(), cfg.parquetDictionaryEnabled());
- // verify rows
- Dataset<Row> result = sqlContext.read().parquet(basePath);
- assertEquals(0, inputRows.except(result).count());
+ Path filePath = new Path(basePath + "/internal_row_writer.parquet");
+
+ try (HoodieInternalRowParquetWriter writer = new HoodieInternalRowParquetWriter(filePath, parquetConfig)) {
+ for (InternalRow row : rows) {
+ writer.writeRow(row.getUTF8String(schema.fieldIndex("record_key")), row);
+ }
}
+
+ // Step 1: Verify rows written correctly
+ Dataset<Row> result = sqlContext.read().parquet(basePath);
+ assertEquals(0, inputRows.except(result).count());
+
+ // Step 2: Assert Parquet metadata was written appropriately
+ List<String> recordKeys =
+ rows.stream().map(r -> r.getString(schema.fieldIndex("record_key"))).collect(Collectors.toList());
+
+ String minKey = recordKeys.stream().min(Comparator.naturalOrder()).get();
+ String maxKey = recordKeys.stream().max(Comparator.naturalOrder()).get();
+
+ FileMetaData parquetMetadata = ParquetUtils.readMetadata(hadoopConf, filePath).getFileMetaData();
+
+ Map<String, String> extraMetadata = parquetMetadata.getKeyValueMetaData();
+
+ assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER), minKey);
+ assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER), maxKey);
+ assertEquals(extraMetadata.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE), BloomFilterTypeCode.DYNAMIC_V0.name());
+
+ // Step 3: Make sure Bloom Filter contains all the record keys
+ BloomFilter bloomFilter = new ParquetUtils().readBloomFilterFromMetadata(hadoopConf, filePath);
+ recordKeys.forEach(recordKey -> {
+ assertTrue(bloomFilter.mightContain(recordKey));
+ });
}
private HoodieRowParquetWriteSupport getWriteSupport(HoodieWriteConfig.Builder writeConfigBuilder, Configuration hadoopConf, boolean parquetWriteLegacyFormatEnabled) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
index c3920211ae..e87364fb90 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
@@ -20,12 +20,14 @@ package org.apache.hudi.avro;
import org.apache.avro.Schema;
import org.apache.hudi.common.bloom.BloomFilter;
-import org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter;
+import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.parquet.avro.AvroWriteSupport;
import org.apache.parquet.hadoop.api.WriteSupport;
import org.apache.parquet.schema.MessageType;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -34,55 +36,45 @@ import java.util.Map;
*/
public class HoodieAvroWriteSupport extends AvroWriteSupport {
- private Option<BloomFilter> bloomFilterOpt;
- private String minRecordKey;
- private String maxRecordKey;
- private Map<String, String> footerMetadata = new HashMap<>();
+ private final Option<HoodieBloomFilterWriteSupport<String>> bloomFilterWriteSupportOpt;
+ private final Map<String, String> footerMetadata = new HashMap<>();
public static final String OLD_HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY = "com.uber.hoodie.bloomfilter";
public static final String HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY = "org.apache.hudi.bloomfilter";
- public static final String HOODIE_MIN_RECORD_KEY_FOOTER = "hoodie_min_record_key";
- public static final String HOODIE_MAX_RECORD_KEY_FOOTER = "hoodie_max_record_key";
- public static final String HOODIE_BLOOM_FILTER_TYPE_CODE = "hoodie_bloom_filter_type_code";
public HoodieAvroWriteSupport(MessageType schema, Schema avroSchema, Option<BloomFilter> bloomFilterOpt) {
super(schema, avroSchema, ConvertingGenericData.INSTANCE);
- this.bloomFilterOpt = bloomFilterOpt;
+ this.bloomFilterWriteSupportOpt = bloomFilterOpt.map(HoodieBloomFilterAvroWriteSupport::new);
}
@Override
public WriteSupport.FinalizedWriteContext finalizeWrite() {
- if (bloomFilterOpt.isPresent()) {
- footerMetadata.put(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY, bloomFilterOpt.get().serializeToString());
- if (minRecordKey != null && maxRecordKey != null) {
- footerMetadata.put(HOODIE_MIN_RECORD_KEY_FOOTER, minRecordKey);
- footerMetadata.put(HOODIE_MAX_RECORD_KEY_FOOTER, maxRecordKey);
- }
- if (bloomFilterOpt.get().getBloomFilterTypeCode().name().contains(HoodieDynamicBoundedBloomFilter.TYPE_CODE_PREFIX)) {
- footerMetadata.put(HOODIE_BLOOM_FILTER_TYPE_CODE, bloomFilterOpt.get().getBloomFilterTypeCode().name());
- }
- }
- return new WriteSupport.FinalizedWriteContext(footerMetadata);
+ Map<String, String> extraMetadata =
+ CollectionUtils.combine(footerMetadata,
+ bloomFilterWriteSupportOpt.map(HoodieBloomFilterWriteSupport::finalizeMetadata)
+ .orElse(Collections.emptyMap())
+ );
+
+ return new WriteSupport.FinalizedWriteContext(extraMetadata);
}
public void add(String recordKey) {
- if (bloomFilterOpt.isPresent()) {
- this.bloomFilterOpt.get().add(recordKey);
- if (minRecordKey != null) {
- minRecordKey = minRecordKey.compareTo(recordKey) <= 0 ? minRecordKey : recordKey;
- } else {
- minRecordKey = recordKey;
- }
-
- if (maxRecordKey != null) {
- maxRecordKey = maxRecordKey.compareTo(recordKey) >= 0 ? maxRecordKey : recordKey;
- } else {
- maxRecordKey = recordKey;
- }
- }
+ this.bloomFilterWriteSupportOpt.ifPresent(bloomFilterWriteSupport ->
+ bloomFilterWriteSupport.addKey(recordKey));
}
public void addFooterMetadata(String key, String value) {
footerMetadata.put(key, value);
}
+
+ private static class HoodieBloomFilterAvroWriteSupport extends HoodieBloomFilterWriteSupport<String> {
+ public HoodieBloomFilterAvroWriteSupport(BloomFilter bloomFilter) {
+ super(bloomFilter);
+ }
+
+ @Override
+ protected byte[] getUTF8Bytes(String key) {
+ return key.getBytes(StandardCharsets.UTF_8);
+ }
+ }
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieBloomFilterWriteSupport.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieBloomFilterWriteSupport.java
new file mode 100644
index 0000000000..1a689791ba
--- /dev/null
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieBloomFilterWriteSupport.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.avro;
+
+import org.apache.hudi.common.bloom.BloomFilter;
+import org.apache.hudi.common.bloom.HoodieDynamicBoundedBloomFilter;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.hudi.avro.HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY;
+
+/**
+ * This is write-support utility base-class taking up handling of
+ *
+ * <ul>
+ * <li>Adding record keys to the Bloom Filter</li>
+ * <li>Keeping track of min/max record key (w/in single file)</li>
+ * </ul>
+ *
+ * @param <T> record-key type being ingested by this clas
+ */
+public abstract class HoodieBloomFilterWriteSupport<T extends Comparable<T>> {
+
+ public static final String HOODIE_MIN_RECORD_KEY_FOOTER = "hoodie_min_record_key";
+ public static final String HOODIE_MAX_RECORD_KEY_FOOTER = "hoodie_max_record_key";
+ public static final String HOODIE_BLOOM_FILTER_TYPE_CODE = "hoodie_bloom_filter_type_code";
+
+ private final BloomFilter bloomFilter;
+
+ private T minRecordKey;
+ private T maxRecordKey;
+
+ public HoodieBloomFilterWriteSupport(BloomFilter bloomFilter) {
+ this.bloomFilter = bloomFilter;
+ }
+
+ public void addKey(T recordKey) {
+ bloomFilter.add(getUTF8Bytes(recordKey));
+
+ if (minRecordKey == null || minRecordKey.compareTo(recordKey) > 0) {
+ minRecordKey = dereference(recordKey);
+ }
+
+ if (maxRecordKey == null || maxRecordKey.compareTo(recordKey) < 0) {
+ maxRecordKey = dereference(recordKey);
+ }
+ }
+
+ public Map<String, String> finalizeMetadata() {
+ HashMap<String, String> extraMetadata = new HashMap<>();
+
+ extraMetadata.put(HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY, bloomFilter.serializeToString());
+ if (bloomFilter.getBloomFilterTypeCode().name().contains(HoodieDynamicBoundedBloomFilter.TYPE_CODE_PREFIX)) {
+ extraMetadata.put(HOODIE_BLOOM_FILTER_TYPE_CODE, bloomFilter.getBloomFilterTypeCode().name());
+ }
+
+ if (minRecordKey != null && maxRecordKey != null) {
+ extraMetadata.put(HOODIE_MIN_RECORD_KEY_FOOTER, minRecordKey.toString());
+ extraMetadata.put(HOODIE_MAX_RECORD_KEY_FOOTER, maxRecordKey.toString());
+ }
+
+ return extraMetadata;
+ }
+
+ /**
+ * Since Bloom Filter ingests record-keys represented as UTF8 encoded byte string,
+ * this method have to be implemented for converting the original record key into one
+ */
+ protected abstract byte[] getUTF8Bytes(T key);
+
+ /**
+ * This method allows to dereference the key object (t/h cloning, for ex) that might be
+ * pointing at a shared mutable buffer, to make sure that we're not keeping references
+ * to mutable objects
+ */
+ protected T dereference(T key) {
+ return key;
+ }
+}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java
index d6391d178e..badb5e37a7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/BaseFileUtils.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.util;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
+import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
@@ -83,7 +84,7 @@ public abstract class BaseFileUtils {
readFooter(configuration, false, filePath,
HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY,
HoodieAvroWriteSupport.OLD_HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY,
- HoodieAvroWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE);
+ HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE);
String footerVal = footerVals.get(HoodieAvroWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY);
if (null == footerVal) {
// We use old style key "com.uber.hoodie.bloomfilter"
@@ -91,9 +92,9 @@ public abstract class BaseFileUtils {
}
BloomFilter toReturn = null;
if (footerVal != null) {
- if (footerVals.containsKey(HoodieAvroWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE)) {
+ if (footerVals.containsKey(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE)) {
toReturn = BloomFilterFactory.fromString(footerVal,
- footerVals.get(HoodieAvroWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE));
+ footerVals.get(HoodieBloomFilterWriteSupport.HOODIE_BLOOM_FILTER_TYPE_CODE));
} else {
toReturn = BloomFilterFactory.fromString(footerVal, BloomFilterTypeCode.SIMPLE.name());
}
@@ -109,14 +110,14 @@ public abstract class BaseFileUtils {
*/
public String[] readMinMaxRecordKeys(Configuration configuration, Path filePath) {
Map<String, String> minMaxKeys = readFooter(configuration, true, filePath,
- HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER, HoodieAvroWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER);
+ HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER, HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER);
if (minMaxKeys.size() != 2) {
throw new HoodieException(
String.format("Could not read min/max record key out of footer correctly from %s. read) : %s",
filePath, minMaxKeys));
}
- return new String[] {minMaxKeys.get(HoodieAvroWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER),
- minMaxKeys.get(HoodieAvroWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER)};
+ return new String[] {minMaxKeys.get(HoodieBloomFilterWriteSupport.HOODIE_MIN_RECORD_KEY_FOOTER),
+ minMaxKeys.get(HoodieBloomFilterWriteSupport.HOODIE_MAX_RECORD_KEY_FOOTER)};
}
/**
diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java
deleted file mode 100644
index 16a77c145c..0000000000
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroWriteSupport.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.avro;
-
-import org.apache.hudi.common.bloom.BloomFilter;
-import org.apache.hudi.common.bloom.BloomFilterFactory;
-import org.apache.hudi.common.bloom.BloomFilterTypeCode;
-import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.util.Option;
-
-import org.apache.avro.Schema;
-import org.apache.avro.generic.GenericData;
-import org.apache.avro.generic.GenericRecord;
-import org.apache.hadoop.fs.Path;
-import org.apache.parquet.avro.AvroSchemaConverter;
-import org.apache.parquet.hadoop.ParquetWriter;
-import org.apache.parquet.hadoop.metadata.CompressionCodecName;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-public class TestHoodieAvroWriteSupport {
-
- @Test
- public void testAddKey(@TempDir java.nio.file.Path tempDir) throws IOException {
- List<String> rowKeys = new ArrayList<>();
- for (int i = 0; i < 1000; i++) {
- rowKeys.add(UUID.randomUUID().toString());
- }
- String filePath = tempDir.resolve("test.parquet").toAbsolutePath().toString();
- Schema schema = HoodieAvroUtils.getRecordKeySchema();
- BloomFilter filter = BloomFilterFactory.createBloomFilter(
- 1000, 0.0001, 10000,
- BloomFilterTypeCode.SIMPLE.name());
- HoodieAvroWriteSupport writeSupport = new HoodieAvroWriteSupport(
- new AvroSchemaConverter().convert(schema), schema, Option.of(filter));
- ParquetWriter writer = new ParquetWriter(new Path(filePath), writeSupport, CompressionCodecName.GZIP,
- 120 * 1024 * 1024, ParquetWriter.DEFAULT_PAGE_SIZE);
- for (String rowKey : rowKeys) {
- GenericRecord rec = new GenericData.Record(schema);
- rec.put(HoodieRecord.RECORD_KEY_METADATA_FIELD, rowKey);
- writer.write(rec);
- writeSupport.add(rowKey);
- }
- writer.close();
- }
-}
[hudi] 02/02: Bumping release candidate number 2
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a commit to branch release-0.12.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit baeff4331dd25742f8280553281b773bc5e570a5
Author: 喻兆靖 <yu...@bytedance.com>
AuthorDate: Sun Oct 9 11:56:52 2022 +0800
Bumping release candidate number 2
---
docker/hoodie/hadoop/base/pom.xml | 2 +-
docker/hoodie/hadoop/base_java11/pom.xml | 2 +-
docker/hoodie/hadoop/datanode/pom.xml | 2 +-
docker/hoodie/hadoop/historyserver/pom.xml | 2 +-
docker/hoodie/hadoop/hive_base/pom.xml | 2 +-
docker/hoodie/hadoop/namenode/pom.xml | 2 +-
docker/hoodie/hadoop/pom.xml | 2 +-
docker/hoodie/hadoop/prestobase/pom.xml | 2 +-
docker/hoodie/hadoop/spark_base/pom.xml | 2 +-
docker/hoodie/hadoop/sparkadhoc/pom.xml | 2 +-
docker/hoodie/hadoop/sparkmaster/pom.xml | 2 +-
docker/hoodie/hadoop/sparkworker/pom.xml | 2 +-
docker/hoodie/hadoop/trinobase/pom.xml | 2 +-
docker/hoodie/hadoop/trinocoordinator/pom.xml | 2 +-
docker/hoodie/hadoop/trinoworker/pom.xml | 2 +-
hudi-aws/pom.xml | 4 ++--
hudi-cli/pom.xml | 2 +-
hudi-client/hudi-client-common/pom.xml | 4 ++--
hudi-client/hudi-flink-client/pom.xml | 4 ++--
hudi-client/hudi-java-client/pom.xml | 4 ++--
hudi-client/hudi-spark-client/pom.xml | 4 ++--
hudi-client/pom.xml | 2 +-
hudi-common/pom.xml | 2 +-
hudi-examples/hudi-examples-common/pom.xml | 2 +-
hudi-examples/hudi-examples-flink/pom.xml | 2 +-
hudi-examples/hudi-examples-java/pom.xml | 2 +-
hudi-examples/hudi-examples-spark/pom.xml | 2 +-
hudi-examples/pom.xml | 2 +-
hudi-flink-datasource/hudi-flink/pom.xml | 4 ++--
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 4 ++--
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 4 ++--
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 4 ++--
hudi-flink-datasource/pom.xml | 4 ++--
hudi-gcp/pom.xml | 2 +-
hudi-hadoop-mr/pom.xml | 2 +-
hudi-integ-test/pom.xml | 2 +-
hudi-kafka-connect/pom.xml | 4 ++--
hudi-spark-datasource/hudi-spark-common/pom.xml | 4 ++--
hudi-spark-datasource/hudi-spark/pom.xml | 4 ++--
hudi-spark-datasource/hudi-spark2-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 4 ++--
hudi-spark-datasource/hudi-spark3-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 4 ++--
hudi-spark-datasource/hudi-spark3.2.x/pom.xml | 4 ++--
hudi-spark-datasource/hudi-spark3.2plus-common/pom.xml | 2 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 4 ++--
hudi-spark-datasource/pom.xml | 2 +-
hudi-sync/hudi-adb-sync/pom.xml | 2 +-
hudi-sync/hudi-datahub-sync/pom.xml | 2 +-
hudi-sync/hudi-hive-sync/pom.xml | 2 +-
hudi-sync/hudi-sync-common/pom.xml | 2 +-
hudi-sync/pom.xml | 2 +-
hudi-tests-common/pom.xml | 2 +-
hudi-timeline-service/pom.xml | 2 +-
hudi-utilities/pom.xml | 2 +-
packaging/hudi-aws-bundle/pom.xml | 2 +-
packaging/hudi-datahub-sync-bundle/pom.xml | 2 +-
packaging/hudi-flink-bundle/pom.xml | 2 +-
packaging/hudi-gcp-bundle/pom.xml | 2 +-
packaging/hudi-hadoop-mr-bundle/pom.xml | 2 +-
packaging/hudi-hive-sync-bundle/pom.xml | 2 +-
packaging/hudi-integ-test-bundle/pom.xml | 2 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 2 +-
packaging/hudi-presto-bundle/pom.xml | 2 +-
packaging/hudi-spark-bundle/pom.xml | 2 +-
packaging/hudi-timeline-server-bundle/pom.xml | 2 +-
packaging/hudi-trino-bundle/pom.xml | 2 +-
packaging/hudi-utilities-bundle/pom.xml | 2 +-
packaging/hudi-utilities-slim-bundle/pom.xml | 2 +-
pom.xml | 2 +-
70 files changed, 87 insertions(+), 87 deletions(-)
diff --git a/docker/hoodie/hadoop/base/pom.xml b/docker/hoodie/hadoop/base/pom.xml
index 39ceb4006b..8cbaa9fc06 100644
--- a/docker/hoodie/hadoop/base/pom.xml
+++ b/docker/hoodie/hadoop/base/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/base_java11/pom.xml b/docker/hoodie/hadoop/base_java11/pom.xml
index 40ad44d67f..85abe7f8ee 100644
--- a/docker/hoodie/hadoop/base_java11/pom.xml
+++ b/docker/hoodie/hadoop/base_java11/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/datanode/pom.xml b/docker/hoodie/hadoop/datanode/pom.xml
index 9913a8b8aa..74c08c56d0 100644
--- a/docker/hoodie/hadoop/datanode/pom.xml
+++ b/docker/hoodie/hadoop/datanode/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/historyserver/pom.xml b/docker/hoodie/hadoop/historyserver/pom.xml
index 40342983a0..3f2c72cad8 100644
--- a/docker/hoodie/hadoop/historyserver/pom.xml
+++ b/docker/hoodie/hadoop/historyserver/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/hive_base/pom.xml b/docker/hoodie/hadoop/hive_base/pom.xml
index 5e06832e72..c43f254326 100644
--- a/docker/hoodie/hadoop/hive_base/pom.xml
+++ b/docker/hoodie/hadoop/hive_base/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/namenode/pom.xml b/docker/hoodie/hadoop/namenode/pom.xml
index 549173d6a8..402f6e5da7 100644
--- a/docker/hoodie/hadoop/namenode/pom.xml
+++ b/docker/hoodie/hadoop/namenode/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/pom.xml b/docker/hoodie/hadoop/pom.xml
index 395a222edc..00e042105b 100644
--- a/docker/hoodie/hadoop/pom.xml
+++ b/docker/hoodie/hadoop/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/docker/hoodie/hadoop/prestobase/pom.xml b/docker/hoodie/hadoop/prestobase/pom.xml
index 40c6f30e90..6b3ece20a9 100644
--- a/docker/hoodie/hadoop/prestobase/pom.xml
+++ b/docker/hoodie/hadoop/prestobase/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/spark_base/pom.xml b/docker/hoodie/hadoop/spark_base/pom.xml
index e928b20271..82221cc420 100644
--- a/docker/hoodie/hadoop/spark_base/pom.xml
+++ b/docker/hoodie/hadoop/spark_base/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/sparkadhoc/pom.xml b/docker/hoodie/hadoop/sparkadhoc/pom.xml
index e043e11c04..1fc65b2e0b 100644
--- a/docker/hoodie/hadoop/sparkadhoc/pom.xml
+++ b/docker/hoodie/hadoop/sparkadhoc/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/sparkmaster/pom.xml b/docker/hoodie/hadoop/sparkmaster/pom.xml
index d1f4576272..19b51b6701 100644
--- a/docker/hoodie/hadoop/sparkmaster/pom.xml
+++ b/docker/hoodie/hadoop/sparkmaster/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/sparkworker/pom.xml b/docker/hoodie/hadoop/sparkworker/pom.xml
index f5076e9f18..6a57fa21b2 100644
--- a/docker/hoodie/hadoop/sparkworker/pom.xml
+++ b/docker/hoodie/hadoop/sparkworker/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/trinobase/pom.xml b/docker/hoodie/hadoop/trinobase/pom.xml
index c4117daae2..831ed8fe30 100644
--- a/docker/hoodie/hadoop/trinobase/pom.xml
+++ b/docker/hoodie/hadoop/trinobase/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/trinocoordinator/pom.xml b/docker/hoodie/hadoop/trinocoordinator/pom.xml
index 0d9cb7989c..f44f2808fe 100644
--- a/docker/hoodie/hadoop/trinocoordinator/pom.xml
+++ b/docker/hoodie/hadoop/trinocoordinator/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/docker/hoodie/hadoop/trinoworker/pom.xml b/docker/hoodie/hadoop/trinoworker/pom.xml
index f2e06de215..78b90cf960 100644
--- a/docker/hoodie/hadoop/trinoworker/pom.xml
+++ b/docker/hoodie/hadoop/trinoworker/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hudi-hadoop-docker</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<packaging>pom</packaging>
diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml
index 23e4b5daf8..5c1ad9fa2b 100644
--- a/hudi-aws/pom.xml
+++ b/hudi-aws/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-aws</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-aws</name>
<packaging>jar</packaging>
diff --git a/hudi-cli/pom.xml b/hudi-cli/pom.xml
index ee78bf24b0..affc246c02 100644
--- a/hudi-cli/pom.xml
+++ b/hudi-cli/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-client/hudi-client-common/pom.xml b/hudi-client/hudi-client-common/pom.xml
index 735b62957d..a4f72c4a78 100644
--- a/hudi-client/hudi-client-common/pom.xml
+++ b/hudi-client/hudi-client-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hudi-client</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-client-common</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-client-common</name>
<packaging>jar</packaging>
diff --git a/hudi-client/hudi-flink-client/pom.xml b/hudi-client/hudi-flink-client/pom.xml
index 705e09767a..291bc27a86 100644
--- a/hudi-client/hudi-flink-client/pom.xml
+++ b/hudi-client/hudi-flink-client/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>hudi-client</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-flink-client</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-flink-client</name>
<packaging>jar</packaging>
diff --git a/hudi-client/hudi-java-client/pom.xml b/hudi-client/hudi-java-client/pom.xml
index 5d65e1e3af..b4122df8a4 100644
--- a/hudi-client/hudi-java-client/pom.xml
+++ b/hudi-client/hudi-java-client/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hudi-client</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-java-client</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-java-client</name>
<packaging>jar</packaging>
diff --git a/hudi-client/hudi-spark-client/pom.xml b/hudi-client/hudi-spark-client/pom.xml
index a7ae3a7049..1f92542425 100644
--- a/hudi-client/hudi-spark-client/pom.xml
+++ b/hudi-client/hudi-spark-client/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hudi-client</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark-client</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark-client</name>
<packaging>jar</packaging>
diff --git a/hudi-client/pom.xml b/hudi-client/pom.xml
index ae6f5430b9..6d73a904ec 100644
--- a/hudi-client/pom.xml
+++ b/hudi-client/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml
index 9f149f464f..406b7bdd64 100644
--- a/hudi-common/pom.xml
+++ b/hudi-common/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-examples/hudi-examples-common/pom.xml b/hudi-examples/hudi-examples-common/pom.xml
index 03b524f501..ea6259b697 100644
--- a/hudi-examples/hudi-examples-common/pom.xml
+++ b/hudi-examples/hudi-examples-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-examples</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-examples/hudi-examples-flink/pom.xml b/hudi-examples/hudi-examples-flink/pom.xml
index fa2f70ee99..1f67a25c59 100644
--- a/hudi-examples/hudi-examples-flink/pom.xml
+++ b/hudi-examples/hudi-examples-flink/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-examples</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-examples/hudi-examples-java/pom.xml b/hudi-examples/hudi-examples-java/pom.xml
index 2c583ddab9..d5917e09a4 100644
--- a/hudi-examples/hudi-examples-java/pom.xml
+++ b/hudi-examples/hudi-examples-java/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-examples</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-examples/hudi-examples-spark/pom.xml b/hudi-examples/hudi-examples-spark/pom.xml
index 4eeb11ecb1..12544f9eba 100644
--- a/hudi-examples/hudi-examples-spark/pom.xml
+++ b/hudi-examples/hudi-examples-spark/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-examples</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-examples/pom.xml b/hudi-examples/pom.xml
index b27a4df6e0..bfc929812f 100644
--- a/hudi-examples/pom.xml
+++ b/hudi-examples/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-flink-datasource/hudi-flink/pom.xml b/hudi-flink-datasource/hudi-flink/pom.xml
index 1d550bfe83..1156e2d57a 100644
--- a/hudi-flink-datasource/hudi-flink/pom.xml
+++ b/hudi-flink-datasource/hudi-flink/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hudi-flink-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-flink</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<packaging>jar</packaging>
<properties>
diff --git a/hudi-flink-datasource/hudi-flink1.13.x/pom.xml b/hudi-flink-datasource/hudi-flink1.13.x/pom.xml
index debaae350b..44301930c1 100644
--- a/hudi-flink-datasource/hudi-flink1.13.x/pom.xml
+++ b/hudi-flink-datasource/hudi-flink1.13.x/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>hudi-flink-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-flink1.13.x</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<packaging>jar</packaging>
<properties>
diff --git a/hudi-flink-datasource/hudi-flink1.14.x/pom.xml b/hudi-flink-datasource/hudi-flink1.14.x/pom.xml
index 4f51b7f736..e075fd7158 100644
--- a/hudi-flink-datasource/hudi-flink1.14.x/pom.xml
+++ b/hudi-flink-datasource/hudi-flink1.14.x/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>hudi-flink-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-flink1.14.x</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<packaging>jar</packaging>
<properties>
diff --git a/hudi-flink-datasource/hudi-flink1.15.x/pom.xml b/hudi-flink-datasource/hudi-flink1.15.x/pom.xml
index 51b37d81fe..1fbf0507d6 100644
--- a/hudi-flink-datasource/hudi-flink1.15.x/pom.xml
+++ b/hudi-flink-datasource/hudi-flink1.15.x/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>hudi-flink-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-flink1.15.x</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<packaging>jar</packaging>
<properties>
diff --git a/hudi-flink-datasource/pom.xml b/hudi-flink-datasource/pom.xml
index 57fc7fe2f2..464087c654 100644
--- a/hudi-flink-datasource/pom.xml
+++ b/hudi-flink-datasource/pom.xml
@@ -20,12 +20,12 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-flink-datasource</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<packaging>pom</packaging>
<properties>
diff --git a/hudi-gcp/pom.xml b/hudi-gcp/pom.xml
index 090f7b8d4b..65fe02c63f 100644
--- a/hudi-gcp/pom.xml
+++ b/hudi-gcp/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../pom.xml</relativePath>
</parent>
diff --git a/hudi-hadoop-mr/pom.xml b/hudi-hadoop-mr/pom.xml
index 451fd0ef27..ca1b31eb29 100644
--- a/hudi-hadoop-mr/pom.xml
+++ b/hudi-hadoop-mr/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-integ-test/pom.xml b/hudi-integ-test/pom.xml
index 2134f80bb0..abb25ed4ab 100644
--- a/hudi-integ-test/pom.xml
+++ b/hudi-integ-test/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>hudi-integ-test</artifactId>
diff --git a/hudi-kafka-connect/pom.xml b/hudi-kafka-connect/pom.xml
index 75d82ab93f..2d96a575ab 100644
--- a/hudi-kafka-connect/pom.xml
+++ b/hudi-kafka-connect/pom.xml
@@ -19,13 +19,13 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-kafka-connect</artifactId>
<description>Kafka Connect Sink Connector for Hudi</description>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<packaging>jar</packaging>
<properties>
diff --git a/hudi-spark-datasource/hudi-spark-common/pom.xml b/hudi-spark-datasource/hudi-spark-common/pom.xml
index a1016299ba..92b2d65f4b 100644
--- a/hudi-spark-datasource/hudi-spark-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark-common/pom.xml
@@ -17,12 +17,12 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark-common_${scala.binary.version}</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark-common_${scala.binary.version}</name>
<packaging>jar</packaging>
diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml
index f55cb3359c..b4cdd95467 100644
--- a/hudi-spark-datasource/hudi-spark/pom.xml
+++ b/hudi-spark-datasource/hudi-spark/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark_${scala.binary.version}</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark_${scala.binary.version}</name>
<packaging>jar</packaging>
diff --git a/hudi-spark-datasource/hudi-spark2-common/pom.xml b/hudi-spark-datasource/hudi-spark2-common/pom.xml
index bba57f85b7..a192eca792 100644
--- a/hudi-spark-datasource/hudi-spark2-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark2-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-spark-datasource/hudi-spark2/pom.xml b/hudi-spark-datasource/hudi-spark2/pom.xml
index f74dd96a5b..fc856ebb93 100644
--- a/hudi-spark-datasource/hudi-spark2/pom.xml
+++ b/hudi-spark-datasource/hudi-spark2/pom.xml
@@ -17,12 +17,12 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark2_${scala.binary.version}</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark2_${scala.binary.version}</name>
<packaging>jar</packaging>
diff --git a/hudi-spark-datasource/hudi-spark3-common/pom.xml b/hudi-spark-datasource/hudi-spark3-common/pom.xml
index 75957d6d4c..dbae9c0259 100644
--- a/hudi-spark-datasource/hudi-spark3-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
index 6768e0ce03..657166f893 100644
--- a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
@@ -17,12 +17,12 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark3.1.x_2.12</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark3.1.x_2.12</name>
<packaging>jar</packaging>
diff --git a/hudi-spark-datasource/hudi-spark3.2.x/pom.xml b/hudi-spark-datasource/hudi-spark3.2.x/pom.xml
index cd6ba3a4b5..7324f6933a 100644
--- a/hudi-spark-datasource/hudi-spark3.2.x/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3.2.x/pom.xml
@@ -17,12 +17,12 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark3.2.x_2.12</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark3.2.x_2.12</name>
<packaging>jar</packaging>
diff --git a/hudi-spark-datasource/hudi-spark3.2plus-common/pom.xml b/hudi-spark-datasource/hudi-spark3.2plus-common/pom.xml
index 513952dad4..6d977ce9a4 100644
--- a/hudi-spark-datasource/hudi-spark3.2plus-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3.2plus-common/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/pom.xml b/hudi-spark-datasource/hudi-spark3.3.x/pom.xml
index 9ab65dca2e..4901b0fd2b 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3.3.x/pom.xml
@@ -17,12 +17,12 @@
<parent>
<artifactId>hudi-spark-datasource</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hudi-spark3.3.x_2.12</artifactId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<name>hudi-spark3.3.x_2.12</name>
<packaging>jar</packaging>
diff --git a/hudi-spark-datasource/pom.xml b/hudi-spark-datasource/pom.xml
index 54f0dd8dd7..7f0f48bff4 100644
--- a/hudi-spark-datasource/pom.xml
+++ b/hudi-spark-datasource/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-sync/hudi-adb-sync/pom.xml b/hudi-sync/hudi-adb-sync/pom.xml
index 826262d36a..1a7df1ad8b 100644
--- a/hudi-sync/hudi-adb-sync/pom.xml
+++ b/hudi-sync/hudi-adb-sync/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/hudi-sync/hudi-datahub-sync/pom.xml b/hudi-sync/hudi-datahub-sync/pom.xml
index f40ecebbfc..dc796cdcb6 100644
--- a/hudi-sync/hudi-datahub-sync/pom.xml
+++ b/hudi-sync/hudi-datahub-sync/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/hudi-sync/hudi-hive-sync/pom.xml b/hudi-sync/hudi-hive-sync/pom.xml
index 7cf31550b6..a94a92215e 100644
--- a/hudi-sync/hudi-hive-sync/pom.xml
+++ b/hudi-sync/hudi-hive-sync/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
diff --git a/hudi-sync/hudi-sync-common/pom.xml b/hudi-sync/hudi-sync-common/pom.xml
index 7d43e51ee7..bd8b774313 100644
--- a/hudi-sync/hudi-sync-common/pom.xml
+++ b/hudi-sync/hudi-sync-common/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-sync/pom.xml b/hudi-sync/pom.xml
index c4687f868c..f5e298d72e 100644
--- a/hudi-sync/pom.xml
+++ b/hudi-sync/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-tests-common/pom.xml b/hudi-tests-common/pom.xml
index 76ccbea095..bbcb09e566 100644
--- a/hudi-tests-common/pom.xml
+++ b/hudi-tests-common/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-timeline-service/pom.xml b/hudi-timeline-service/pom.xml
index 2a392b7b01..ed800b357f 100644
--- a/hudi-timeline-service/pom.xml
+++ b/hudi-timeline-service/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml
index 0c2a612d78..c236f36846 100644
--- a/hudi-utilities/pom.xml
+++ b/hudi-utilities/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-aws-bundle/pom.xml b/packaging/hudi-aws-bundle/pom.xml
index 3edfe54847..3b40fa4a91 100644
--- a/packaging/hudi-aws-bundle/pom.xml
+++ b/packaging/hudi-aws-bundle/pom.xml
@@ -24,7 +24,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-datahub-sync-bundle/pom.xml b/packaging/hudi-datahub-sync-bundle/pom.xml
index ac45e41ce5..8b0666952f 100644
--- a/packaging/hudi-datahub-sync-bundle/pom.xml
+++ b/packaging/hudi-datahub-sync-bundle/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-flink-bundle/pom.xml b/packaging/hudi-flink-bundle/pom.xml
index 797cbce07e..b458303035 100644
--- a/packaging/hudi-flink-bundle/pom.xml
+++ b/packaging/hudi-flink-bundle/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-gcp-bundle/pom.xml b/packaging/hudi-gcp-bundle/pom.xml
index 59b5e404bd..4b116d1b7f 100644
--- a/packaging/hudi-gcp-bundle/pom.xml
+++ b/packaging/hudi-gcp-bundle/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-hadoop-mr-bundle/pom.xml b/packaging/hudi-hadoop-mr-bundle/pom.xml
index b2c6e61579..5b23252397 100644
--- a/packaging/hudi-hadoop-mr-bundle/pom.xml
+++ b/packaging/hudi-hadoop-mr-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-hive-sync-bundle/pom.xml b/packaging/hudi-hive-sync-bundle/pom.xml
index 51e11c29ab..c70be54cc3 100644
--- a/packaging/hudi-hive-sync-bundle/pom.xml
+++ b/packaging/hudi-hive-sync-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-integ-test-bundle/pom.xml b/packaging/hudi-integ-test-bundle/pom.xml
index d1789b863a..d167da110f 100644
--- a/packaging/hudi-integ-test-bundle/pom.xml
+++ b/packaging/hudi-integ-test-bundle/pom.xml
@@ -17,7 +17,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-kafka-connect-bundle/pom.xml b/packaging/hudi-kafka-connect-bundle/pom.xml
index 6a024ac02f..7268713b08 100644
--- a/packaging/hudi-kafka-connect-bundle/pom.xml
+++ b/packaging/hudi-kafka-connect-bundle/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-presto-bundle/pom.xml b/packaging/hudi-presto-bundle/pom.xml
index ab9be03983..7969b33fb4 100644
--- a/packaging/hudi-presto-bundle/pom.xml
+++ b/packaging/hudi-presto-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml
index fa149b9859..700f03284d 100644
--- a/packaging/hudi-spark-bundle/pom.xml
+++ b/packaging/hudi-spark-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-timeline-server-bundle/pom.xml b/packaging/hudi-timeline-server-bundle/pom.xml
index 8b9b251b89..e2e68419d9 100644
--- a/packaging/hudi-timeline-server-bundle/pom.xml
+++ b/packaging/hudi-timeline-server-bundle/pom.xml
@@ -21,7 +21,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-trino-bundle/pom.xml b/packaging/hudi-trino-bundle/pom.xml
index b0eeb0435e..f83f04c8ef 100644
--- a/packaging/hudi-trino-bundle/pom.xml
+++ b/packaging/hudi-trino-bundle/pom.xml
@@ -20,7 +20,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-utilities-bundle/pom.xml b/packaging/hudi-utilities-bundle/pom.xml
index b58729bdc9..c5de885be7 100644
--- a/packaging/hudi-utilities-bundle/pom.xml
+++ b/packaging/hudi-utilities-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/packaging/hudi-utilities-slim-bundle/pom.xml b/packaging/hudi-utilities-slim-bundle/pom.xml
index e98a70ac84..36560c1708 100644
--- a/packaging/hudi-utilities-slim-bundle/pom.xml
+++ b/packaging/hudi-utilities-slim-bundle/pom.xml
@@ -19,7 +19,7 @@
<parent>
<artifactId>hudi</artifactId>
<groupId>org.apache.hudi</groupId>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
diff --git a/pom.xml b/pom.xml
index 47e53fed97..1da28ec52d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.hudi</groupId>
<artifactId>hudi</artifactId>
<packaging>pom</packaging>
- <version>0.12.1</version>
+ <version>0.12.1-rc2</version>
<description>Apache Hudi brings stream style processing on big data</description>
<url>https://github.com/apache/hudi</url>
<name>Hudi</name>