You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2025/12/14 12:19:51 UTC
(hudi) 01/01: Add usages of HoodieAvroUtils
This is an automated email from the ASF dual-hosted git repository.
vhs pushed a commit to branch phase-18-HoodieAvroUtils-removal
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b9112642b9557b2a46373bc84bed7f93147cf3d9
Author: voon <vo...@gmail.com>
AuthorDate: Sun Dec 14 19:56:38 2025 +0800
Add usages of HoodieAvroUtils
---
.../apache/hudi/cli/commands/ExportCommand.java | 8 +--
.../apache/hudi/cli/commands/TestTableCommand.java | 2 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 2 +-
.../bootstrap/HoodieBootstrapSchemaProvider.java | 2 +-
.../SimpleSchemaConflictResolutionStrategy.java | 2 +-
.../utils/LegacyArchivedMetaEntryReader.java | 2 +-
.../org/apache/hudi/index/HoodieIndexUtils.java | 10 ++--
.../java/org/apache/hudi/io/HoodieCDCLogger.java | 2 +-
.../java/org/apache/hudi/keygen/KeyGenUtils.java | 8 +--
.../keygen/TimestampBasedAvroKeyGenerator.java | 2 +-
.../hudi/metadata/HoodieMetadataWriteUtils.java | 2 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 2 +-
...ConcurrentSchemaEvolutionTableSchemaGetter.java | 2 +-
.../keygen/RawTripTestPayloadKeyGenerator.java | 2 +-
.../GenericRecordValidationTestUtils.java | 2 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 6 +--
.../hudi/testutils/HoodieWriteableTestTable.java | 8 +--
.../hudi/client/model/HoodieFlinkRecord.java | 2 +-
.../hudi/util/SchemaEvolvingRowDataProjection.java | 2 +-
.../testutils/HoodieFlinkWriteableTestTable.java | 2 +-
.../run/strategy/JavaExecutionStrategy.java | 4 +-
.../TestJavaBulkInsertInternalPartitioner.java | 2 +-
.../HoodieSparkBootstrapSchemaProvider.java | 4 +-
.../MultipleSparkJobExecutionStrategy.java | 2 +-
...SparkJobConsistentHashingExecutionStrategy.java | 2 +-
.../client/utils/SparkMetadataWriterUtils.java | 2 +-
.../hudi/execution/SparkLazyInsertIterable.java | 2 +-
.../bulkinsert/RDDBucketIndexPartitioner.java | 2 +-
.../bootstrap/BaseBootstrapMetadataHandler.java | 4 +-
.../TestMarkerBasedRollbackStrategy.java | 2 +-
.../org/apache/hudi/avro/AvroRecordContext.java | 2 +-
.../org/apache/hudi/avro/AvroRecordSerializer.java | 4 +-
.../apache/hudi/avro/AvroSchemaCompatibility.java | 2 +-
.../apache/hudi/avro/MercifulJsonConverter.java | 2 +-
.../processors/DecimalLogicalTypeProcessor.java | 2 +-
.../apache/hudi/common/model/BaseAvroPayload.java | 2 +-
.../common/model/DefaultHoodieRecordPayload.java | 4 +-
.../hudi/common/model/HoodieAvroIndexedRecord.java | 12 ++---
.../hudi/common/model/HoodieAvroPayload.java | 6 +--
.../apache/hudi/common/model/HoodieAvroRecord.java | 12 ++---
.../common/model/PartialUpdateAvroPayload.java | 2 +-
.../common/model/SerializableIndexedRecord.java | 4 +-
.../debezium/AbstractDebeziumAvroPayload.java | 2 +-
.../apache/hudi/common/schema/HoodieSchema.java | 6 +--
.../hudi/common/schema/HoodieSchemaField.java | 6 +--
.../hudi/common/schema/HoodieSchemaUtils.java | 18 ++++---
.../hudi/common/table/TableSchemaResolver.java | 12 ++---
.../table/log/block/HoodieAvroDataBlock.java | 8 +--
.../common/table/log/block/HoodieDeleteBlock.java | 2 +-
.../common/table/read/PartialUpdateHandler.java | 2 +-
.../table/timeline/TimelineMetadataUtils.java | 4 +-
.../apache/hudi/common/util/FileFormatUtils.java | 4 +-
.../org/apache/hudi/common/util/SortUtils.java | 4 +-
.../apache/hudi/common/util/SpillableMapUtils.java | 4 +-
.../hudi/io/storage/HoodieAvroFileWriter.java | 4 +-
.../io/storage/HoodieAvroHFileReaderImplBase.java | 2 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 6 +--
.../main/java/org/apache/hudi/stats/ValueType.java | 8 +--
.../apache/parquet/schema/AvroSchemaRepair.java | 6 +--
.../hudi/avro/TestMercifulJsonConverter.java | 2 +-
...tOverwriteNonDefaultsWithLatestAvroPayload.java | 2 +-
.../common/model/TestPartialUpdateAvroPayload.java | 8 +--
.../common/model/TestSerializableIndexRecord.java | 4 +-
.../hudi/common/schema/TestHoodieSchemaUtils.java | 4 +-
.../table/read/TestHoodieFileGroupReaderBase.java | 6 +--
.../common/testutils/HoodieTestDataGenerator.java | 2 +-
.../hudi/common/testutils/SchemaTestUtil.java | 4 +-
.../testutils/reader/HoodieRecordTestPayload.java | 6 +--
.../hudi/common/util/TestObjectSizeCalculator.java | 4 +-
.../hudi/common/util/TestSerializableSchema.java | 6 +--
.../schema/utils/TestAvroSchemaEvolutionUtils.java | 6 +--
.../common/HoodieExampleDataGenerator.java | 2 +-
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 4 +-
.../java/org/apache/hudi/common/util/OrcUtils.java | 2 +-
.../org/apache/hudi/common/util/ParquetUtils.java | 4 +-
.../hudi/io/hadoop/HoodieAvroHFileWriter.java | 4 +-
.../apache/hudi/io/hadoop/HoodieAvroOrcReader.java | 4 +-
.../hudi/io/hadoop/HoodieAvroParquetReader.java | 2 +-
.../io/hadoop/HoodieAvroParquetReaderIterator.java | 2 +-
.../common/functional/TestHoodieLogFormat.java | 62 +++++++++++-----------
.../apache/hudi/common/util/TestParquetUtils.java | 4 +-
.../common/util/collection/TestBitCaskDiskMap.java | 6 +--
.../hudi/metadata/TestHoodieTableMetadataUtil.java | 2 +-
.../hudi/hadoop/avro/HoodieAvroParquetReader.java | 2 +-
.../realtime/RealtimeCompactedRecordReader.java | 2 +-
.../hudi/hadoop/utils/HiveAvroSerializer.java | 5 +-
.../utils/HoodieArrayWritableSchemaUtils.java | 12 ++---
.../hive/TestHoodieCombineHiveInputFormat.java | 2 +-
.../realtime/TestHoodieRealtimeRecordReader.java | 2 +-
.../hudi/hadoop/testutils/InputFormatTestUtil.java | 6 +--
.../utils/TestHoodieArrayWritableSchemaUtils.java | 6 +--
.../hudi/writers/TestAbstractConnectWriter.java | 4 +-
.../SparkFullBootstrapDataProviderBase.java | 2 +-
.../java/org/apache/hudi/TestDataSourceUtils.java | 20 +++----
.../hudi/client/TestTableSchemaEvolution.java | 2 +-
.../apache/hudi/functional/TestHoodieIndex.java | 2 +-
.../utilities/schema/KafkaOffsetPostProcessor.java | 2 +-
.../hudi/utilities/schema/LazyCastingIterator.java | 2 +-
.../DeleteSupportSchemaPostProcessor.java | 2 +-
.../DropColumnSchemaPostProcessor.java | 2 +-
.../add/AddPrimitiveColumnSchemaPostProcessor.java | 2 +-
.../sources/helpers/SanitizationUtils.java | 6 +--
.../utilities/streamer/HoodieStreamerUtils.java | 10 ++--
.../utilities/streamer/SourceFormatAdapter.java | 2 +-
.../apache/hudi/utilities/streamer/StreamSync.java | 2 +-
...TestHoodieDeltaStreamerSchemaEvolutionBase.java | 2 +-
.../utilities/sources/TestAvroKafkaSource.java | 4 +-
.../utilities/sources/TestGenericRddTransform.java | 2 +-
.../utilities/testutils/SanitizationTestUtils.java | 2 +-
109 files changed, 255 insertions(+), 252 deletions(-)
diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
index 8692efb8f6ba..bb85f73a6f73 100644
--- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
+++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java
@@ -176,7 +176,7 @@ public class ExportCommand {
continue;
}
final String outPath = localFolder + StoragePath.SEPARATOR + instantTime + "." + action;
- writeToFile(outPath, HoodieAvroUtils.avroToJson(metadata, true));
+ writeToFile(outPath, /*~~>*//*~~>*/HoodieAvroUtils.avroToJson(metadata, true));
}
}
}
@@ -203,7 +203,7 @@ public class ExportCommand {
switch (instant.getAction()) {
case HoodieTimeline.CLEAN_ACTION: {
HoodieCleanMetadata metadata = timeline.readCleanMetadata(instant);
- data = HoodieAvroUtils.avroToJson(metadata, true);
+ data = /*~~>*//*~~>*/HoodieAvroUtils.avroToJson(metadata, true);
break;
}
case HoodieTimeline.DELTA_COMMIT_ACTION:
@@ -215,12 +215,12 @@ public class ExportCommand {
}
case HoodieTimeline.ROLLBACK_ACTION: {
HoodieRollbackMetadata metadata = timeline.readRollbackMetadata(instant);
- data = HoodieAvroUtils.avroToJson(metadata, true);
+ data = /*~~>*//*~~>*/HoodieAvroUtils.avroToJson(metadata, true);
break;
}
case HoodieTimeline.SAVEPOINT_ACTION: {
HoodieSavepointMetadata metadata = timeline.readSavepointMetadata(instant);
- data = HoodieAvroUtils.avroToJson(metadata, true);
+ data = /*~~>*//*~~>*/HoodieAvroUtils.avroToJson(metadata, true);
break;
}
default: {
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java
index 061ba21f2b07..8d873e4b8348 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java
@@ -262,7 +262,7 @@ public class TestTableCommand extends CLIFunctionalTestHarness {
Schema actualSchema = new Schema.Parser().parse(actualSchemaStr);
Schema expectedSchema = new Schema.Parser().parse(schemaStr);
- expectedSchema = HoodieAvroUtils.addMetadataFields(expectedSchema);
+ expectedSchema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(expectedSchema);
assertEquals(actualSchema, expectedSchema);
File file = File.createTempFile("temp", null);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index 86efd7db1e08..6fa441f77bd6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -341,7 +341,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
FileBasedInternalSchemaStorageManager schemasManager = new FileBasedInternalSchemaStorageManager(table.getMetaClient());
if (!historySchemaStr.isEmpty() || Boolean.parseBoolean(config.getString(HoodieCommonConfig.RECONCILE_SCHEMA.key()))) {
InternalSchema internalSchema;
- Schema avroSchema = HoodieAvroUtils.createHoodieWriteSchema(config.getSchema(), config.allowOperationMetadataField());
+ Schema avroSchema = /*~~>*//*~~>*/HoodieAvroUtils.createHoodieWriteSchema(config.getSchema(), config.allowOperationMetadataField());
if (historySchemaStr.isEmpty()) {
internalSchema = SerDeHelper.fromJson(config.getInternalSchema()).orElseGet(() -> InternalSchemaConverter.convert(HoodieSchema.fromAvroSchema(avroSchema)));
internalSchema.setSchemaId(Long.parseLong(instantTime));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/HoodieBootstrapSchemaProvider.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/HoodieBootstrapSchemaProvider.java
index ce4c42ba1665..5658b38a944a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/HoodieBootstrapSchemaProvider.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/bootstrap/HoodieBootstrapSchemaProvider.java
@@ -49,7 +49,7 @@ public abstract class HoodieBootstrapSchemaProvider {
if (writeConfig.getSchema() != null) {
// Use schema specified by user if set
Schema userSchema = new Schema.Parser().parse(writeConfig.getSchema());
- if (!HoodieAvroUtils.getNullSchema().equals(userSchema)) {
+ if (!/*~~>*//*~~>*/HoodieAvroUtils.getNullSchema().equals(userSchema)) {
return userSchema;
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java
index 688f14c38d42..e196952c3f96 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/transaction/SimpleSchemaConflictResolutionStrategy.java
@@ -69,7 +69,7 @@ public class SimpleSchemaConflictResolutionStrategy implements SchemaConflictRes
Schema writerSchemaOfTxn = new Schema.Parser().parse(config.getWriteSchema());
// If a writer does not come with a meaningful schema, skip the schema resolution.
ConcurrentSchemaEvolutionTableSchemaGetter schemaResolver = new ConcurrentSchemaEvolutionTableSchemaGetter(table.getMetaClient());
- if (isSchemaNull(writerSchemaOfTxn)) {
+ if (/*~~>*/isSchemaNull(writerSchemaOfTxn)) {
return getTableSchemaAtInstant(schemaResolver, currTxnOwnerInstant.get());
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java
index 3c8493e08083..94452efd84b5 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/LegacyArchivedMetaEntryReader.java
@@ -100,7 +100,7 @@ public class LegacyArchivedMetaEntryReader {
Object actionData = record.get(key);
if (actionData != null) {
if (actionData instanceof IndexedRecord) {
- return HoodieAvroUtils.avroToFileBytes((IndexedRecord) actionData);
+ return /*~~>*//*~~>*/HoodieAvroUtils.avroToFileBytes((IndexedRecord) actionData);
} else {
// should be json bytes.
try {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
index dca188c8ed02..7988f494315f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/HoodieIndexUtils.java
@@ -138,7 +138,7 @@ public class HoodieIndexUtils {
*/
static boolean validateDataTypeForSecondaryIndex(List<String> sourceFields, Schema tableSchema) {
return sourceFields.stream().allMatch(fieldToIndex -> {
- Schema schema = getNestedFieldSchemaFromWriteSchema(tableSchema, fieldToIndex);
+ Schema schema = /*~~>*/getNestedFieldSchemaFromWriteSchema(tableSchema, fieldToIndex);
return isSecondaryIndexSupportedType(schema);
});
}
@@ -152,7 +152,7 @@ public class HoodieIndexUtils {
*/
public static boolean validateDataTypeForSecondaryOrExpressionIndex(List<String> sourceFields, Schema tableSchema) {
return sourceFields.stream().anyMatch(fieldToIndex -> {
- Schema schema = getNestedFieldSchemaFromWriteSchema(tableSchema, fieldToIndex);
+ Schema schema = /*~~>*/getNestedFieldSchemaFromWriteSchema(tableSchema, fieldToIndex);
return schema.getType() != Schema.Type.RECORD && schema.getType() != Schema.Type.ARRAY && schema.getType() != Schema.Type.MAP;
});
}
@@ -727,7 +727,7 @@ public class HoodieIndexUtils {
// First check if the field exists
try {
- getNestedFieldSchemaFromWriteSchema(tableSchema, columnName);
+ /*~~>*/getNestedFieldSchemaFromWriteSchema(tableSchema, columnName);
} catch (Exception e) {
throw new HoodieMetadataIndexException(String.format(
"Cannot create %s index '%s': Column '%s' does not exist in the table schema. "
@@ -738,7 +738,7 @@ public class HoodieIndexUtils {
// Check for complex types (RECORD, ARRAY, MAP) - not supported for any index type
if (!validateDataTypeForSecondaryOrExpressionIndex(sourceFields, tableSchema)) {
- Schema fieldSchema = getNestedFieldSchemaFromWriteSchema(tableSchema, columnName);
+ Schema fieldSchema = /*~~>*/getNestedFieldSchemaFromWriteSchema(tableSchema, columnName);
throw new HoodieMetadataIndexException(String.format(
"Cannot create %s index '%s': Column '%s' has unsupported data type '%s'. "
+ "Complex types (RECORD, ARRAY, MAP) are not supported for indexing. "
@@ -750,7 +750,7 @@ public class HoodieIndexUtils {
// For secondary index, apply stricter data type validation
if (indexType.equals(PARTITION_NAME_SECONDARY_INDEX)) {
if (!validateDataTypeForSecondaryIndex(sourceFields, tableSchema)) {
- Schema fieldSchema = getNestedFieldSchemaFromWriteSchema(tableSchema, columnName);
+ Schema fieldSchema = /*~~>*/getNestedFieldSchemaFromWriteSchema(tableSchema, columnName);
String actualType = fieldSchema.getType().toString();
if (fieldSchema.getLogicalType() != null) {
actualType += " with logical type " + fieldSchema.getLogicalType();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
index 4e7a65ab3dd9..1ee4d43d61ec 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCDCLogger.java
@@ -275,7 +275,7 @@ public class HoodieCDCLogger implements Closeable {
}
private GenericRecord removeCommitMetadata(GenericRecord record) {
- return record == null ? null : HoodieAvroUtils.rewriteRecordWithNewSchema(record, dataSchema.toAvroSchema(), Collections.emptyMap());
+ return record == null ? null : /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(record, dataSchema.toAvroSchema(), Collections.emptyMap());
}
// -------------------------------------------------------------------------
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java
index 6cd7d193b78a..05ecb3569f44 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/KeyGenUtils.java
@@ -225,7 +225,7 @@ public class KeyGenUtils {
public static String getRecordKey(GenericRecord record, List<String> recordKeyFields, boolean consistentLogicalTimestampEnabled) {
BiFunction<String, Integer, String> valueFunction = (recordKeyField, index) -> {
try {
- return HoodieAvroUtils.getNestedFieldValAsString(record, recordKeyField, false, consistentLogicalTimestampEnabled);
+ return /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, recordKeyField, false, consistentLogicalTimestampEnabled);
} catch (HoodieException e) {
throw new HoodieKeyException("Record key field '" + recordKeyField + "' does not exist in the input record");
}
@@ -242,7 +242,7 @@ public class KeyGenUtils {
StringBuilder partitionPath = new StringBuilder();
for (int i = 0; i < partitionPathFields.size(); i++) {
String partitionPathField = partitionPathFields.get(i);
- String fieldVal = HoodieAvroUtils.getNestedFieldValAsString(record, partitionPathField, true, consistentLogicalTimestampEnabled);
+ String fieldVal = /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, partitionPathField, true, consistentLogicalTimestampEnabled);
if (fieldVal == null || fieldVal.isEmpty()) {
if (hiveStylePartitioning) {
partitionPath.append(partitionPathField).append("=");
@@ -265,7 +265,7 @@ public class KeyGenUtils {
}
public static String getRecordKey(GenericRecord record, String recordKeyField, boolean consistentLogicalTimestampEnabled) {
- String recordKey = HoodieAvroUtils.getNestedFieldValAsString(record, recordKeyField, true, consistentLogicalTimestampEnabled);
+ String recordKey = /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, recordKeyField, true, consistentLogicalTimestampEnabled);
if (recordKey == null || recordKey.isEmpty()) {
throw new HoodieKeyException("recordKey value: \"" + recordKey + "\" for field: \"" + recordKeyField + "\" cannot be null or empty.");
}
@@ -274,7 +274,7 @@ public class KeyGenUtils {
public static String getPartitionPath(GenericRecord record, String partitionPathField,
boolean hiveStylePartitioning, boolean encodePartitionPath, boolean consistentLogicalTimestampEnabled) {
- String partitionPath = HoodieAvroUtils.getNestedFieldValAsString(record, partitionPathField, true, consistentLogicalTimestampEnabled);
+ String partitionPath = /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, partitionPathField, true, consistentLogicalTimestampEnabled);
if (partitionPath == null || partitionPath.isEmpty()) {
partitionPath = HUDI_DEFAULT_PARTITION_PATH;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
index b38a85cba6c9..3f4a31b2ad55 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/TimestampBasedAvroKeyGenerator.java
@@ -114,7 +114,7 @@ public class TimestampBasedAvroKeyGenerator extends SimpleAvroKeyGenerator {
@Override
public String getPartitionPath(GenericRecord record) {
- Object partitionVal = HoodieAvroUtils.getNestedFieldVal(record, getPartitionPathFields().get(0), true, isConsistentLogicalTimestampEnabled());
+ Object partitionVal = /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal(record, getPartitionPathFields().get(0), true, isConsistentLogicalTimestampEnabled());
if (partitionVal == null) {
partitionVal = getDefaultPartitionVal();
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
index c970e0d1361c..80142cef83b1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieMetadataWriteUtils.java
@@ -417,7 +417,7 @@ public class HoodieMetadataWriteUtils {
? Option.empty()
: Option.of(new Schema.Parser().parse(writerSchemaStr)));
HoodieTableConfig tableConfig = dataMetaClient.getTableConfig();
- Option<HoodieSchema> tableSchema = writerSchema.map(schema -> tableConfig.populateMetaFields() ? addMetadataFields(schema) : schema).map(HoodieSchema::fromAvroSchema);
+ Option<HoodieSchema> tableSchema = writerSchema.map(schema -> tableConfig.populateMetaFields() ? /*~~>*/addMetadataFields(schema) : schema).map(HoodieSchema::fromAvroSchema);
if (tableSchema.isEmpty()) {
return engineContext.emptyHoodieData();
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
index 523c1afd2919..4f513f1ad593 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
@@ -112,7 +112,7 @@ public class TestHoodieAvroParquetWriter {
private static List<String> toJson(List<GenericRecord> records) {
return records.stream().map(r -> {
try {
- return new String(HoodieAvroUtils.avroToJson(r, true));
+ return new String(/*~~>*//*~~>*/HoodieAvroUtils.avroToJson(r, true));
} catch (IOException e) {
throw new RuntimeException(e);
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java
index 4e02d74d1983..27291af43987 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestConcurrentSchemaEvolutionTableSchemaGetter.java
@@ -124,7 +124,7 @@ public class TestConcurrentSchemaEvolutionTableSchemaGetter extends HoodieCommon
+ "}";
private static Schema SCHEMA_WITHOUT_METADATA2 = new Schema.Parser().parse(SCHEMA_WITHOUT_METADATA_STR2);
private static Schema SCHEMA_WITHOUT_METADATA = new Schema.Parser().parse(SCHEMA_WITHOUT_METADATA_STR);
- private static Schema SCHEMA_WITH_METADATA = addMetadataFields(SCHEMA_WITHOUT_METADATA, false);
+ private static Schema SCHEMA_WITH_METADATA = /*~~>*/addMetadataFields(SCHEMA_WITHOUT_METADATA, false);
private static Schema SCHEMA_WITH_PARTITION_COLUMN = new Schema.Parser().parse(SCHEMA_WITH_PARTITION_COLUMN_STR);
@BeforeEach
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/RawTripTestPayloadKeyGenerator.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/RawTripTestPayloadKeyGenerator.java
index bfe2997706c4..54ab823092d1 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/RawTripTestPayloadKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/RawTripTestPayloadKeyGenerator.java
@@ -36,7 +36,7 @@ public class RawTripTestPayloadKeyGenerator extends SimpleAvroKeyGenerator {
@Override
public String getPartitionPath(GenericRecord record) {
- String partitionPath = HoodieAvroUtils
+ String partitionPath = /*~~>*//*~~>*/HoodieAvroUtils
.getNestedFieldValAsString(record, getPartitionPathFields().get(0), true, isConsistentLogicalTimestampEnabled());
return HoodieTestUtils.extractPartitionFromTimeField(partitionPath);
}
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
index 67d189ed1b1d..5959f892dc60 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/GenericRecordValidationTestUtils.java
@@ -126,7 +126,7 @@ public class GenericRecordValidationTestUtils {
// Verify row count.
assertEquals(prevRecordsMap.size(), newRecordsMap.size());
- Schema readerSchema = HoodieAvroUtils.addMetadataFields(
+ Schema readerSchema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(
new Schema.Parser().parse(config.getSchema()), config.allowOperationMetadataField());
// Verify every field.
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java
index cd90c8d2613a..cf621f0fd061 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java
@@ -85,8 +85,8 @@ public class HoodieMergeOnReadTestUtils {
String hiveColumnTypes;
if (populateMetaFields) {
- schema = HoodieAvroUtils.addMetadataFields(rawSchema);
- hiveColumnTypes = HoodieAvroUtils.addMetadataColumnTypes(rawHiveColumnTypes);
+ schema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(rawSchema);
+ hiveColumnTypes = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataColumnTypes(rawHiveColumnTypes);
} else {
schema = rawSchema;
hiveColumnTypes = rawHiveColumnTypes;
@@ -101,7 +101,7 @@ public class HoodieMergeOnReadTestUtils {
fields = schema.getFields();
}
final Schema projectedSchema = Schema.createRecord(fields.stream()
- .map(HoodieAvroUtils::createNewSchemaField)
+ .map(/*~~>*/HoodieAvroUtils::/*~~>*/createNewSchemaField)
.collect(Collectors.toList()));
List<GenericRecord> records = new ArrayList<>();
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
index 8d8fe7086900..afde87b20cfc 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
@@ -127,8 +127,8 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
for (HoodieRecord record : records) {
GenericRecord avroRecord = (GenericRecord) record.rewriteRecordWithNewSchema(schema, CollectionUtils.emptyProps(), schema).getData();
if (populateMetaFields) {
- HoodieAvroUtils.addCommitMetadataToRecord(avroRecord, currentInstantTime, String.valueOf(seqId++));
- HoodieAvroUtils.addHoodieKeyToRecord(avroRecord, record.getRecordKey(), record.getPartitionPath(), fileName);
+ /*~~>*//*~~>*/HoodieAvroUtils.addCommitMetadataToRecord(avroRecord, currentInstantTime, String.valueOf(seqId++));
+ /*~~>*//*~~>*/HoodieAvroUtils.addHoodieKeyToRecord(avroRecord, record.getRecordKey(), record.getPartitionPath(), fileName);
writer.writeAvro(record.getRecordKey(), avroRecord);
filter.ifPresent(f -> f.add(record.getRecordKey()));
} else {
@@ -149,8 +149,8 @@ public class HoodieWriteableTestTable extends HoodieMetadataTestTable {
int seqId = 1;
for (HoodieRecord record : records) {
GenericRecord avroRecord = (GenericRecord) record.toIndexedRecord(schema, CollectionUtils.emptyProps()).get().getData();
- HoodieAvroUtils.addCommitMetadataToRecord(avroRecord, currentInstantTime, String.valueOf(seqId++));
- HoodieAvroUtils.addHoodieKeyToRecord(avroRecord, record.getRecordKey(), record.getPartitionPath(), fileName);
+ /*~~>*//*~~>*/HoodieAvroUtils.addCommitMetadataToRecord(avroRecord, currentInstantTime, String.valueOf(seqId++));
+ /*~~>*//*~~>*/HoodieAvroUtils.addHoodieKeyToRecord(avroRecord, record.getRecordKey(), record.getPartitionPath(), fileName);
writer.writeAvro(record.getRecordKey(), avroRecord);
filter.ifPresent(f -> f.add(record.getRecordKey()));
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java
index 4ef1874f5d5b..d482677b3a94 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/client/model/HoodieFlinkRecord.java
@@ -293,6 +293,6 @@ public class HoodieFlinkRecord extends HoodieRecord<RowData> {
HoodieStorageConfig.WRITE_UTC_TIMEZONE.key(), HoodieStorageConfig.WRITE_UTC_TIMEZONE.defaultValue().toString()));
RowDataQueryContext rowDataQueryContext = RowDataAvroQueryContexts.fromAvroSchema(recordSchema, utcTimezone);
IndexedRecord indexedRecord = (IndexedRecord) rowDataQueryContext.getRowDataToAvroConverter().convert(recordSchema, getData());
- return HoodieAvroUtils.avroToBytesStream(indexedRecord);
+ return /*~~>*//*~~>*/HoodieAvroUtils.avroToBytesStream(indexedRecord);
}
}
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
index a0741ea67059..4ba699d0f83b 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/util/SchemaEvolvingRowDataProjection.java
@@ -135,7 +135,7 @@ public class SchemaEvolvingRowDataProjection implements RowProjection {
RowField field = toRowType.getFields().get(i);
fieldNameStack.push(field.getName());
// trying to get origin name for the field, if not found in mapping, no renaming happens
- String fieldName = renamedColumns.getOrDefault(HoodieAvroUtils.createFullName(fieldNameStack), field.getName());
+ String fieldName = renamedColumns.getOrDefault(/*~~>*//*~~>*/HoodieAvroUtils.createFullName(fieldNameStack), field.getName());
int fieldIdxInFrom = fromFieldNames.indexOf(fieldName);
// this is a new added column
if (fieldIdxInFrom == -1) {
diff --git a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java
index 1405057019a8..f66a90505d6d 100644
--- a/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java
+++ b/hudi-client/hudi-flink-client/src/test/java/org/apache/hudi/testutils/HoodieFlinkWriteableTestTable.java
@@ -147,7 +147,7 @@ public class HoodieFlinkWriteableTestTable extends HoodieWriteableTestTable {
try {
GenericRecord val =
(GenericRecord) ((HoodieRecordPayload) r.getData()).getInsertValue(schema).get();
- HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), "");
+ /*~~>*//*~~>*/HoodieAvroUtils.addHoodieKeyToRecord(val, r.getRecordKey(), r.getPartitionPath(), "");
return (IndexedRecord) val;
} catch (IOException e) {
log.warn("Failed to convert record " + r.toString(), e);
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
index bd6c3f298ff0..e2b3e64403af 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/JavaExecutionStrategy.java
@@ -113,7 +113,7 @@ public abstract class JavaExecutionStrategy<T>
if (strategyParams.containsKey(PLAN_STRATEGY_SORT_COLUMNS.key())) {
return new JavaCustomColumnsSortPartitioner(
strategyParams.get(PLAN_STRATEGY_SORT_COLUMNS.key()).split(","),
- HoodieAvroUtils.addMetadataFields(schema), getWriteConfig());
+ /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema), getWriteConfig());
} else {
return JavaBulkInsertInternalPartitionerFactory.get(getWriteConfig().getBulkInsertSortMode());
}
@@ -126,7 +126,7 @@ public abstract class JavaExecutionStrategy<T>
HoodieClusteringGroup clusteringGroup, Map<String, String> strategyParams,
boolean preserveHoodieMetadata, String instantTime) {
List<HoodieRecord<T>> inputRecords = readRecordsForGroup(clusteringGroup, instantTime);
- Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
+ Schema readerSchema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
List<HoodieFileGroupId> inputFileIds = clusteringGroup.getSlices().stream()
.map(info -> new HoodieFileGroupId(info.getPartitionPath(), info.getFileId()))
.collect(Collectors.toList());
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
index ff8909b85ddc..1df4a3d3f099 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestJavaBulkInsertInternalPartitioner.java
@@ -77,7 +77,7 @@ public class TestJavaBulkInsertInternalPartitioner extends HoodieJavaClientTestH
private Comparator<HoodieRecord> getCustomColumnComparator(Schema schema, String[] sortColumns) {
return Comparator.comparing(record ->
FlatLists.ofComparableArray(
- HoodieAvroUtils.getRecordColumnValues(record, sortColumns, schema, false)));
+ /*~~>*//*~~>*/HoodieAvroUtils.getRecordColumnValues(record, sortColumns, schema, false)));
}
private void verifyRecordAscendingOrder(List<HoodieRecord> records,
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/bootstrap/HoodieSparkBootstrapSchemaProvider.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/bootstrap/HoodieSparkBootstrapSchemaProvider.java
index b8aa55975a17..61fd0907bba3 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/bootstrap/HoodieSparkBootstrapSchemaProvider.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/bootstrap/HoodieSparkBootstrapSchemaProvider.java
@@ -81,7 +81,7 @@ public class HoodieSparkBootstrapSchemaProvider extends HoodieBootstrapSchemaPro
.option("basePath", writeConfig.getBootstrapSourceBasePath())
.parquet(filePath.toString())
.schema();
- String tableName = HoodieAvroUtils.sanitizeName(writeConfig.getTableName());
+ String tableName = /*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(writeConfig.getTableName());
String structName = tableName + "_record";
String recordNamespace = "hoodie." + tableName;
@@ -96,7 +96,7 @@ public class HoodieSparkBootstrapSchemaProvider extends HoodieBootstrapSchemaPro
throw new HoodieException("Could not determine schema from the ORC data files.");
}
TypeDescription orcSchema = orcReader.getSchema();
- String tableName = HoodieAvroUtils.sanitizeName(writeConfig.getTableName());
+ String tableName = /*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(writeConfig.getTableName());
String structName = tableName + "_record";
String recordNamespace = "hoodie." + tableName;
return AvroOrcUtils.createAvroSchemaWithDefaultValue(orcSchema, structName, recordNamespace, true);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
index 69d50c1367ca..97b6aef78f6b 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/MultipleSparkJobExecutionStrategy.java
@@ -230,7 +230,7 @@ public abstract class MultipleSparkJobExecutionStrategy<T>
return CompletableFuture.supplyAsync(() -> {
JavaSparkContext jsc = HoodieSparkEngineContext.getSparkContext(getEngineContext());
HoodieData<HoodieRecord<T>> inputRecords = readRecordsForGroup(jsc, clusteringGroup, instantTime);
- Schema readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
+ Schema readerSchema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(getWriteConfig().getSchema()));
// NOTE: Record have to be cloned here to make sure if it holds low-level engine-specific
// payload pointing into a shared, mutable (underlying) buffer we get a clean copy of
// it since these records will be shuffled later.
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java
index c753d83d3462..65bab60dd92f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/clustering/run/strategy/SingleSparkJobConsistentHashingExecutionStrategy.java
@@ -78,7 +78,7 @@ public class SingleSparkJobConsistentHashingExecutionStrategy<T> extends SingleS
HoodieWriteConfig writeConfig) {
super(table, engineContext, writeConfig);
this.indexKeyFields = table.getConfig().getBucketIndexHashField();
- this.readerSchema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(writeConfig.getSchema()));
+ this.readerSchema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(writeConfig.getSchema()));
}
@Override
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
index 9b1f36908b59..d0be6affb10f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/utils/SparkMetadataWriterUtils.java
@@ -392,7 +392,7 @@ public class SparkMetadataWriterUtils {
? Option.empty()
: Option.of(new Schema.Parser().parse(writerSchemaStr)));
HoodieTableConfig tableConfig = dataMetaClient.getTableConfig();
- HoodieSchema tableSchema = writerSchema.map(schema -> tableConfig.populateMetaFields() ? addMetadataFields(schema) : schema).map(HoodieSchema::fromAvroSchema)
+ HoodieSchema tableSchema = writerSchema.map(schema -> tableConfig.populateMetaFields() ? /*~~>*/addMetadataFields(schema) : schema).map(HoodieSchema::fromAvroSchema)
.orElseThrow(() -> new IllegalStateException(String.format("Expected writer schema in commit metadata %s", commitMetadata)));
List<Pair<String, HoodieSchema>> columnsToIndexSchemaMap = columnsToIndex.stream()
.map(columnToIndex -> HoodieSchemaUtils.getNestedField(tableSchema, columnToIndex))
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java
index 45e29a18110d..8325ea8d750f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/SparkLazyInsertIterable.java
@@ -71,7 +71,7 @@ public class SparkLazyInsertIterable<T> extends HoodieLazyInsertIterable<T> {
// config.getSchema is not canonicalized, while config.getWriteSchema is canonicalized. So, we have to use the canonicalized schema to read the existing data.
Schema schema = new Schema.Parser().parse(hoodieConfig.getWriteSchema());
if (useWriterSchema) {
- schema = HoodieAvroUtils.addMetadataFields(schema);
+ schema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema);
}
schema = AvroSchemaCache.intern(schema);
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java
index 8391f8f9c1a7..e0dbfc49ce1f 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/RDDBucketIndexPartitioner.java
@@ -87,7 +87,7 @@ public abstract class RDDBucketIndexPartitioner<T> extends BucketIndexBulkInsert
*/
private JavaRDD<HoodieRecord<T>> doPartitionAndCustomColumnSort(JavaRDD<HoodieRecord<T>> records, Partitioner partitioner) {
final String[] sortColumns = sortColumnNames;
- final SerializableSchema schema = new SerializableSchema(HoodieAvroUtils.addMetadataFields((new Schema.Parser().parse(table.getConfig().getSchema()))));
+ final SerializableSchema schema = new SerializableSchema(/*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields((new Schema.Parser().parse(table.getConfig().getSchema()))));
Comparator<HoodieRecord<T>> comparator = (Comparator<HoodieRecord<T>> & Serializable) (t1, t2) -> {
FlatLists.ComparableList obj1 = FlatLists.ofComparableArray(utf8StringFactory.wrapArrayOfObjects(
t1.getColumnValues(schema.get(), sortColumns, consistentLogicalTimestampEnabled)));
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java
index a36111c83419..087f1aad2f86 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/bootstrap/BaseBootstrapMetadataHandler.java
@@ -60,9 +60,9 @@ public abstract class BaseBootstrapMetadataHandler implements BootstrapMetadataH
try {
Schema avroSchema = getAvroSchema(sourceFilePath);
List<String> recordKeyColumns = keyGenerator.getRecordKeyFieldNames().stream()
- .map(HoodieAvroUtils::getRootLevelFieldName)
+ .map(/*~~>*/HoodieAvroUtils::/*~~>*/getRootLevelFieldName)
.collect(Collectors.toList());
- Schema recordKeySchema = HoodieAvroUtils.generateProjectionSchema(avroSchema, recordKeyColumns);
+ Schema recordKeySchema = /*~~>*//*~~>*/HoodieAvroUtils.generateProjectionSchema(avroSchema, recordKeyColumns);
LOG.info("Schema to be used for reading record keys: " + recordKeySchema);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java
index 271cea4574f1..0817d70a5190 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestMarkerBasedRollbackStrategy.java
@@ -317,7 +317,7 @@ public class TestMarkerBasedRollbackStrategy extends HoodieClientTestBase {
initMetaClient(tableType, props);
String partition = "partA";
HoodieSparkWriteableTestTable testTable = HoodieSparkWriteableTestTable.of(
- metaClient, addMetadataFields(HoodieTestUtils.SIMPLE_RECORD_SCHEMA.toAvroSchema()));
+ metaClient, /*~~>*/addMetadataFields(HoodieTestUtils.SIMPLE_RECORD_SCHEMA.toAvroSchema()));
String fileId = UUID.randomUUID().toString();
HoodieRecord tripRecord = createSimpleRecord("key1", "2016-01-31T03:16:41.415Z", 123);
String instantTime1 = "001";
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java
index 947a292196ae..d04a985ea9f5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordContext.java
@@ -182,7 +182,7 @@ public class AvroRecordContext extends RecordContext<IndexedRecord> {
@Override
public UnaryOperator<IndexedRecord> projectRecord(HoodieSchema from, HoodieSchema to, Map<String, String> renamedColumns) {
- return record -> HoodieAvroUtils.rewriteRecordWithNewSchema(record, to.toAvroSchema(), renamedColumns);
+ return record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(record, to.toAvroSchema(), renamedColumns);
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordSerializer.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordSerializer.java
index a7e8f20bba01..b8baa3ca3730 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordSerializer.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroRecordSerializer.java
@@ -39,13 +39,13 @@ public class AvroRecordSerializer implements RecordSerializer<IndexedRecord> {
@Override
public byte[] serialize(IndexedRecord input) {
- return HoodieAvroUtils.avroToBytes(input);
+ return /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(input);
}
@Override
public IndexedRecord deserialize(byte[] bytes, int schemaId) {
try {
- return HoodieAvroUtils.bytesToAvro(bytes, schemaFunc.apply(schemaId));
+ return /*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(bytes, schemaFunc.apply(schemaId));
} catch (IOException e) {
throw new HoodieException("Failed to deserialize Avro record bytes.",e);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
index a1ad79236e2b..dcb39d545654 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
@@ -415,7 +415,7 @@ public class AvroSchemaCompatibility {
return (writer.getType() == Type.STRING) ? result : result.mergedWith(typeMismatch(reader, writer, locations));
}
case STRING: {
- return (isTypeNumeric(writer.getType()) || (writer.getType() == Schema.Type.BYTES)
+ return (/*~~>*/isTypeNumeric(writer.getType()) || (writer.getType() == Schema.Type.BYTES)
? result : result.mergedWith(typeMismatch(reader, writer, locations)));
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
index f12a07c8a84e..c2a06ea67184 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/MercifulJsonConverter.java
@@ -146,7 +146,7 @@ public class MercifulJsonConverter {
for (String inputFieldName : inputJson.keySet()) {
// we expect many fields won't need sanitization so check if un-sanitized field name is already present
if (!schemaToJsonFieldNames.containsKey(inputFieldName)) {
- String sanitizedJsonFieldName = HoodieAvroUtils.sanitizeName(inputFieldName, invalidCharMask);
+ String sanitizedJsonFieldName = /*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(inputFieldName, invalidCharMask);
schemaToJsonFieldNames.putIfAbsent(sanitizedJsonFieldName, inputFieldName);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/processors/DecimalLogicalTypeProcessor.java b/hudi-common/src/main/java/org/apache/hudi/avro/processors/DecimalLogicalTypeProcessor.java
index 93e02dd49836..96983a9393fd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/processors/DecimalLogicalTypeProcessor.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/processors/DecimalLogicalTypeProcessor.java
@@ -63,7 +63,7 @@ public abstract class DecimalLogicalTypeProcessor extends JsonFieldProcessor {
// Case 2: Object is a number in String format.
try {
//encoded big decimal
- bigDecimal = HoodieAvroUtils.convertBytesToBigDecimal(decodeStringToBigDecimalBytes(obj),
+ bigDecimal = /*~~>*//*~~>*/HoodieAvroUtils.convertBytesToBigDecimal(decodeStringToBigDecimalBytes(obj),
(LogicalTypes.Decimal) schema.getLogicalType());
} catch (IllegalArgumentException e) {
//no-op
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java
index 6ce9f0b1b326..91b7a250cc4a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BaseAvroPayload.java
@@ -110,7 +110,7 @@ public abstract class BaseAvroPayload implements Serializable, KryoSerializable
if (record == null) {
recordBytes = new byte[0];
} else {
- recordBytes = HoodieAvroUtils.avroToBytes(record);
+ recordBytes = /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record);
}
}
return recordBytes;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/DefaultHoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/DefaultHoodieRecordPayload.java
index 5c9d04028a6b..95dabd900450 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/DefaultHoodieRecordPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/DefaultHoodieRecordPayload.java
@@ -170,10 +170,10 @@ public class DefaultHoodieRecordPayload extends OverwriteWithLatestAvroPayload {
KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue()));
Comparable persistedOrderingVal = OrderingValues.create(
orderingFields,
- field -> (Comparable) HoodieAvroUtils.getNestedFieldVal((GenericRecord) currentValue, field, true, consistentLogicalTimestampEnabled));
+ field -> (Comparable) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal((GenericRecord) currentValue, field, true, consistentLogicalTimestampEnabled));
Comparable incomingOrderingVal = incomingRecord.map(record -> OrderingValues.create(
orderingFields,
- field -> (Comparable) HoodieAvroUtils.getNestedFieldVal((GenericRecord) record, field, true, consistentLogicalTimestampEnabled)))
+ field -> (Comparable) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal((GenericRecord) record, field, true, consistentLogicalTimestampEnabled)))
.orElse(orderingVal);
// If the incoming record is a delete record without an ordering value, it is processed as "commit time" ordering.
if (incomingRecord.isEmpty() && OrderingValues.isDefault(incomingOrderingVal)) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
index 9416035afd34..6332c17d2934 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
@@ -195,7 +195,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
@Override
public HoodieRecord joinWith(HoodieRecord other, Schema targetSchema) {
decodeRecord(targetSchema);
- GenericRecord record = HoodieAvroUtils.stitchRecords((GenericRecord) data, (GenericRecord) other.getData(), targetSchema);
+ GenericRecord record = /*~~>*//*~~>*/HoodieAvroUtils.stitchRecords((GenericRecord) data, (GenericRecord) other.getData(), targetSchema);
return new HoodieAvroIndexedRecord(key, record, operation, metaData, orderingValue, isDelete);
}
@@ -222,7 +222,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
@Override
public HoodieRecord rewriteRecordWithNewSchema(Schema recordSchema, Properties props, Schema newSchema, Map<String, String> renameCols) {
decodeRecord(recordSchema);
- GenericRecord record = HoodieAvroUtils.rewriteRecordWithNewSchema(data, newSchema, renameCols);
+ GenericRecord record = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(data, newSchema, renameCols);
return new HoodieAvroIndexedRecord(key, record, operation, metaData, orderingValue, isDelete);
}
@@ -269,7 +269,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
String payloadClass = ConfigUtils.getPayloadClass(props);
String[] orderingFields = ConfigUtils.getOrderingFields(props);
decodeRecord(recordSchema);
- return HoodieAvroUtils.createHoodieRecordFromAvro(data, payloadClass, orderingFields, simpleKeyGenFieldsOpt, withOperation, partitionNameOp, populateMetaFields, schemaWithoutMetaFields);
+ return /*~~>*//*~~>*/HoodieAvroUtils.createHoodieRecordFromAvro(data, payloadClass, orderingFields, simpleKeyGenFieldsOpt, withOperation, partitionNameOp, populateMetaFields, schemaWithoutMetaFields);
}
@Override
@@ -311,7 +311,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue()));
return OrderingValues.create(
orderingFields,
- field -> (Comparable<?>) HoodieAvroUtils.getNestedFieldVal((GenericRecord) data, field, true, consistentLogicalTimestampEnabled));
+ field -> (Comparable<?>) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal((GenericRecord) data, field, true, consistentLogicalTimestampEnabled));
}
private void decodeRecord(Schema recordSchema) {
@@ -326,7 +326,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
@Override
public ByteArrayOutputStream getAvroBytes(Schema recordSchema, Properties props) {
- return HoodieAvroUtils.avroToBytesStream(data);
+ return /*~~>*//*~~>*/HoodieAvroUtils.avroToBytesStream(data);
}
/**
@@ -358,7 +358,7 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
public Object convertColumnValueForLogicalType(Schema fieldSchema,
Object fieldValue,
boolean keepConsistentLogicalTimestamp) {
- return HoodieAvroUtils.convertValueForAvroLogicalTypes(
+ return /*~~>*//*~~>*/HoodieAvroUtils.convertValueForAvroLogicalTypes(
fieldSchema, fieldValue, keepConsistentLogicalTimestamp);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java
index 3fbcb8a620e0..1d27e121ad4d 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroPayload.java
@@ -39,12 +39,12 @@ public class HoodieAvroPayload implements HoodieRecordPayload<HoodieAvroPayload>
private final Comparable<?> orderingVal;
public HoodieAvroPayload(GenericRecord record, Comparable<?> orderingVal) {
- this.recordBytes = record == null ? new byte[0] : HoodieAvroUtils.avroToBytes(record);
+ this.recordBytes = record == null ? new byte[0] : /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record);
this.orderingVal = orderingVal;
}
public HoodieAvroPayload(Option<GenericRecord> record) {
- this.recordBytes = record.isPresent() ? HoodieAvroUtils.avroToBytes(record.get()) : new byte[0];
+ this.recordBytes = record.isPresent() ? /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record.get()) : new byte[0];
this.orderingVal = 0;
}
@@ -63,7 +63,7 @@ public class HoodieAvroPayload implements HoodieRecordPayload<HoodieAvroPayload>
if (recordBytes.length == 0) {
return Option.empty();
}
- return Option.of(HoodieAvroUtils.bytesToAvro(recordBytes, schema));
+ return Option.of(/*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(recordBytes, schema));
}
// for examples
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
index 2b3822851feb..afff94002497 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroRecord.java
@@ -126,7 +126,7 @@ public class HoodieAvroRecord<T extends HoodieRecordPayload> extends HoodieRecor
@Override
public Object[] getColumnValues(Schema recordSchema, String[] columns, boolean consistentLogicalTimestampEnabled) {
- return HoodieAvroUtils.getRecordColumnValues(this, columns, recordSchema, consistentLogicalTimestampEnabled);
+ return /*~~>*//*~~>*/HoodieAvroUtils.getRecordColumnValues(this, columns, recordSchema, consistentLogicalTimestampEnabled);
}
@Override
@@ -152,7 +152,7 @@ public class HoodieAvroRecord<T extends HoodieRecordPayload> extends HoodieRecor
public HoodieRecord prependMetaFields(Schema recordSchema, Schema targetSchema, MetadataValues metadataValues, Properties props) {
try {
Option<IndexedRecord> avroRecordOpt = getData().getInsertValue(recordSchema, props);
- GenericRecord newAvroRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecordOpt.get(), targetSchema);
+ GenericRecord newAvroRecord = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecordOpt.get(), targetSchema);
updateMetadataValuesInternal(newAvroRecord, metadataValues);
return new HoodieAvroIndexedRecord(getKey(), newAvroRecord, getOperation(), this.currentLocation, this.newLocation);
} catch (IOException e) {
@@ -177,7 +177,7 @@ public class HoodieAvroRecord<T extends HoodieRecordPayload> extends HoodieRecor
public HoodieRecord rewriteRecordWithNewSchema(Schema recordSchema, Properties props, Schema newSchema, Map<String, String> renameCols) {
try {
GenericRecord oldRecord = (GenericRecord) getData().getInsertValue(recordSchema, props).get();
- GenericRecord rewriteRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(oldRecord, newSchema, renameCols);
+ GenericRecord rewriteRecord = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(oldRecord, newSchema, renameCols);
return new HoodieAvroIndexedRecord(getKey(), rewriteRecord, getOperation(), this.currentLocation, this.newLocation);
} catch (IOException e) {
throw new HoodieIOException("Failed to deserialize record!", e);
@@ -234,7 +234,7 @@ public class HoodieAvroRecord<T extends HoodieRecordPayload> extends HoodieRecor
IndexedRecord indexedRecord = (IndexedRecord) data.getInsertValue(recordSchema, props).get();
String payloadClass = ConfigUtils.getPayloadClass(props);
String[] orderingFields = ConfigUtils.getOrderingFields(props);
- return HoodieAvroUtils.createHoodieRecordFromAvro(indexedRecord, payloadClass, orderingFields, simpleKeyGenFieldsOpt,
+ return /*~~>*//*~~>*/HoodieAvroUtils.createHoodieRecordFromAvro(indexedRecord, payloadClass, orderingFields, simpleKeyGenFieldsOpt,
withOperation, partitionNameOp, populateMetaFields, schemaWithoutMetaFields);
}
@@ -269,7 +269,7 @@ public class HoodieAvroRecord<T extends HoodieRecordPayload> extends HoodieRecor
return baos;
} else {
Option<IndexedRecord> avroData = getData().getInsertValue(recordSchema, props);
- return avroData.map(HoodieAvroUtils::avroToBytesStream).orElse(new ByteArrayOutputStream(0));
+ return avroData.map(/*~~>*/HoodieAvroUtils::/*~~>*/avroToBytesStream).orElse(new ByteArrayOutputStream(0));
}
}
@@ -290,7 +290,7 @@ public class HoodieAvroRecord<T extends HoodieRecordPayload> extends HoodieRecor
public Object convertColumnValueForLogicalType(Schema fieldSchema,
Object fieldValue,
boolean keepConsistentLogicalTimestamp) {
- return HoodieAvroUtils.convertValueForAvroLogicalTypes(
+ return /*~~>*//*~~>*/HoodieAvroUtils.convertValueForAvroLogicalTypes(
fieldSchema, fieldValue, keepConsistentLogicalTimestamp);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java
index 2e7c74489f27..b3f402c21602 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/PartialUpdateAvroPayload.java
@@ -269,7 +269,7 @@ public class PartialUpdateAvroPayload extends OverwriteNonDefaultsWithLatestAvro
Comparable oldOrderingVal = OrderingValues.create(
orderingFields,
- field -> (Comparable) HoodieAvroUtils.getNestedFieldVal((GenericRecord) record, field, true, consistentLogicalTimestampEnabled));
+ field -> (Comparable) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal((GenericRecord) record, field, true, consistentLogicalTimestampEnabled));
// pick the payload with greater ordering value as insert record
return oldOrderingVal != null
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/SerializableIndexedRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/SerializableIndexedRecord.java
index 336a8effd939..ca7f6b11c906 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/SerializableIndexedRecord.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/SerializableIndexedRecord.java
@@ -91,7 +91,7 @@ public class SerializableIndexedRecord implements GenericRecord, KryoSerializabl
byte[] encodeRecord() {
if (recordBytes == null) {
- recordBytes = HoodieAvroUtils.avroToBytes(record);
+ recordBytes = /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record);
}
return recordBytes;
}
@@ -106,7 +106,7 @@ public class SerializableIndexedRecord implements GenericRecord, KryoSerializabl
public IndexedRecord getData() {
if (record == null) {
try {
- record = HoodieAvroUtils.bytesToAvro(recordBytes, schema);
+ record = /*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(recordBytes, schema);
recordBytes = null;
schema = null;
} catch (IOException e) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java
index da0467075dd0..eb6de3b3e932 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/AbstractDebeziumAvroPayload.java
@@ -111,7 +111,7 @@ public abstract class AbstractDebeziumAvroPayload extends OverwriteWithLatestAvr
}
private static boolean isDebeziumDeleteRecord(GenericRecord record) {
- Object value = HoodieAvroUtils.getFieldVal(record, DebeziumConstants.FLATTENED_OP_COL_NAME);
+ Object value = /*~~>*//*~~>*/HoodieAvroUtils.getFieldVal(record, DebeziumConstants.FLATTENED_OP_COL_NAME);
return value != null && value.toString().equalsIgnoreCase(DebeziumConstants.DELETE_OP);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
index 03a5c92a47dc..30eac8d1ad42 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchema.java
@@ -500,7 +500,7 @@ public class HoodieSchema implements Serializable {
public static HoodieSchema createHoodieWriteSchema(String schemaStr, boolean withOperationField) {
ValidationUtils.checkArgument(schemaStr != null && !schemaStr.trim().isEmpty(),
"Schema string cannot be null or empty");
- Schema avroSchema = HoodieAvroUtils.createHoodieWriteSchema(schemaStr, withOperationField);
+ Schema avroSchema = /*~~>*//*~~>*/HoodieAvroUtils.createHoodieWriteSchema(schemaStr, withOperationField);
return HoodieSchema.fromAvroSchema(avroSchema);
}
@@ -516,7 +516,7 @@ public class HoodieSchema implements Serializable {
public static HoodieSchema addMetadataFields(HoodieSchema schema, boolean withOperationField) {
ValidationUtils.checkArgument(schema != null, "Schema cannot be null");
Schema avroSchema = schema.toAvroSchema();
- Schema resultAvro = HoodieAvroUtils.addMetadataFields(avroSchema, withOperationField);
+ Schema resultAvro = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(avroSchema, withOperationField);
return HoodieSchema.fromAvroSchema(resultAvro);
}
@@ -531,7 +531,7 @@ public class HoodieSchema implements Serializable {
public static HoodieSchema removeMetadataFields(HoodieSchema schema) {
ValidationUtils.checkArgument(schema != null, "Schema cannot be null");
Schema avroSchema = schema.toAvroSchema();
- Schema resultAvro = HoodieAvroUtils.removeMetadataFields(avroSchema);
+ Schema resultAvro = /*~~>*//*~~>*/HoodieAvroUtils.removeMetadataFields(avroSchema);
return HoodieSchema.fromAvroSchema(resultAvro);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
index cd19ad3bfe61..da96f6054898 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaField.java
@@ -110,7 +110,7 @@ public class HoodieSchemaField implements Serializable {
Schema avroSchema = schema.getAvroSchema();
ValidationUtils.checkState(avroSchema != null, "Schema's Avro schema cannot be null");
- Schema.Field avroField = HoodieAvroUtils.createNewSchemaField(name, avroSchema, doc, defaultVal, order.toAvroOrder());
+ Schema.Field avroField = /*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(name, avroSchema, doc, defaultVal, order.toAvroOrder());
return new HoodieSchemaField(avroField);
}
@@ -270,7 +270,7 @@ public class HoodieSchemaField implements Serializable {
public HoodieSchemaField withName(String newName) {
ValidationUtils.checkArgument(newName != null && !newName.isEmpty(), "Field name cannot be null or empty");
- Schema.Field newAvroField = HoodieAvroUtils.createNewSchemaField(newName, avroField.schema(), avroField.doc(),
+ Schema.Field newAvroField = /*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(newName, avroField.schema(), avroField.doc(),
avroField.hasDefaultValue() ? avroField.defaultVal() : null, avroField.order());
return new HoodieSchemaField(newAvroField);
}
@@ -287,7 +287,7 @@ public class HoodieSchemaField implements Serializable {
Schema newAvroSchema = newSchema.getAvroSchema();
ValidationUtils.checkState(newAvroSchema != null, "New schema's Avro schema cannot be null");
- Schema.Field newAvroField = HoodieAvroUtils.createNewSchemaField(avroField.name(), newAvroSchema, avroField.doc(),
+ Schema.Field newAvroField = /*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(avroField.name(), newAvroSchema, avroField.doc(),
avroField.hasDefaultValue() ? avroField.defaultVal() : null, avroField.order());
return new HoodieSchemaField(newAvroField);
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
index 4a2b458044d9..ba2e40544aa7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieSchemaUtils.java
@@ -75,7 +75,7 @@ public final class HoodieSchemaUtils {
"Schema string cannot be null or empty");
// Delegate to existing HoodieAvroUtils, convert result to HoodieSchema
- Schema avroSchema = HoodieAvroUtils.createHoodieWriteSchema(schema, withOperationField);
+ Schema avroSchema = /*~~>*//*~~>*/HoodieAvroUtils.createHoodieWriteSchema(schema, withOperationField);
return HoodieSchema.fromAvroSchema(avroSchema);
}
@@ -105,7 +105,7 @@ public final class HoodieSchemaUtils {
// Convert to Avro, delegate to existing utility, convert back
Schema avroSchema = schema.toAvroSchema();
- Schema resultAvro = HoodieAvroUtils.addMetadataFields(avroSchema, withOperationField);
+ Schema resultAvro = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(avroSchema, withOperationField);
return HoodieSchema.fromAvroSchema(resultAvro);
}
@@ -122,7 +122,7 @@ public final class HoodieSchemaUtils {
// Convert to Avro, delegate to existing utility, convert back
Schema avroSchema = schema.toAvroSchema();
- Schema resultAvro = HoodieAvroUtils.removeMetadataFields(avroSchema);
+ Schema resultAvro = /*~~>*//*~~>*/HoodieAvroUtils.removeMetadataFields(avroSchema);
return HoodieSchema.fromAvroSchema(resultAvro);
}
@@ -302,7 +302,7 @@ public final class HoodieSchemaUtils {
ValidationUtils.checkArgument(schema != null, "Field schema cannot be null");
// Delegate to HoodieAvroUtils
- Schema.Field avroField = HoodieAvroUtils.createNewSchemaField(
+ Schema.Field avroField = /*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(
name, schema.toAvroSchema(), doc, defaultValue);
return HoodieSchemaField.fromAvroField(avroField);
}
@@ -422,7 +422,7 @@ public final class HoodieSchemaUtils {
ValidationUtils.checkArgument(fieldNames != null, "Field names cannot be null");
// Delegate to HoodieAvroUtils
- Schema projectedAvro = HoodieAvroUtils.generateProjectionSchema(originalSchema.toAvroSchema(), fieldNames);
+ Schema projectedAvro = /*~~>*//*~~>*/HoodieAvroUtils.generateProjectionSchema(originalSchema.toAvroSchema(), fieldNames);
return HoodieSchema.fromAvroSchema(projectedAvro);
}
@@ -589,13 +589,17 @@ public final class HoodieSchemaUtils {
/**
* Fetches projected schema given list of fields to project. The field can be nested in format `a.b.c` where a is
* the top level field, b is at second level and so on.
- * This is equivalent to {@link HoodieAvroUtils#projectSchema(Schema, List)} but operates on HoodieSchema.
+ * This is equivalent to {@link ~~>~~>HoodieAvroUtils#projectSchema(Schema, List)} but operates on HoodieSchema.
*
* @param fileSchema the original schema
* @param fields list of fields to project
* @return projected schema containing only specified fields
*/
public static HoodieSchema projectSchema(HoodieSchema fileSchema, List<String> fields) {
- return HoodieSchema.fromAvroSchema(HoodieAvroUtils.projectSchema(fileSchema.toAvroSchema(), fields));
+ return HoodieSchema.fromAvroSchema(/*~~>*//*~~>*/HoodieAvroUtils.projectSchema(fileSchema.toAvroSchema(), fields));
+ }
+
+ public static boolean isMetadataField(String fieldName) {
+ return HoodieRecord.HOODIE_META_COLUMNS_WITH_OPERATION.contains(fieldName);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
index 2309ef5ab743..d79d68bedbc5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -224,14 +224,14 @@ public class TableSchemaResolver {
metaClient.getTableConfig().getTableCreateSchema()
.map(tableSchema ->
includeMetadataFields
- ? HoodieAvroUtils.addMetadataFields(tableSchema, hasOperationField.get())
+ ? /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(tableSchema, hasOperationField.get())
: tableSchema)
)
.or(() -> {
Option<Schema> schemaFromDataFile = getTableAvroSchemaFromDataFileInternal();
return includeMetadataFields
? schemaFromDataFile
- : schemaFromDataFile.map(HoodieAvroUtils::removeMetadataFields);
+ : schemaFromDataFile.map(/*~~>*/HoodieAvroUtils::/*~~>*/removeMetadataFields);
});
// TODO partition columns have to be appended in all read-paths
@@ -253,9 +253,9 @@ public class TableSchemaResolver {
String schemaStr = commitMetadata.getMetadata(HoodieCommitMetadata.SCHEMA_KEY);
Schema schema = new Schema.Parser().parse(schemaStr);
if (includeMetadataFields) {
- schema = HoodieAvroUtils.addMetadataFields(schema, hasOperationField.get());
+ schema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema, hasOperationField.get());
} else {
- schema = HoodieAvroUtils.removeMetadataFields(schema);
+ schema = /*~~>*//*~~>*/HoodieAvroUtils.removeMetadataFields(schema);
}
return Option.of(schema);
} else {
@@ -274,9 +274,9 @@ public class TableSchemaResolver {
Schema schema = new Schema.Parser().parse(existingSchemaStr);
if (includeMetadataFields) {
- schema = HoodieAvroUtils.addMetadataFields(schema, hasOperationField.get());
+ schema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema, hasOperationField.get());
} else {
- schema = HoodieAvroUtils.removeMetadataFields(schema);
+ schema = /*~~>*//*~~>*/HoodieAvroUtils.removeMetadataFields(schema);
}
return Option.of(schema);
} catch (Exception e) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
index 26b8cbde6c79..665e1f5a813e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieAvroDataBlock.java
@@ -194,7 +194,7 @@ public class HoodieAvroDataBlock extends HoodieDataBlock {
// and for tables having no logical ts column.
Schema repairedWriterSchema = enableLogicalTimestampFieldRepair
? AvroSchemaRepair.repairLogicalTypes(writerSchema.toAvroSchema(), readerSchema.toAvroSchema()) : writerSchema.toAvroSchema();
- if (recordNeedsRewriteForExtendedAvroTypePromotion(repairedWriterSchema, readerSchema.toAvroSchema())) {
+ if (/*~~>*/recordNeedsRewriteForExtendedAvroTypePromotion(repairedWriterSchema, readerSchema.toAvroSchema())) {
this.reader = new GenericDatumReader<>(repairedWriterSchema, repairedWriterSchema);
this.promotedSchema = Option.of(readerSchema);
} else {
@@ -233,7 +233,7 @@ public class HoodieAvroDataBlock extends HoodieDataBlock {
this.dis.skipBytes(recordLength);
this.readRecords++;
if (this.promotedSchema.isPresent()) {
- return HoodieAvroUtils.rewriteRecordWithNewSchema(record, this.promotedSchema.get().toAvroSchema());
+ return /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(record, this.promotedSchema.get().toAvroSchema());
}
return record;
} catch (IOException e) {
@@ -280,7 +280,7 @@ public class HoodieAvroDataBlock extends HoodieDataBlock {
}
Schema repairedWriterSchema = AvroSchemaRepair.repairLogicalTypes(writerSchema.toAvroSchema(), readerSchema.toAvroSchema());
- if (recordNeedsRewriteForExtendedAvroTypePromotion(repairedWriterSchema, readerSchema.toAvroSchema())) {
+ if (/*~~>*/recordNeedsRewriteForExtendedAvroTypePromotion(repairedWriterSchema, readerSchema.toAvroSchema())) {
this.reader = new GenericDatumReader<>(repairedWriterSchema, repairedWriterSchema);
this.promotedSchema = Option.of(readerSchema);
} else {
@@ -331,7 +331,7 @@ public class HoodieAvroDataBlock extends HoodieDataBlock {
buffer.position(buffer.position() + recordLength);
this.readRecords++;
if (this.promotedSchema.isPresent()) {
- return HoodieAvroUtils.rewriteRecordWithNewSchema(record, this.promotedSchema.get().toAvroSchema());
+ return /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(record, this.promotedSchema.get().toAvroSchema());
}
return record;
} catch (IOException e) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
index 54e7f301e6a8..8fd6b1cb0376 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
@@ -167,7 +167,7 @@ public class HoodieDeleteBlock extends HoodieLogBlock {
} else {
DatumReader<GenericRecord> reader = new GenericDatumReader<>(HoodieDeleteRecordList.SCHEMA$);
BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, 0, data.length, null);
- List<HoodieDeleteRecord> deleteRecordList = HoodieAvroUtils.convertToSpecificRecord(HoodieDeleteRecordList.class, reader.read(null, decoder))
+ List<HoodieDeleteRecord> deleteRecordList = /*~~>*//*~~>*/HoodieAvroUtils.convertToSpecificRecord(HoodieDeleteRecordList.class, reader.read(null, decoder))
.getDeleteRecordList();
return deleteRecordList.stream()
.map(record -> DeleteRecord.create(
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/read/PartialUpdateHandler.java b/hudi-common/src/main/java/org/apache/hudi/common/table/read/PartialUpdateHandler.java
index 9af83d5054f9..a166d22959a2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/read/PartialUpdateHandler.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/read/PartialUpdateHandler.java
@@ -117,7 +117,7 @@ public class PartialUpdateHandler<T> implements Serializable {
String fieldName = field.name();
// The default value only from the top-level data type is validated. That means,
// for nested columns, we do not check the leaf level data type defaults.
- Object defaultValue = HoodieAvroUtils.toJavaDefaultValue(field.getAvroField());
+ Object defaultValue = /*~~>*//*~~>*/HoodieAvroUtils.toJavaDefaultValue(field.getAvroField());
Object newValue = recordContext.getValue(highOrderRecord.getRecord(), highOrderSchema, fieldName);
if (defaultValue == newValue) {
fieldVals[idx++] = recordContext.getValue(lowOrderRecord.getRecord(), lowOrderSchema, fieldName);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java
index ff1939c2a38a..354b6c266a56 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/TimelineMetadataUtils.java
@@ -129,7 +129,7 @@ public class TimelineMetadataUtils {
DatumReader<GenericRecord> reader = new GenericDatumReader<>(SpecificData.getForClass(clazz).getSchema(clazz));
FileReader<GenericRecord> fileReader = DataFileReader.openReader(new SeekableByteArrayInput(bytes), reader);
ValidationUtils.checkArgument(fileReader.hasNext(), "Could not deserialize metadata of type " + clazz);
- return HoodieAvroUtils.convertToSpecificRecord(clazz, fileReader.next());
+ return /*~~>*//*~~>*/HoodieAvroUtils.convertToSpecificRecord(clazz, fileReader.next());
}
public static <T extends SpecificRecordBase> T deserializeAvroMetadata(InputStream inputStream, Class<T> clazz)
@@ -137,7 +137,7 @@ public class TimelineMetadataUtils {
DatumReader<GenericRecord> reader = new GenericDatumReader<>(SpecificData.getForClass(clazz).getSchema(clazz));
try (DataFileStream<GenericRecord> fileReader = new DataFileStream<>(inputStream, reader)) {
ValidationUtils.checkArgument(fileReader.hasNext(), "Could not deserialize metadata of type " + clazz);
- return HoodieAvroUtils.convertToSpecificRecord(clazz, fileReader.next());
+ return /*~~>*//*~~>*/HoodieAvroUtils.convertToSpecificRecord(clazz, fileReader.next());
}
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
index a1fb702ca558..5fbfbbf12a5f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/FileFormatUtils.java
@@ -289,9 +289,9 @@ public abstract class FileFormatUtils {
List<String> fields = new ArrayList<>();
fields.addAll(keyGenerator.getRecordKeyFieldNames());
fields.addAll(keyGenerator.getPartitionPathFields());
- return HoodieAvroUtils.projectSchema(readAvroSchema(storage, filePath), fields);
+ return /*~~>*//*~~>*/HoodieAvroUtils.projectSchema(readAvroSchema(storage, filePath), fields);
})
- .orElse(partitionPath.isPresent() ? HoodieAvroUtils.getRecordKeySchema() : HoodieAvroUtils.getRecordKeyPartitionPathSchema());
+ .orElse(partitionPath.isPresent() ? /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeySchema() : /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeyPartitionPathSchema());
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/SortUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SortUtils.java
index c043f7a2fef0..2f414e759bf5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/SortUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SortUtils.java
@@ -71,7 +71,7 @@ public class SortUtils {
return FlatLists.ofComparableArray(prependPartitionPath(record.getPartitionPath(), columnValues));
} else if (record.getRecordType() == HoodieRecord.HoodieRecordType.AVRO) {
return FlatLists.ofComparableArray(
- HoodieAvroUtils.getSortColumnValuesWithPartitionPathAndRecordKey(
+ /*~~>*//*~~>*/HoodieAvroUtils.getSortColumnValuesWithPartitionPathAndRecordKey(
record, sortColumnNames, schema, suffixRecordKey, consistentLogicalTimestampEnabled
));
}
@@ -105,7 +105,7 @@ public class SortUtils {
return FlatLists.ofComparableArray(wrapUTF8StringFunc.apply(prependPartitionPath(record.getPartitionPath(), columnValues)));
} else if (record.getRecordType() == HoodieRecord.HoodieRecordType.AVRO) {
return FlatLists.ofComparableArray(wrapUTF8StringFunc.apply(
- HoodieAvroUtils.getSortColumnValuesWithPartitionPathAndRecordKey(
+ /*~~>*//*~~>*/HoodieAvroUtils.getSortColumnValuesWithPartitionPathAndRecordKey(
record, sortColumnNames, schema.toAvroSchema(), suffixRecordKey, consistentLogicalTimestampEnabled
)));
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java
index 3950a6ead060..829eebaaab18 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/SpillableMapUtils.java
@@ -141,7 +141,7 @@ public class SpillableMapUtils {
Comparable preCombineVal = getPreCombineVal(record, preCombineFields);
HoodieOperation operation = withOperationField
- ? HoodieOperation.fromName(getNullableValAsString(record, HoodieRecord.OPERATION_METADATA_FIELD)) : null;
+ ? HoodieOperation.fromName(/*~~>*/getNullableValAsString(record, HoodieRecord.OPERATION_METADATA_FIELD)) : null;
if (schemaWithoutMetaFields.isPresent()) {
Schema schema = schemaWithoutMetaFields.get();
@@ -172,7 +172,7 @@ public class SpillableMapUtils {
return OrderingValues.create(
preCombineFields,
field -> {
- Object orderingValue = HoodieAvroUtils.getNestedFieldVal(rec, field, true, false);
+ Object orderingValue = /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal(rec, field, true, false);
return orderingValue == null ? OrderingValues.getDefault() : (Comparable) orderingValue;
});
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileWriter.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileWriter.java
index d4a0031c88a0..d0a8816c2cf3 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileWriter.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroFileWriter.java
@@ -53,7 +53,7 @@ public interface HoodieAvroFileWriter extends HoodieFileWriter {
default void prepRecordWithMetadata(HoodieKey key, IndexedRecord avroRecord, String instantTime, Integer partitionId, long recordIndex, String fileName) {
String seqId = HoodieRecord.generateSequenceId(instantTime, partitionId, recordIndex);
- HoodieAvroUtils.addHoodieKeyToRecord((GenericRecord) avroRecord, key.getRecordKey(), key.getPartitionPath(), fileName);
- HoodieAvroUtils.addCommitMetadataToRecord((GenericRecord) avroRecord, instantTime, seqId);
+ /*~~>*//*~~>*/HoodieAvroUtils.addHoodieKeyToRecord((GenericRecord) avroRecord, key.getRecordKey(), key.getPartitionPath(), fileName);
+ /*~~>*//*~~>*/HoodieAvroUtils.addCommitMetadataToRecord((GenericRecord) avroRecord, instantTime, seqId);
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroHFileReaderImplBase.java b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroHFileReaderImplBase.java
index 551f58000a70..865ef6d0f5ba 100644
--- a/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroHFileReaderImplBase.java
+++ b/hudi-common/src/main/java/org/apache/hudi/io/storage/HoodieAvroHFileReaderImplBase.java
@@ -91,7 +91,7 @@ public abstract class HoodieAvroHFileReaderImplBase extends HoodieAvroFileReader
final byte[] valueBytes, int valueOffset, int valueLength,
GenericDatumReader<GenericRecord> datumReader,
HoodieSchemaField keyFieldSchema) throws IOException {
- BinaryDecoder binaryDecoder = getBinaryDecoder(valueBytes, valueOffset, valueLength);
+ BinaryDecoder binaryDecoder = /*~~>*/getBinaryDecoder(valueBytes, valueOffset, valueLength);
GenericRecord avroRecord = datumReader.read(null, binaryDecoder);
if (keyFieldSchema == null) {
return avroRecord;
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index 7fa98d0c6ebb..b1a21edba919 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -328,7 +328,7 @@ public class HoodieTableMetadataUtil {
Object fieldValue;
HoodieSchemaType fieldSchemaType = fieldSchema.getType();
if (record.getRecordType() == HoodieRecordType.AVRO) {
- fieldValue = HoodieAvroUtils.getRecordColumnValues(record, new String[]{fieldName}, recordSchema.toAvroSchema(), false)[0];
+ fieldValue = /*~~>*//*~~>*/HoodieAvroUtils.getRecordColumnValues(record, new String[]{fieldName}, recordSchema.toAvroSchema(), false)[0];
if (fieldValue != null && fieldSchemaType.equals(HoodieSchemaType.DATE)) {
fieldValue = java.sql.Date.valueOf(fieldValue.toString());
}
@@ -1609,7 +1609,7 @@ public class HoodieTableMetadataUtil {
// NOTE: Writer schema added to commit metadata will not contain Hudi's metadata fields
Option<HoodieSchema> tableSchema = writerSchema.isEmpty()
? tableConfig.getTableCreateSchema().map(HoodieSchema::fromAvroSchema) // the write schema does not set up correctly
- : writerSchema.map(schema -> tableConfig.populateMetaFields() ? addMetadataFields(schema) : schema)
+ : writerSchema.map(schema -> tableConfig.populateMetaFields() ? /*~~>*/addMetadataFields(schema) : schema)
.map(HoodieSchema::fromAvroSchema);
HoodieIndexVersion indexVersion = existingIndexVersionOrDefault(PARTITION_NAME_COLUMN_STATS, dataMetaClient);
@@ -2585,7 +2585,7 @@ public class HoodieTableMetadataUtil {
List<String> mergedFields = new ArrayList<>(partitionFields.size() + sourceFields.size());
mergedFields.addAll(partitionFields);
mergedFields.addAll(sourceFields);
- return HoodieSchema.fromAvroSchema(addMetadataFields(projectSchema(tableSchema.toAvroSchema(), mergedFields)));
+ return HoodieSchema.fromAvroSchema(/*~~>*/addMetadataFields(/*~~>*/projectSchema(tableSchema.toAvroSchema(), mergedFields)));
}
public static StoragePath filePath(StoragePath basePath, String partition, String filename) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/stats/ValueType.java b/hudi-common/src/main/java/org/apache/hudi/stats/ValueType.java
index d99467fdb5b3..40917c51bfd2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/stats/ValueType.java
+++ b/hudi-common/src/main/java/org/apache/hudi/stats/ValueType.java
@@ -432,11 +432,11 @@ public enum ValueType {
if (val instanceof BigDecimal) {
return (BigDecimal) val;
} else if (val instanceof GenericData.Fixed) {
- return convertBytesToBigDecimal(((GenericData.Fixed) val).bytes(), precision, scale);
+ return /*~~>*/convertBytesToBigDecimal(((GenericData.Fixed) val).bytes(), precision, scale);
} else if (val instanceof ByteBuffer) {
- return convertBytesToBigDecimal(((ByteBuffer) val).array(), precision, scale);
+ return /*~~>*/convertBytesToBigDecimal(((ByteBuffer) val).array(), precision, scale);
} else if (val instanceof byte[]) {
- return convertBytesToBigDecimal((byte[]) val, precision, scale);
+ return /*~~>*/convertBytesToBigDecimal((byte[]) val, precision, scale);
} else if (val instanceof Integer) {
return BigDecimal.valueOf((Integer) val, scale).round(new MathContext(precision, RoundingMode.HALF_UP));
} else if (val instanceof Long) {
@@ -558,7 +558,7 @@ public enum ValueType {
public static BigDecimal toDecimal(Comparable<?> val, ValueMetadata meta) {
ValueMetadata.DecimalMetadata decimalMeta = (ValueMetadata.DecimalMetadata) meta;
- return convertBytesToBigDecimal(((ByteBuffer) val).array(), decimalMeta.getPrecision(), decimalMeta.getScale());
+ return /*~~>*/convertBytesToBigDecimal(((ByteBuffer) val).array(), decimalMeta.getPrecision(), decimalMeta.getScale());
}
public static ByteBuffer fromDecimal(Comparable<?> val, ValueMetadata meta) {
diff --git a/hudi-common/src/main/java/org/apache/parquet/schema/AvroSchemaRepair.java b/hudi-common/src/main/java/org/apache/parquet/schema/AvroSchemaRepair.java
index 4ee0a2c989f9..5a8d4b94af07 100644
--- a/hudi-common/src/main/java/org/apache/parquet/schema/AvroSchemaRepair.java
+++ b/hudi-common/src/main/java/org/apache/parquet/schema/AvroSchemaRepair.java
@@ -169,12 +169,12 @@ public class AvroSchemaRepair {
for (int i = 0; i < firstChangedIndex; i++) {
Schema.Field field = fields.get(i);
// Must create new Field since they cannot be reused
- repairedFields.add(HoodieAvroUtils.createNewSchemaField(field));
+ repairedFields.add(/*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(field));
}
// Add the first changed field (using cached repaired schema)
Schema.Field firstChangedField = fields.get(firstChangedIndex);
- repairedFields.add(HoodieAvroUtils.createNewSchemaField(
+ repairedFields.add(/*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(
firstChangedField.name(),
firstRepairedSchema,
firstChangedField.doc(),
@@ -194,7 +194,7 @@ public class AvroSchemaRepair {
}
// Must create new Field since they cannot be reused
- repairedFields.add(HoodieAvroUtils.createNewSchemaField(
+ repairedFields.add(/*~~>*//*~~>*/HoodieAvroUtils.createNewSchemaField(
requestedField.name(),
repairedSchema,
requestedField.doc(),
diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestMercifulJsonConverter.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestMercifulJsonConverter.java
index 6b7687133f5b..0313b3e8e8a9 100644
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestMercifulJsonConverter.java
+++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestMercifulJsonConverter.java
@@ -548,7 +548,7 @@ public class TestMercifulJsonConverter extends MercifulJsonConverterTestBase {
TRIP_ENCODED_DECIMAL_SCHEMA.replace("6", Integer.toString(scale)).replace("10", Integer.toString(precision)));
GenericRecord genrec = CONVERTER.convert(json, tripSchema.toAvroSchema());
HoodieSchema decimalFieldSchema = tripSchema.getField("decfield").get().schema();
- BigDecimal decoded = HoodieAvroUtils.convertBytesToBigDecimal(((ByteBuffer) genrec.get("decfield")).array(),
+ BigDecimal decoded = /*~~>*//*~~>*/HoodieAvroUtils.convertBytesToBigDecimal(((ByteBuffer) genrec.get("decfield")).array(),
(LogicalTypes.Decimal) decimalFieldSchema.toAvroSchema().getLogicalType());
assertEquals(decfield, decoded);
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java
index 97b0ac5c108c..9e990106c83e 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestOverwriteNonDefaultsWithLatestAvroPayload.java
@@ -56,7 +56,7 @@ public class TestOverwriteNonDefaultsWithLatestAvroPayload {
@Test
public void testActiveRecords() throws IOException {
- Schema writerSchema = HoodieAvroUtils.addMetadataFields(schema);
+ Schema writerSchema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema);
GenericRecord record1 = new GenericData.Record(schema);
record1.put("id", "1");
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java
index 51c28f795cf4..848470a79dab 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestPartialUpdateAvroPayload.java
@@ -160,19 +160,19 @@ public class TestPartialUpdateAvroPayload {
PartialUpdateAvroPayload payload2 = new PartialUpdateAvroPayload(record2, 2L);
PartialUpdateAvroPayload mergedPayload = payload1.preCombine(delPayload, schema, new Properties());
- assertTrue(HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(true));
+ assertTrue(/*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(true));
assertArrayEquals(mergedPayload.getRecordBytes(), delPayload.getRecordBytes());
mergedPayload = delPayload.preCombine(payload1, schema, new Properties());
- assertTrue(HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(true));
+ assertTrue(/*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(true));
assertArrayEquals(mergedPayload.getRecordBytes(), delPayload.getRecordBytes());
mergedPayload = payload2.preCombine(delPayload, schema, new Properties());
- assertTrue(HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(false));
+ assertTrue(/*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(false));
assertArrayEquals(mergedPayload.getRecordBytes(), payload2.getRecordBytes());
mergedPayload = delPayload.preCombine(payload2, schema, new Properties());
- assertTrue(HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(false));
+ assertTrue(/*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(mergedPayload.getRecordBytes(), schema).get("_hoodie_is_deleted").equals(false));
assertArrayEquals(mergedPayload.getRecordBytes(), payload2.getRecordBytes());
assertEquals(record1, payload1.getInsertValue(schema).get());
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/model/TestSerializableIndexRecord.java b/hudi-common/src/test/java/org/apache/hudi/common/model/TestSerializableIndexRecord.java
index 39c012684c6e..74e12842419e 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/model/TestSerializableIndexRecord.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/model/TestSerializableIndexRecord.java
@@ -72,7 +72,7 @@ class TestSerializableIndexRecord {
SerializableIndexedRecord serializableIndexedRecord = SerializableIndexedRecord.createInstance(originalRecord);
byte[] encoded = serializableIndexedRecord.encodeRecord();
- assertEquals(originalRecord, HoodieAvroUtils.bytesToAvro(encoded, schema));
+ assertEquals(originalRecord, /*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(encoded, schema));
byte[] reEncoded = serializableIndexedRecord.encodeRecord();
// encode should use existing bytes
assertSame(encoded, reEncoded);
@@ -98,7 +98,7 @@ class TestSerializableIndexRecord {
.set("field_1", "value1")
.set("field_2", 42)
.build();
- byte[] encoded = HoodieAvroUtils.avroToBytes(originalRecord);
+ byte[] encoded = /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(originalRecord);
SerializableIndexedRecord serializableIndexedRecord = SerializableIndexedRecord.fromAvroBytes(schema, encoded);
assertEquals(originalRecord, serializableIndexedRecord.getRecord());
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
index b170ff30613f..60864ec69189 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/schema/TestHoodieSchemaUtils.java
@@ -317,7 +317,7 @@ public class TestHoodieSchemaUtils {
String schemaString = SIMPLE_SCHEMA;
// Compare createHoodieWriteSchema results
- Schema avroResult = HoodieAvroUtils.createHoodieWriteSchema(schemaString, true);
+ Schema avroResult = /*~~>*//*~~>*/HoodieAvroUtils.createHoodieWriteSchema(schemaString, true);
HoodieSchema hoodieResult = HoodieSchemaUtils.createHoodieWriteSchema(schemaString, true);
// Should produce equivalent schemas
@@ -816,7 +816,7 @@ public class TestHoodieSchemaUtils {
Set<String> fieldsToRemove = new HashSet<>(Arrays.asList("age", "email"));
// Apply removeFields using both approaches
- Schema avroResult = HoodieAvroUtils.removeFields(avroSchema, fieldsToRemove);
+ Schema avroResult = /*~~>*//*~~>*/HoodieAvroUtils.removeFields(avroSchema, fieldsToRemove);
HoodieSchema hoodieResult = HoodieSchemaUtils.removeFields(hoodieSchema, fieldsToRemove);
// Should produce equivalent schemas
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
index b1ff1656157d..b7904fc03ab9 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/table/read/TestHoodieFileGroupReaderBase.java
@@ -677,11 +677,11 @@ public abstract class TestHoodieFileGroupReaderBase<T> {
assertEquals(expectedRecords.size(), actualRecordList.size());
actualRecordList.forEach(r -> assertRecordMatchesSchema(schema, r));
Set<GenericRecord> actualRecordSet = actualRecordList.stream().map(r -> readerContext.getRecordContext().convertToAvroRecord(r, schema))
- .map(r -> HoodieAvroUtils.removeFields(r, metaCols))
+ .map(r -> /*~~>*//*~~>*/HoodieAvroUtils.removeFields(r, metaCols))
.collect(Collectors.toSet());
Set<GenericRecord> expectedRecordSet = expectedRecords.stream()
.map(r -> resetByteBufferPosition((GenericRecord) r.getRight()))
- .map(r -> HoodieAvroUtils.rewriteRecordWithNewSchema(r, schemaWithoutMeta.toAvroSchema()))
+ .map(r -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(r, schemaWithoutMeta.toAvroSchema()))
.collect(Collectors.toSet());
compareRecordSets(expectedRecordSet, actualRecordSet);
}
@@ -988,7 +988,7 @@ public abstract class TestHoodieFileGroupReaderBase<T> {
if (value == null) {
return;
}
- Schema.Type fieldType = HoodieAvroUtils.unwrapNullable(fieldSchema).getType();
+ Schema.Type fieldType = /*~~>*//*~~>*/HoodieAvroUtils.unwrapNullable(fieldSchema).getType();
if (fieldType == Schema.Type.BYTES || fieldType == Schema.Type.FIXED) {
// Reset position of ByteBuffer or Fixed type fields
if (value instanceof ByteBuffer) {
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
index cc84de43e382..5ba5ecab2eb8 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
@@ -239,7 +239,7 @@ public class HoodieTestDataGenerator implements AutoCloseable {
public static final Schema AVRO_SCHEMA_WITH_SPECIFIC_COLUMNS = new Schema.Parser().parse(TRIP_EXAMPLE_SCHEMA_WITH_PAYLOAD_SPECIFIC_COLS);
public static final Schema NESTED_AVRO_SCHEMA = new Schema.Parser().parse(TRIP_NESTED_EXAMPLE_SCHEMA);
public static final Schema AVRO_SCHEMA_WITH_METADATA_FIELDS =
- HoodieAvroUtils.addMetadataFields(AVRO_SCHEMA);
+ /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(AVRO_SCHEMA);
public static final HoodieSchema HOODIE_SCHEMA_WITH_METADATA_FIELDS = HoodieSchema.fromAvroSchema(AVRO_SCHEMA_WITH_METADATA_FIELDS);
public static final Schema AVRO_SHORT_TRIP_SCHEMA = new Schema.Parser().parse(SHORT_TRIP_SCHEMA);
public static final Schema AVRO_TRIP_ENCODED_DECIMAL_SCHEMA = new Schema.Parser().parse(TRIP_ENCODED_DECIMAL_SCHEMA);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java
index 9298017070eb..13f35e951649 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/SchemaTestUtil.java
@@ -233,7 +233,7 @@ public final class SchemaTestUtil {
HoodieSchema hoodieFieldsSchema = HoodieSchemaUtils.addMetadataFields(getSimpleSchema());
List<IndexedRecord> recordsWithMetaFields = new ArrayList<>();
for (int i = 0; i < recordKeyList.size(); i++) {
- GenericRecord newRecord = HoodieAvroUtils.rewriteRecord((GenericRecord) records.get(i), hoodieFieldsSchema.toAvroSchema());
+ GenericRecord newRecord = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) records.get(i), hoodieFieldsSchema.toAvroSchema());
newRecord.put(HoodieRecord.RECORD_KEY_METADATA_FIELD, recordKeyList.get(i));
newRecord.put(HoodieRecord.PARTITION_PATH_METADATA_FIELD, partitionPath);
newRecord.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, instantTime);
@@ -245,7 +245,7 @@ public final class SchemaTestUtil {
public List<HoodieRecord> generateHoodieTestRecords(int from, int limit, HoodieSchema schema)
throws IOException, URISyntaxException {
List<IndexedRecord> records = generateTestRecords(from, limit);
- return records.stream().map(s -> HoodieAvroUtils.rewriteRecord((GenericRecord) s, schema.toAvroSchema()))
+ return records.stream().map(s -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) s, schema.toAvroSchema()))
.map(p -> convertToHoodieRecords(p, genRandomUUID(), "000/00/00")).collect(Collectors.toList());
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieRecordTestPayload.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieRecordTestPayload.java
index bd762d689e04..63d78c847529 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieRecordTestPayload.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/reader/HoodieRecordTestPayload.java
@@ -174,7 +174,7 @@ public class HoodieRecordTestPayload extends OverwriteWithLatestAvroPayload {
return Option.empty();
}
return Option.ofNullable(
- HoodieAvroUtils.getNestedFieldVal(
+ /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal(
record,
eventTimeField,
true,
@@ -212,10 +212,10 @@ public class HoodieRecordTestPayload extends OverwriteWithLatestAvroPayload {
KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue()));
Comparable<?> persistedOrderingVal = OrderingValues.create(
orderingFields,
- field -> (Comparable) HoodieAvroUtils.getNestedFieldVal((GenericRecord) currentValue, field, true, consistentLogicalTimestampEnabled));
+ field -> (Comparable) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal((GenericRecord) currentValue, field, true, consistentLogicalTimestampEnabled));
Comparable<?> incomingOrderingVal = OrderingValues.create(
orderingFields,
- field -> (Comparable) HoodieAvroUtils.getNestedFieldVal((GenericRecord) incomingRecord, field, true, consistentLogicalTimestampEnabled));
+ field -> (Comparable) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal((GenericRecord) incomingRecord, field, true, consistentLogicalTimestampEnabled));
return persistedOrderingVal == null || ((Comparable) persistedOrderingVal).compareTo(incomingOrderingVal) <= 0;
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java
index be938ef9a481..47ac156a8958 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestObjectSizeCalculator.java
@@ -65,7 +65,7 @@ public class TestObjectSizeCalculator {
assertEquals(168, getObjectSize(stringArray));
assertEquals(144, getObjectSize(stringBuilder));
assertEquals(72, getObjectSize(DayOfWeek.TUESDAY));
- assertEquals(HoodieAvroUtils.gteqAvro1_9() ? 1256 : 1176,
+ assertEquals(/*~~>*//*~~>*/HoodieAvroUtils.gteqAvro1_9() ? 1256 : 1176,
getObjectSize(Schema.create(Schema.Type.STRING)));
assertEquals(96, getObjectSize(person));
} else {
@@ -75,7 +75,7 @@ public class TestObjectSizeCalculator {
assertEquals(80, getObjectSize(DayOfWeek.TUESDAY));
// Since avro 1.9, Schema use ConcurrentHashMap instead of LinkedHashMap to
// implement props, which will change the size of the object.
- assertEquals(HoodieAvroUtils.gteqAvro1_9() ? 1320 : 1240,
+ assertEquals(/*~~>*//*~~>*/HoodieAvroUtils.gteqAvro1_9() ? 1320 : 1240,
getObjectSize(Schema.create(Schema.Type.STRING)));
assertEquals(104, getObjectSize(person));
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java
index 731e88353ba3..c33c3f43da65 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestSerializableSchema.java
@@ -42,11 +42,11 @@ public class TestSerializableSchema {
@Test
public void testSerDeser() throws IOException {
verifySchema(HoodieTestDataGenerator.AVRO_TRIP_SCHEMA);
- verifySchema(HoodieAvroUtils.addMetadataFields(HoodieTestDataGenerator.AVRO_TRIP_SCHEMA));
+ verifySchema(/*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(HoodieTestDataGenerator.AVRO_TRIP_SCHEMA));
verifySchema(HoodieTestDataGenerator.AVRO_SHORT_TRIP_SCHEMA);
- verifySchema(HoodieAvroUtils.addMetadataFields(HoodieTestDataGenerator.AVRO_SHORT_TRIP_SCHEMA));
+ verifySchema(/*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(HoodieTestDataGenerator.AVRO_SHORT_TRIP_SCHEMA));
verifySchema(HoodieTestDataGenerator.FLATTENED_AVRO_SCHEMA);
- verifySchema(HoodieAvroUtils.addMetadataFields(HoodieTestDataGenerator.FLATTENED_AVRO_SCHEMA));
+ verifySchema(/*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(HoodieTestDataGenerator.FLATTENED_AVRO_SCHEMA));
verifySchema(HoodieTestDataGenerator.AVRO_SCHEMA_WITH_METADATA_FIELDS);
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java
index 6a4e9dbb506d..20e885c0c83c 100644
--- a/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/internal/schema/utils/TestAvroSchemaEvolutionUtils.java
@@ -357,7 +357,7 @@ public class TestAvroSchemaEvolutionUtils {
.updateColumnType("enum", Types.StringType.get());
InternalSchema newSchema = SchemaChangeUtils.applyTableChanges2Schema(internalSchema, updateChange);
HoodieSchema newHoodieSchema = InternalSchemaConverter.convert(newSchema, hoodieSchema.getFullName());
- GenericRecord newRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newHoodieSchema.toAvroSchema(), Collections.emptyMap());
+ GenericRecord newRecord = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newHoodieSchema.toAvroSchema(), Collections.emptyMap());
Assertions.assertEquals("ENUM1", newRecord.get("enum"));
Assertions.assertEquals(GenericData.get().validate(newHoodieSchema.toAvroSchema(), newRecord), true);
@@ -423,7 +423,7 @@ public class TestAvroSchemaEvolutionUtils {
);
Schema newAvroSchema = InternalSchemaConverter.convert(newRecord, schema.getName()).toAvroSchema();
- GenericRecord newAvroRecord = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newAvroSchema, Collections.emptyMap());
+ GenericRecord newAvroRecord = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, newAvroSchema, Collections.emptyMap());
// test the correctly of rewrite
Assertions.assertEquals(GenericData.get().validate(newAvroSchema, newAvroRecord), true);
@@ -440,7 +440,7 @@ public class TestAvroSchemaEvolutionUtils {
InternalSchema internalSchemaRename = SchemaChangeUtils.applyTableChanges2Schema(internalSchema, updateChange);
HoodieSchema hoodieSchemaRename = InternalSchemaConverter.convert(internalSchemaRename, schema.getFullName());
Map<String, String> renameCols = InternalSchemaUtils.collectRenameCols(internalSchema, internalSchemaRename);
- GenericRecord avroRecordRename = HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, hoodieSchemaRename.toAvroSchema(), renameCols);
+ GenericRecord avroRecordRename = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(avroRecord, hoodieSchemaRename.toAvroSchema(), renameCols);
// test the correctly of rewrite
assertTrue(GenericData.get().validate(hoodieSchemaRename.toAvroSchema(), avroRecordRename));
}
diff --git a/hudi-examples/hudi-examples-common/src/main/java/org/apache/hudi/examples/common/HoodieExampleDataGenerator.java b/hudi-examples/hudi-examples-common/src/main/java/org/apache/hudi/examples/common/HoodieExampleDataGenerator.java
index 6bdb4b0a4620..7b51508b5fdc 100644
--- a/hudi-examples/hudi-examples-common/src/main/java/org/apache/hudi/examples/common/HoodieExampleDataGenerator.java
+++ b/hudi-examples/hudi-examples-common/src/main/java/org/apache/hudi/examples/common/HoodieExampleDataGenerator.java
@@ -199,7 +199,7 @@ public class HoodieExampleDataGenerator<T extends HoodieRecordPayload<T>> {
private Option<String> convertToString(HoodieRecord<T> record) {
try {
- String str = HoodieAvroUtils
+ String str = /*~~>*//*~~>*/HoodieAvroUtils
.bytesToAvro(((HoodieAvroPayload) record.getData()).getRecordBytes(), AVRO_SCHEMA)
.toString();
str = "{" + str.substring(str.indexOf("\"ts\":"));
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HiveSchemaUtils.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HiveSchemaUtils.java
index 561b73effdc7..5adb0c538b81 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HiveSchemaUtils.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/catalog/HiveSchemaUtils.java
@@ -18,8 +18,8 @@
package org.apache.hudi.table.catalog;
-import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.configuration.FlinkOptions;
@@ -64,7 +64,7 @@ public class HiveSchemaUtils {
public static org.apache.flink.table.api.Schema convertTableSchema(Table hiveTable) {
List<FieldSchema> allCols = hiveTable.getSd().getCols().stream()
// filter out the metadata columns
- .filter(s -> !HoodieAvroUtils.isMetadataField(s.getName()))
+ .filter(s -> !HoodieSchemaUtils.isMetadataField(s.getName()))
.collect(Collectors.toList());
// need to refactor the partition key field positions: they are not always in the last
allCols.addAll(hiveTable.getPartitionKeys());
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
index b41ca863b8d8..741e2402e1cf 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/OrcUtils.java
@@ -283,7 +283,7 @@ public class OrcUtils extends FileFormatUtils {
public void writeMetaFile(HoodieStorage storage, StoragePath filePath, Properties props) throws IOException {
// Since we are only interested in saving metadata to the footer, the schema, blocksizes and other
// parameters are not important.
- Schema schema = HoodieAvroUtils.getRecordKeySchema();
+ Schema schema = /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeySchema();
OrcFile.WriterOptions writerOptions = OrcFile.writerOptions(storage.getConf().unwrapAs(Configuration.class))
.fileSystem((FileSystem) storage.getFileSystem())
.setSchema(AvroOrcUtils.createOrcSchema(schema));
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
index 1de5513e424d..cfca23fa030b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/ParquetUtils.java
@@ -109,7 +109,7 @@ public class ParquetUtils extends FileFormatUtils {
*/
@Override
public Set<Pair<String, Long>> filterRowKeys(HoodieStorage storage, StoragePath filePath, Set<String> filter) {
- return filterParquetRowKeys(storage, new Path(filePath.toUri()), filter, HoodieAvroUtils.getRecordKeySchema());
+ return filterParquetRowKeys(storage, new Path(filePath.toUri()), filter, /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeySchema());
}
public static ParquetMetadata readMetadata(HoodieStorage storage, StoragePath parquetFilePath) {
@@ -399,7 +399,7 @@ public class ParquetUtils extends FileFormatUtils {
Properties props) throws IOException {
// Since we are only interested in saving metadata to the footer, the schema, blocksizes and other
// parameters are not important.
- Schema schema = HoodieAvroUtils.getRecordKeySchema();
+ Schema schema = /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeySchema();
MessageType type = Types.buildMessage().optional(PrimitiveType.PrimitiveTypeName.INT64).named("dummyint").named("dummy");
HoodieAvroWriteSupport writeSupport = new HoodieAvroWriteSupport(type, schema, Option.empty(), new Properties());
try (ParquetWriter writer = new ParquetWriter(new Path(filePath.toUri()), writeSupport, CompressionCodecName.UNCOMPRESSED, 1024, 1024)) {
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java
index ddc58d5a118e..0059227b6f0b 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java
@@ -140,13 +140,13 @@ public class HoodieAvroHFileWriter
if (isKeyAvailable) {
Object originalKey = keyExcludedRecord.get(keyFieldPos);
keyExcludedRecord.put(keyFieldPos, EMPTY_STRING);
- value = HoodieAvroUtils.avroToBytes(keyExcludedRecord);
+ value = /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(keyExcludedRecord);
keyExcludedRecord.put(keyFieldPos, originalKey);
isRecordSerialized = true;
}
}
if (!isRecordSerialized) {
- value = HoodieAvroUtils.avroToBytes(record);
+ value = /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record);
}
writer.append(recordKey, value);
if (hfileConfig.useBloomFilter()) {
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
index 2be916884f50..4b31599cd15f 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
@@ -97,7 +97,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
Set<String> existingFields = fileSchema.getFields().stream()
.map(Schema.Field::name)
.collect(Collectors.toSet());
- Schema prunedFileSchema = HoodieAvroUtils.projectSchema(fileSchema, requestedSchema.getFields().stream().map(HoodieSchemaField::name)
+ Schema prunedFileSchema = /*~~>*//*~~>*/HoodieAvroUtils.projectSchema(fileSchema, requestedSchema.getFields().stream().map(HoodieSchemaField::name)
.filter(existingFields::contains).collect(Collectors.toList()));
TypeDescription orcSchema = AvroOrcUtils.createOrcSchema(prunedFileSchema);
RecordReader recordReader = reader.rows(new Options(hadoopConf).schema(orcSchema));
@@ -106,7 +106,7 @@ public class HoodieAvroOrcReader extends HoodieAvroFileReader {
return recordIterator;
} else {
//TODO boundary for now to revisit HoodieAvroUtils in later pr to use HoodieSchema
- return new CloseableMappingIterator<>(recordIterator, data -> HoodieAvroUtils.rewriteRecordWithNewSchema(data, requestedSchema.getAvroSchema(), renamedColumns));
+ return new CloseableMappingIterator<>(recordIterator, data -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(data, requestedSchema.getAvroSchema(), renamedColumns));
}
} catch (IOException io) {
throw new HoodieIOException("Unable to create an ORC reader.", io);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
index 902a15bd03ef..0bc1d13dff93 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
@@ -188,7 +188,7 @@ public class HoodieAvroParquetReader extends HoodieAvroFileReader {
//TODO boundary for now to revisit in later pr to use HoodieSchema
Schema repairedFileSchema = AvroSchemaRepair.repairLogicalTypes(getSchema().getAvroSchema(), schema);
Option<Schema> promotedSchema = Option.empty();
- if (!renamedColumns.isEmpty() || HoodieAvroUtils.recordNeedsRewriteForExtendedAvroTypePromotion(repairedFileSchema, schema)) {
+ if (!renamedColumns.isEmpty() || /*~~>*//*~~>*/HoodieAvroUtils.recordNeedsRewriteForExtendedAvroTypePromotion(repairedFileSchema, schema)) {
AvroReadSupport.setAvroReadSchema(hadoopConf, repairedFileSchema);
AvroReadSupport.setRequestedProjection(hadoopConf, repairedFileSchema);
promotedSchema = Option.of(schema);
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java
index c0eaf16d08b5..8699ce4ffb5d 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java
@@ -40,6 +40,6 @@ public class HoodieAvroParquetReaderIterator extends ParquetReaderIterator<Index
@Override
public IndexedRecord next() {
- return HoodieAvroUtils.rewriteRecordWithNewSchema(super.next(), promotedSchema, renamedColumns);
+ return /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordWithNewSchema(super.next(), promotedSchema, renamedColumns);
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
index 7f71303c629e..0cab5f5eb1ac 100755
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
@@ -439,7 +439,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieSchema schema = getSimpleSchema();
List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords = records.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
@@ -475,7 +475,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieSchema schema = getSimpleSchema();
List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, 1000);
List<IndexedRecord> copyOfRecords = records.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
@@ -526,7 +526,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
List<IndexedRecord> records1 = SchemaTestUtil.generateTestRecords(0, 100);
HoodieSchema schema = getSimpleSchema();
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
@@ -545,7 +545,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
(FSDataOutputStream) storage.append(writer.getLogFile().getPath()));
List<IndexedRecord> records2 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema()))
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema()))
.collect(Collectors.toList());
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
dataBlock = getDataBlock(dataBlockType, records2, header);
@@ -564,7 +564,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
(FSDataOutputStream) storage.append(writer.getLogFile().getPath()));
List<IndexedRecord> records3 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords3 = records3.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema()))
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema()))
.collect(Collectors.toList());
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, getSimpleSchema().toString());
dataBlock = getDataBlock(dataBlockType, records3, header);
@@ -1163,7 +1163,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
// Write 1
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
@@ -1174,7 +1174,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
// Write 2
List<IndexedRecord> records2 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records2, header);
writer.appendBlock(dataBlock);
@@ -1207,7 +1207,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
@@ -1237,7 +1237,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "102");
List<IndexedRecord> records3 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords3 = records3.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records3, header);
writer.appendBlock(dataBlock);
@@ -1271,7 +1271,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -1307,7 +1307,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "103");
List<IndexedRecord> records3 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords3 = records3.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records3, header);
@@ -1342,7 +1342,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -1353,7 +1353,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101");
List<IndexedRecord> records2 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records2, header);
writer.appendBlock(dataBlock);
@@ -1486,7 +1486,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -1497,7 +1497,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101");
List<IndexedRecord> records2 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> allRecordsInserted = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
dataBlock = getDataBlock(HoodieLogBlockType.AVRO_DATA_BLOCK, records2, header);
writer.appendBlock(dataBlock);
allRecordsInserted.addAll(copyOfRecords1);
@@ -1595,7 +1595,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -1606,7 +1606,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101");
List<IndexedRecord> records2 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records2, header);
writer.appendBlock(dataBlock);
@@ -1729,7 +1729,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.TARGET_INSTANT_TIME, "100");
@@ -1799,7 +1799,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.TARGET_INSTANT_TIME, "100");
@@ -1890,7 +1890,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.TARGET_INSTANT_TIME, "100");
@@ -1959,7 +1959,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
List<IndexedRecord> records1 =
testUtil.generateHoodieTestRecords(0, recordKeyList, "0000/00/00", "100");
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -1970,7 +1970,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101");
List<IndexedRecord> records2 = testUtil.generateHoodieTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records2, header);
writer.appendBlock(dataBlock);
@@ -2225,7 +2225,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
SchemaTestUtil testUtil = new SchemaTestUtil();
List<IndexedRecord> records1 = testUtil.generateHoodieTestRecords(0, 100);
Set<String> recordKeysOfFirstTwoBatches = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())
.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString()).collect(Collectors.toSet());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
@@ -2238,7 +2238,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
// Write 2nd data block
List<IndexedRecord> records2 = testUtil.generateHoodieTestRecords(0, 100);
recordKeysOfFirstTwoBatches.addAll(records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())
.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString()).collect(Collectors.toList()));
header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "101");
@@ -2252,7 +2252,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
List<IndexedRecord> records3 = testUtil.generateHoodieTestRecords(0, 100);
Set<String> recordKeysOfFirstThreeBatches = new HashSet<>(recordKeysOfFirstTwoBatches);
recordKeysOfFirstThreeBatches.addAll(records3.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())
.get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString()).collect(Collectors.toList()));
header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "102");
@@ -2525,7 +2525,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieSchema schema = getSimpleSchema();
List<IndexedRecord> records1 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -2535,14 +2535,14 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
// append 100 more records
List<IndexedRecord> records2 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords2 = records2.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records2, header);
writer.appendBlock(dataBlock);
// append 100 more records
List<IndexedRecord> records3 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords3 = records3.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema()))
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema()))
.collect(Collectors.toList());
dataBlock = getDataBlock(DEFAULT_DATA_BLOCK_TYPE, records3, header);
writer.appendBlock(dataBlock);
@@ -2657,7 +2657,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieSchema schema = getSimpleSchema();
List<IndexedRecord> records1 = SchemaTestUtil.generateTestRecords(0, 100);
List<IndexedRecord> copyOfRecords1 = records1.stream()
- .map(record -> HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
+ .map(record -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, schema.toAvroSchema())).collect(Collectors.toList());
Map<HoodieLogBlock.HeaderMetadataType, String> header = new HashMap<>();
header.put(HoodieLogBlock.HeaderMetadataType.INSTANT_TIME, "100");
header.put(HoodieLogBlock.HeaderMetadataType.SCHEMA, schema.toString());
@@ -2769,7 +2769,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
HoodieSchema projectedSchema = HoodieSchemaUtils.generateProjectionSchema(schema, Collections.singletonList("name"));
- List<GenericRecord> projectedRecords = HoodieAvroUtils.rewriteRecords(records, projectedSchema.toAvroSchema());
+ List<GenericRecord> projectedRecords = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecords(records, projectedSchema.toAvroSchema());
try (Reader reader = HoodieLogFormat.newReader(storage, writer.getLogFile(), projectedSchema, false)) {
assertTrue(reader.hasNext(), "First block should be available");
@@ -2784,7 +2784,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
put(HoodieLogBlockType.AVRO_DATA_BLOCK, 0); // not supported
put(HoodieLogBlockType.HFILE_DATA_BLOCK, 0); // not supported
put(HoodieLogBlockType.PARQUET_DATA_BLOCK,
- HoodieAvroUtils.gteqAvro1_9()
+ /*~~>*//*~~>*/HoodieAvroUtils.gteqAvro1_9()
? getJavaVersion() == 17 || getJavaVersion() == 11 ? 1803 : 1802
: 1809);
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
index ac1e91944de9..98e0da310990 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestParquetUtils.java
@@ -157,7 +157,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
}
String filePath = Paths.get(basePath, "test.parquet").toUri().toString();
- Schema schema = HoodieAvroUtils.getRecordKeyPartitionPathSchema();
+ Schema schema = /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeyPartitionPathSchema();
writeParquetFile(typeCode, filePath, rowKeys, schema, true, partitionPath);
// Read and verify
@@ -329,7 +329,7 @@ public class TestParquetUtils extends HoodieCommonTestHarness {
}
private void writeParquetFile(String typeCode, String filePath, List<String> rowKeys) throws Exception {
- writeParquetFile(typeCode, filePath, rowKeys, HoodieAvroUtils.getRecordKeySchema(), false, "");
+ writeParquetFile(typeCode, filePath, rowKeys, /*~~>*//*~~>*/HoodieAvroUtils.getRecordKeySchema(), false, "");
}
private void writeParquetFile(String typeCode, String filePath, List<String> rowKeys, Schema schema, boolean addPartitionPathField, String partitionPath) throws Exception {
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
index cb6d03972da5..991438fb7025 100755
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java
@@ -93,7 +93,7 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
assert recordKeys.contains(rec.getRecordKey());
IndexedRecord originalRecord = originalRecords.get(rec.getRecordKey());
HoodieAvroPayload payload = (HoodieAvroPayload) rec.getData();
- Option<IndexedRecord> value = payload.getInsertValue(HoodieAvroUtils.addMetadataFields(getSimpleSchema().toAvroSchema()));
+ Option<IndexedRecord> value = payload.getInsertValue(/*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(getSimpleSchema().toAvroSchema()));
assertEquals(originalRecord, value.get());
}
@@ -143,7 +143,7 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
@ParameterizedTest
@ValueSource(booleans = {false, true})
public void testSimpleUpsert(boolean isCompressionEnabled) throws IOException, URISyntaxException {
- Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema().toAvroSchema());
+ Schema schema = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(getSimpleSchema().toAvroSchema());
try (BitCaskDiskMap records = new BitCaskDiskMap<>(basePath, new DefaultSerializer<>(), isCompressionEnabled)) {
SchemaTestUtil testUtil = new SchemaTestUtil();
@@ -221,7 +221,7 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
hoodieRecords = indexedRecords.stream()
.map(r -> new HoodieAvroRecord<>(new HoodieKey(UUID.randomUUID().toString(), "0000/00/00"),
new AvroBinaryTestPayload(
- Option.of(HoodieAvroUtils.rewriteRecord((GenericRecord) r, simpleSchemaWithMetadata.toAvroSchema())))))
+ Option.of(/*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) r, simpleSchemaWithMetadata.toAvroSchema())))))
.collect(Collectors.toList());
payloadSize = SpillableMapUtils.computePayloadSize(hoodieRecords.remove(0), new HoodieRecordSizeEstimator(schema.toAvroSchema()));
assertTrue(payloadSize > 0);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
index 6f6cf310a0d4..26a3f30f53af 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
@@ -519,7 +519,7 @@ public class TestHoodieTableMetadataUtil extends HoodieCommonTestHarness {
assertListEquality(expected, new ArrayList<>(HoodieTableMetadataUtil.getColumnsToIndex(tableConfig, metadataConfig,
Lazy.eagerly(Option.of(schema)), false, V1).keySet()));
//test with avro schema with meta cols
- HoodieSchema hoodieSchemaWithMetadataFields = HoodieSchema.fromAvroSchema(HoodieAvroUtils.addMetadataFields(schema.toAvroSchema()));
+ HoodieSchema hoodieSchemaWithMetadataFields = HoodieSchema.fromAvroSchema(/*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema.toAvroSchema()));
assertListEquality(expected, new ArrayList<>(HoodieTableMetadataUtil.getColumnsToIndex(tableConfig, metadataConfig,
Lazy.eagerly(Option.of(hoodieSchemaWithMetadataFields)), false, V1).keySet()));
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
index 4052098a07ff..22944fd9bffa 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/avro/HoodieAvroParquetReader.java
@@ -78,7 +78,7 @@ public class HoodieAvroParquetReader extends RecordReader<Void, ArrayWritable> {
// if exists read columns, we need to filter columns.
List<String> readColNames = Arrays.asList(HoodieColumnProjectionUtils.getReadColumnNames(conf));
if (!readColNames.isEmpty()) {
- Schema filterSchema = HoodieAvroUtils.generateProjectionSchema(baseSchema, readColNames);
+ Schema filterSchema = /*~~>*//*~~>*/HoodieAvroUtils.generateProjectionSchema(baseSchema, readColNames);
AvroReadSupport.setAvroReadSchema(conf, filterSchema);
AvroReadSupport.setRequestedProjection(conf, filterSchema);
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
index d910145bd464..67f0002f68c8 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/RealtimeCompactedRecordReader.java
@@ -169,7 +169,7 @@ public class RealtimeCompactedRecordReader extends AbstractRealtimeRecordReader
if (usesCustomPayload) {
// If using a custom payload, return only the projection fields. The readerSchema is a schema derived from
// the writerSchema with only the projection fields
- recordToReturn = HoodieAvroUtils.rewriteRecord((GenericRecord) rec.get().getData(), getReaderSchema().toAvroSchema());
+ recordToReturn = /*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) rec.get().getData(), getReaderSchema().toAvroSchema());
}
// we assume, a later safe record in the log, is newer than what we have in the map &
// replace it. Since we want to return an arrayWritable which is the same length as the elements in the latest
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
index 96210d4b9956..b617e3bc7efb 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
@@ -22,6 +22,7 @@ import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaField;
import org.apache.hudi.common.schema.HoodieSchemaType;
+import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieAvroSchemaException;
@@ -68,8 +69,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
-import static org.apache.hudi.avro.HoodieAvroUtils.isMetadataField;
-
/**
* Helper class to serialize hive writable type to avro record.
*/
@@ -483,7 +482,7 @@ public class HiveAvroSerializer {
GenericRecord newRecord = new GenericData.Record(newSchema.toAvroSchema());
boolean isSpecificRecord = oldRecord instanceof SpecificRecordBase;
for (HoodieSchemaField f : newSchema.getFields()) {
- if (!(isSpecificRecord && isMetadataField(f.name()))) {
+ if (!(isSpecificRecord && HoodieSchemaUtils.isMetadataField(f.name()))) {
copyOldValueOrSetDefault(oldRecord, newRecord, f);
}
}
diff --git a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieArrayWritableSchemaUtils.java b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieArrayWritableSchemaUtils.java
index a177ccd2eaa2..c0c144cc8d1a 100644
--- a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieArrayWritableSchemaUtils.java
+++ b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieArrayWritableSchemaUtils.java
@@ -91,7 +91,7 @@ public class HoodieArrayWritableSchemaUtils {
// projection will keep the size from the "from" schema because it gets recycled
// and if the size changes the reader will fail
boolean noFieldsRenaming = renameCols.isEmpty();
- String namePrefix = createNamePrefix(noFieldsRenaming, fieldNames);
+ String namePrefix = /*~~>*/createNamePrefix(noFieldsRenaming, fieldNames);
Writable[] values = new Writable[Math.max(fields.size(), arrayWritable.get().length)];
for (int i = 0; i < fields.size(); i++) {
HoodieSchemaField newField = newSchema.getFields().get(i);
@@ -99,14 +99,14 @@ public class HoodieArrayWritableSchemaUtils {
fieldNames.push(newFieldName);
Option<HoodieSchemaField> oldFieldOpt = noFieldsRenaming
? oldSchema.getField(newFieldName)
- : oldSchema.getField(getOldFieldNameWithRenaming(namePrefix, newFieldName, renameCols));
+ : oldSchema.getField(/*~~>*/getOldFieldNameWithRenaming(namePrefix, newFieldName, renameCols));
if (oldFieldOpt.isPresent()) {
HoodieSchemaField oldField = oldFieldOpt.get();
values[i] = rewriteRecordWithNewSchema(arrayWritable.get()[oldField.pos()], oldField.schema(), newField.schema(), renameCols, fieldNames);
} else if (newField.defaultVal().isPresent() && newField.defaultVal().get().equals(HoodieSchema.NULL_VALUE)) {
values[i] = NullWritable.get();
} else if (!newField.schema().isNullable() && newField.defaultVal().isEmpty()) {
- throw new SchemaCompatibilityException("Field " + createFullName(fieldNames) + " has no default value and is non-nullable");
+ throw new SchemaCompatibilityException("Field " + /*~~>*/createFullName(fieldNames) + " has no default value and is non-nullable");
} else if (newField.defaultVal().isPresent()) {
switch (HoodieSchemaUtils.getNonNullTypeFromUnion(newField.schema()).getType()) {
case BOOLEAN:
@@ -128,7 +128,7 @@ public class HoodieArrayWritableSchemaUtils {
values[i] = new Text(newField.defaultVal().toString());
break;
default:
- throw new SchemaCompatibilityException("Field " + createFullName(fieldNames) + " has no default value");
+ throw new SchemaCompatibilityException("Field " + /*~~>*/createFullName(fieldNames) + " has no default value");
}
}
fieldNames.pop();
@@ -229,7 +229,7 @@ public class HoodieArrayWritableSchemaUtils {
break;
case DATE:
if (oldSchema.getType() == HoodieSchemaType.STRING) {
- return HoodieHiveUtils.getDateWriteable((HoodieAvroUtils.fromJavaDate(java.sql.Date.valueOf(writable.toString()))));
+ return HoodieHiveUtils.getDateWriteable((/*~~>*//*~~>*/HoodieAvroUtils.fromJavaDate(java.sql.Date.valueOf(writable.toString()))));
}
break;
case LONG:
@@ -268,7 +268,7 @@ public class HoodieArrayWritableSchemaUtils {
return new Text(StringUtils.fromUTF8Bytes(((BytesWritable) writable).getBytes()));
}
if (oldSchema.getType() == HoodieSchemaType.DATE) {
- return new Text(toJavaDate(((IntWritable) writable).get()).toString());
+ return new Text(/*~~>*/toJavaDate(((IntWritable) writable).get()).toString());
}
if (oldSchema.getType() == HoodieSchemaType.INT
|| oldSchema.getType() == HoodieSchemaType.LONG
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java
index e90956d175e1..e7715eb16d08 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/hive/TestHoodieCombineHiveInputFormat.java
@@ -336,7 +336,7 @@ public class TestHoodieCombineHiveInputFormat extends HoodieCommonTestHarness {
String hiveColumnNames = fields.stream().map(HoodieSchemaField::name).collect(Collectors.joining(","));
hiveColumnNames = hiveColumnNames + ",year,month,day";
- String modifiedHiveColumnTypes = HoodieAvroUtils.addMetadataColumnTypes(tripsHiveColumnTypes);
+ String modifiedHiveColumnTypes = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataColumnTypes(tripsHiveColumnTypes);
modifiedHiveColumnTypes = modifiedHiveColumnTypes + ",string,string,string";
jobConf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
jobConf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, modifiedHiveColumnTypes);
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java
index 2a258d40123d..e258aa061ca5 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/realtime/TestHoodieRealtimeRecordReader.java
@@ -970,7 +970,7 @@ public class TestHoodieRealtimeRecordReader {
// In some queries, generic records that Hudi gets are just part of the full records.
// Here test the case that some fields are missing in the record.
- Schema schemaWithMetaFields = HoodieAvroUtils.addMetadataFields(schema.toAvroSchema());
+ Schema schemaWithMetaFields = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataFields(schema.toAvroSchema());
ArrayWritable aWritable2 = (ArrayWritable) HoodieRealtimeRecordReaderUtils.avroToArrayWritable(record, schemaWithMetaFields);
assertEquals(schemaWithMetaFields.getFields().size(), aWritable2.get().length);
}
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
index a8148715dc87..a6dfd329bb6a 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/testutils/InputFormatTestUtil.java
@@ -320,7 +320,7 @@ public class InputFormatTestUtil {
List<IndexedRecord> records = SchemaTestUtil.generateTestRecords(0, numberOfRecords);
HoodieSchema hoodieFieldsSchema = HoodieSchemaUtils.addMetadataFields(schema);
for (IndexedRecord record : records) {
- GenericRecord p = HoodieAvroUtils.rewriteRecord((GenericRecord) record, hoodieFieldsSchema.toAvroSchema());
+ GenericRecord p = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) record, hoodieFieldsSchema.toAvroSchema());
p.put(HoodieRecord.RECORD_KEY_METADATA_FIELD, UUID.randomUUID().toString());
p.put(HoodieRecord.PARTITION_PATH_METADATA_FIELD, "0000/00/00");
p.put(HoodieRecord.COMMIT_TIME_METADATA_FIELD, commitNumber);
@@ -468,7 +468,7 @@ public class InputFormatTestUtil {
String hiveColumnNames = fields.stream().filter(field -> !field.name().equalsIgnoreCase("datestr"))
.map(Schema.Field::name).collect(Collectors.joining(","));
hiveColumnNames = hiveColumnNames + ",datestr";
- String modifiedHiveColumnTypes = HoodieAvroUtils.addMetadataColumnTypes(hiveColumnTypes);
+ String modifiedHiveColumnTypes = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataColumnTypes(hiveColumnTypes);
modifiedHiveColumnTypes = modifiedHiveColumnTypes + ",string";
jobConf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
jobConf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, modifiedHiveColumnTypes);
@@ -495,7 +495,7 @@ public class InputFormatTestUtil {
String hiveColumnNames = fields.stream().filter(field -> !field.name().equalsIgnoreCase("datestr"))
.map(Schema.Field::name).collect(Collectors.joining(","));
hiveColumnNames = hiveColumnNames + ",datestr";
- String modifiedHiveColumnTypes = HoodieAvroUtils.addMetadataColumnTypes(hiveColumnTypes);
+ String modifiedHiveColumnTypes = /*~~>*//*~~>*/HoodieAvroUtils.addMetadataColumnTypes(hiveColumnTypes);
modifiedHiveColumnTypes = modifiedHiveColumnTypes + ",string";
jobConf.set(hive_metastoreConstants.META_TABLE_COLUMNS, hiveColumnNames);
jobConf.set(hive_metastoreConstants.META_TABLE_COLUMN_TYPES, modifiedHiveColumnTypes);
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHoodieArrayWritableSchemaUtils.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHoodieArrayWritableSchemaUtils.java
index 026a0cdbea61..92e29e9bcd7e 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHoodieArrayWritableSchemaUtils.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/utils/TestHoodieArrayWritableSchemaUtils.java
@@ -131,7 +131,7 @@ public class TestHoodieArrayWritableSchemaUtils {
HoodieSchema newSchema = HoodieSchema.create(HoodieSchemaType.DATE);
Writable oldWritable = new Text("2023-01-01");
Writable result = HoodieArrayWritableSchemaUtils.rewritePrimaryType(oldWritable, oldSchema, newSchema);
- Writable expected = HoodieHiveUtils.getDateWriteable(HoodieAvroUtils.fromJavaDate(Date.valueOf("2023-01-01")));
+ Writable expected = HoodieHiveUtils.getDateWriteable(/*~~>*//*~~>*/HoodieAvroUtils.fromJavaDate(Date.valueOf("2023-01-01")));
assertEquals(expected, result);
validateRewriteWithAvro(oldWritable, oldSchema, result, newSchema);
}
@@ -335,7 +335,7 @@ public class TestHoodieArrayWritableSchemaUtils {
} else if (javaInput instanceof byte[]) {
javaInput = ByteBuffer.wrap((byte[]) javaInput);
}
- Object javaOutput = HoodieAvroUtils.rewritePrimaryType(javaInput, oldSchema.toAvroSchema(), newSchema.toAvroSchema());
+ Object javaOutput = /*~~>*//*~~>*/HoodieAvroUtils.rewritePrimaryType(javaInput, oldSchema.toAvroSchema(), newSchema.toAvroSchema());
Object javaExpected = ObjectInspectorConverters.getConverter(writableOINew, newObjectInspector).convert(newWritable);
if (newSchema.getType() == HoodieSchemaType.DECIMAL) {
@@ -343,7 +343,7 @@ public class TestHoodieArrayWritableSchemaUtils {
BigDecimal expectedDecimal = getDecimalValue(javaExpected, (HoodieSchema.Decimal) newSchema);
assertEquals(0, outputDecimal.compareTo(expectedDecimal));
} else if (newSchema.getType() == HoodieSchemaType.DATE) {
- assertEquals(HoodieAvroUtils.toJavaDate((int) javaOutput), javaExpected);
+ assertEquals(/*~~>*//*~~>*/HoodieAvroUtils.toJavaDate((int) javaOutput), javaExpected);
} else {
assertEquals(javaOutput, javaExpected);
}
diff --git a/hudi-kafka-connect/src/test/java/org/apache/hudi/writers/TestAbstractConnectWriter.java b/hudi-kafka-connect/src/test/java/org/apache/hudi/writers/TestAbstractConnectWriter.java
index ce279c89df6a..fffa7a11a38d 100644
--- a/hudi-kafka-connect/src/test/java/org/apache/hudi/writers/TestAbstractConnectWriter.java
+++ b/hudi-kafka-connect/src/test/java/org/apache/hudi/writers/TestAbstractConnectWriter.java
@@ -87,7 +87,7 @@ public class TestAbstractConnectWriter {
inputRecords = SchemaTestUtil.generateTestJsonRecords(0, NUM_RECORDS);
expectedRecords = ((List<String>) inputRecords).stream().map(s -> {
try {
- return HoodieAvroUtils.rewriteRecord((GenericRecord) reader.read(null, DecoderFactory.get().jsonDecoder(schema.toAvroSchema(), s)),
+ return /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) reader.read(null, DecoderFactory.get().jsonDecoder(schema.toAvroSchema(), s)),
schema.toAvroSchema());
} catch (IOException exception) {
throw new HoodieException("Error converting JSON records to AVRO");
@@ -97,7 +97,7 @@ public class TestAbstractConnectWriter {
case AVRO:
formatConverter = AbstractConnectWriter.KAFKA_AVRO_CONVERTER;
inputRecords = SchemaTestUtil.generateTestRecords(0, NUM_RECORDS);
- expectedRecords = inputRecords.stream().map(s -> HoodieAvroUtils.rewriteRecord((GenericRecord) s, schema.toAvroSchema()))
+ expectedRecords = inputRecords.stream().map(s -> /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord((GenericRecord) s, schema.toAvroSchema()))
.map(p -> convertToHoodieRecords(p, p.get(RECORD_KEY_INDEX).toString(), "000/00/00")).collect(Collectors.toList());
break;
default:
diff --git a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java
index 0a98a97fcb62..2e8680e32b8d 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java
+++ b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/bootstrap/SparkFullBootstrapDataProviderBase.java
@@ -77,7 +77,7 @@ public abstract class SparkFullBootstrapDataProviderBase extends FullRecordBoots
RDD<GenericRecord> genericRecords = HoodieSparkUtils.createRdd(inputDataset, structName, namespace, false,
Option.empty());
return genericRecords.toJavaRDD().map(gr -> {
- String orderingVal = HoodieAvroUtils.getNestedFieldValAsString(
+ String orderingVal = /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(
gr, orderingFieldsStr, false, props.getBoolean(
KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.key(),
Boolean.parseBoolean(KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED.defaultValue())));
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceUtils.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceUtils.java
index 98793ed4c33d..752fe9d8f8cc 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceUtils.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/TestDataSourceUtils.java
@@ -139,17 +139,17 @@ public class TestDataSourceUtils extends HoodieClientTestBase {
record.put("event_cost2", genericFixed);
record.put("event_cost3", genericFixed);
- assertEquals(LocalDate.ofEpochDay(18000).toString(), HoodieAvroUtils.getNestedFieldValAsString(record, "event_date1",
+ assertEquals(LocalDate.ofEpochDay(18000).toString(), /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_date1",
true, false));
- assertEquals(LocalDate.ofEpochDay(18001).toString(), HoodieAvroUtils.getNestedFieldValAsString(record, "event_date2",
+ assertEquals(LocalDate.ofEpochDay(18001).toString(), /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_date2",
true, false));
- assertEquals(LocalDate.ofEpochDay(18002).toString(), HoodieAvroUtils.getNestedFieldValAsString(record, "event_date3",
+ assertEquals(LocalDate.ofEpochDay(18002).toString(), /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_date3",
true, false));
- assertEquals("Hudi Meetup", HoodieAvroUtils.getNestedFieldValAsString(record, "event_name", true, false));
- assertEquals("Hudi PMC", HoodieAvroUtils.getNestedFieldValAsString(record, "event_organizer", true, false));
- assertEquals(bigDecimal.toString(), HoodieAvroUtils.getNestedFieldValAsString(record, "event_cost1", true, false));
- assertEquals(bigDecimal.toString(), HoodieAvroUtils.getNestedFieldValAsString(record, "event_cost2", true, false));
- assertEquals(bigDecimal.toString(), HoodieAvroUtils.getNestedFieldValAsString(record, "event_cost3", true, false));
+ assertEquals("Hudi Meetup", /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_name", true, false));
+ assertEquals("Hudi PMC", /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_organizer", true, false));
+ assertEquals(bigDecimal.toString(), /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_cost1", true, false));
+ assertEquals(bigDecimal.toString(), /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_cost2", true, false));
+ assertEquals(bigDecimal.toString(), /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldValAsString(record, "event_cost3", true, false));
}
@Test
@@ -301,8 +301,8 @@ public class TestDataSourceUtils extends HoodieClientTestBase {
.findFirst().get();
IndexedRecord record = hoodieMetadataPayload.getData().getInsertValue(null).get();
- byte[] recordToBytes = HoodieAvroUtils.avroToBytes(record);
- GenericRecord genericRecord = HoodieAvroUtils.bytesToAvro(recordToBytes, record.getSchema());
+ byte[] recordToBytes = /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record);
+ GenericRecord genericRecord = /*~~>*//*~~>*/HoodieAvroUtils.bytesToAvro(recordToBytes, record.getSchema());
HoodieMetadataPayload genericRecordHoodieMetadataPayload = new HoodieMetadataPayload(Option.of(genericRecord));
byte[] bytes = SerializationUtils.serialize(genericRecordHoodieMetadataPayload);
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
index 081b962b44a8..d6335f6fd4e0 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestTableSchemaEvolution.java
@@ -384,7 +384,7 @@ public class TestTableSchemaEvolution extends HoodieClientTestBase {
return records.stream().map(r -> {
HoodieKey key = r.getKey();
GenericRecord payload = (GenericRecord) ((HoodieAvroIndexedRecord) r).getData();
- GenericRecord newPayload = HoodieAvroUtils.rewriteRecord(payload, newSchema.toAvroSchema());
+ GenericRecord newPayload = /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecord(payload, newSchema.toAvroSchema());
return new HoodieAvroIndexedRecord(key, newPayload);
}).collect(Collectors.toList());
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieIndex.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieIndex.java
index 9734c7d2492b..0ede960cea3d 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieIndex.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieIndex.java
@@ -416,7 +416,7 @@ public class TestHoodieIndex extends TestHoodieMetadataBase {
}
// We create three parquet files, each having one record (two different partitions)
- HoodieSparkWriteableTestTable testTable = HoodieSparkWriteableTestTable.of(metaClient, addMetadataFields(SIMPLE_RECORD_SCHEMA.toAvroSchema()), metadataWriter);
+ HoodieSparkWriteableTestTable testTable = HoodieSparkWriteableTestTable.of(metaClient, /*~~>*/addMetadataFields(SIMPLE_RECORD_SCHEMA.toAvroSchema()), metadataWriter);
final String fileId1 = "fileID1";
final String fileId2 = "fileID2";
final String fileId3 = "fileID3";
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/KafkaOffsetPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/KafkaOffsetPostProcessor.java
index b93f73ad4005..248764a7cfee 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/KafkaOffsetPostProcessor.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/KafkaOffsetPostProcessor.java
@@ -73,7 +73,7 @@ public class KafkaOffsetPostProcessor extends SchemaPostProcessor {
}
try {
List<Schema.Field> newFieldList = fieldList.stream()
- .map(HoodieAvroUtils::createNewSchemaField).collect(Collectors.toList());
+ .map(/*~~>*/HoodieAvroUtils::/*~~>*/createNewSchemaField).collect(Collectors.toList());
// handle case where source schema provider may have already set 1 or more of these fields
if (!fieldNames.contains(KAFKA_SOURCE_OFFSET_COLUMN)) {
newFieldList.add(new Schema.Field(KAFKA_SOURCE_OFFSET_COLUMN, Schema.create(Schema.Type.LONG), "offset column", 0));
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/LazyCastingIterator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/LazyCastingIterator.java
index eb654a69269c..e0ea4d56e5a5 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/LazyCastingIterator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/LazyCastingIterator.java
@@ -37,6 +37,6 @@ public class LazyCastingIterator extends LazyIterableIterator<GenericRecord,Gene
@Override
protected GenericRecord computeNext() {
- return HoodieAvroUtils.rewriteRecordDeep(inputItr.next(), targetSchema);
+ return /*~~>*//*~~>*/HoodieAvroUtils.rewriteRecordDeep(inputItr.next(), targetSchema);
}
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java
index a479e67af5cf..c49c260dd48f 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DeleteSupportSchemaPostProcessor.java
@@ -56,7 +56,7 @@ public class DeleteSupportSchemaPostProcessor extends SchemaPostProcessor {
List<Schema.Field> targetFields = new ArrayList<>(sourceFields.size() + 1);
// copy existing columns
for (Schema.Field sourceField : sourceFields) {
- targetFields.add(createNewSchemaField(sourceField));
+ targetFields.add(/*~~>*/createNewSchemaField(sourceField));
}
// add _hoodie_is_deleted column
targetFields.add(new Schema.Field(HoodieRecord.HOODIE_IS_DELETED_FIELD, Schema.create(Schema.Type.BOOLEAN), null, false));
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java
index 020825e2d6f7..7b8bbf34c3eb 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/DropColumnSchemaPostProcessor.java
@@ -83,7 +83,7 @@ public class DropColumnSchemaPostProcessor extends SchemaPostProcessor {
for (Schema.Field sourceField : sourceFields) {
if (!columnsToDelete.contains(sourceField.name().toLowerCase(Locale.ROOT))) {
- targetFields.add(createNewSchemaField(sourceField));
+ targetFields.add(/*~~>*/createNewSchemaField(sourceField));
}
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/AddPrimitiveColumnSchemaPostProcessor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/AddPrimitiveColumnSchemaPostProcessor.java
index 2be4942ab80b..54d800fe3bcb 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/AddPrimitiveColumnSchemaPostProcessor.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/schema/postprocessor/add/AddPrimitiveColumnSchemaPostProcessor.java
@@ -69,7 +69,7 @@ public class AddPrimitiveColumnSchemaPostProcessor extends SchemaPostProcessor {
for (Schema.Field sourceField : sourceFields) {
- targetFields.add(createNewSchemaField(sourceField));
+ targetFields.add(/*~~>*/createNewSchemaField(sourceField));
}
// add new column to the end
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/SanitizationUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/SanitizationUtils.java
index 3c4bf133221f..eb7d735091e4 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/SanitizationUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/SanitizationUtils.java
@@ -96,7 +96,7 @@ public class SanitizationUtils {
StructField[] structFields = structType.fields();
for (StructField s : structFields) {
DataType currFieldDataTypeSanitized = sanitizeDataTypeForAvro(s.dataType(), invalidCharMask);
- StructField structFieldCopy = new StructField(HoodieAvroUtils.sanitizeName(s.name(), invalidCharMask),
+ StructField structFieldCopy = new StructField(/*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(s.name(), invalidCharMask),
currFieldDataTypeSanitized, s.nullable(), s.metadata());
sanitizedStructType = sanitizedStructType.add(structFieldCopy);
}
@@ -113,7 +113,7 @@ public class SanitizationUtils {
// which are structurally similar except for actual column names itself. So casting is safe and sufficient.
targetDataset = targetDataset.withColumn(sf.name(), targetDataset.col(sf.name()).cast(sanitizedFieldDataType));
}
- String possibleRename = HoodieAvroUtils.sanitizeName(sf.name(), invalidCharMask);
+ String possibleRename = /*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(sf.name(), invalidCharMask);
if (!sf.name().equals(possibleRename)) {
targetDataset = targetDataset.withColumnRenamed(sf.name(), possibleRename);
}
@@ -182,7 +182,7 @@ public class SanitizationUtils {
} else if (kv.getValue() instanceof String) {
String currentStrValue = (String) kv.getValue();
if (kv.getKey().equals(AVRO_FIELD_NAME_KEY)) {
- return Pair.of(kv.getKey(), HoodieAvroUtils.sanitizeName(currentStrValue, invalidCharMask));
+ return Pair.of(kv.getKey(), /*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(currentStrValue, invalidCharMask));
}
return Pair.of(kv.getKey(), currentStrValue);
} else {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java
index da7c02ec20e9..7aed99712eb9 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java
@@ -105,7 +105,7 @@ public class HoodieStreamerUtils {
return avroRDDOptional.map(avroRDD -> {
SerializableSchema avroSchema = new SerializableSchema(schemaProvider.getTargetHoodieSchema().toAvroSchema());
- SerializableSchema processedAvroSchema = new SerializableSchema(isDropPartitionColumns(props) ? HoodieAvroUtils.removeMetadataFields(avroSchema.get()) : avroSchema.get());
+ SerializableSchema processedAvroSchema = new SerializableSchema(isDropPartitionColumns(props) ? /*~~>*//*~~>*/HoodieAvroUtils.removeMetadataFields(avroSchema.get()) : avroSchema.get());
JavaRDD<Either<HoodieRecord,String>> records;
if (recordType == HoodieRecord.HoodieRecordType.AVRO) {
records = avroRDD.mapPartitions(
@@ -130,11 +130,11 @@ public class HoodieStreamerUtils {
}
}
HoodieKey hoodieKey = new HoodieKey(builtinKeyGenerator.getRecordKey(genRec), builtinKeyGenerator.getPartitionPath(genRec));
- GenericRecord gr = isDropPartitionColumns(props) ? HoodieAvroUtils.removeFields(genRec, partitionColumns) : genRec;
+ GenericRecord gr = isDropPartitionColumns(props) ? /*~~>*//*~~>*/HoodieAvroUtils.removeFields(genRec, partitionColumns) : genRec;
boolean isDelete = AvroRecordContext.getFieldAccessorInstance().isDeleteRecord(gr, deleteContext);
Comparable orderingValue = shouldUseOrderingField
? OrderingValues.create(orderingFieldsStr.split(","),
- field -> (Comparable) HoodieAvroUtils.getNestedFieldVal(gr, field, false, useConsistentLogicalTimestamp))
+ field -> (Comparable) /*~~>*//*~~>*/HoodieAvroUtils.getNestedFieldVal(gr, field, false, useConsistentLogicalTimestamp))
: null;
HoodieRecord record = shouldUseOrderingField ? HoodieRecordUtils.createHoodieRecord(gr, orderingValue, hoodieKey, payloadClassName, requiresPayload, isDelete)
: HoodieRecordUtils.createHoodieRecord(gr, hoodieKey, payloadClassName, requiresPayload, isDelete);
@@ -156,7 +156,7 @@ public class HoodieStreamerUtils {
BuiltinKeyGenerator builtinKeyGenerator = (BuiltinKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
StructType baseStructType = AvroConversionUtils.convertAvroSchemaToStructType(processedAvroSchema.get());
StructType targetStructType = isDropPartitionColumns(props) ? AvroConversionUtils
- .convertAvroSchemaToStructType(HoodieAvroUtils.removeFields(processedAvroSchema.get(), partitionColumns)) : baseStructType;
+ .convertAvroSchemaToStructType(/*~~>*//*~~>*/HoodieAvroUtils.removeFields(processedAvroSchema.get(), partitionColumns)) : baseStructType;
HoodieAvroDeserializer deserializer = SparkAdapterSupport$.MODULE$.sparkAdapter().createAvroDeserializer(processedAvroSchema.get(), baseStructType);
return new CloseableMappingIterator<>(ClosableIterator.wrap(itr), rec -> {
@@ -199,7 +199,7 @@ public class HoodieStreamerUtils {
}
}
try {
- return Either.right(HoodieAvroUtils.safeAvroToJsonString(genRec));
+ return Either.right(/*~~>*//*~~>*/HoodieAvroUtils.safeAvroToJsonString(genRec));
} catch (Exception ex) {
throw new HoodieException("Failed to convert illegal record to json", ex);
}
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java
index d23e91b18a1d..b017653728c6 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/SourceFormatAdapter.java
@@ -259,7 +259,7 @@ public class SourceFormatAdapter implements Closeable {
// Decimal fields need additional decoding from json generated by kafka-connect. JSON -> ROW conversion is done through
// a spark library that has not implemented this decoding
if (isFieldNameSanitizingEnabled()
- || (HoodieAvroUtils.hasDecimalField(sourceSchema) && source instanceof KafkaSource)) {
+ || (/*~~>*//*~~>*/HoodieAvroUtils.hasDecimalField(sourceSchema) && source instanceof KafkaSource)) {
StructType dataType = AvroConversionUtils.convertAvroSchemaToStructType(sourceSchema);
JavaRDD<Row> rowRDD = transformJsonToRowRdd(r);
if (rowRDD != null) {
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index dc5513d1fdab..266c14012895 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -1089,7 +1089,7 @@ public class StreamSync implements Serializable, Closeable {
private void reInitWriteClient(HoodieSchema sourceSchema, HoodieSchema targetSchema, Option<JavaRDD<HoodieRecord>> recordsOpt, HoodieTableMetaClient metaClient) throws IOException {
LOG.info("Setting up new Hoodie Write Client");
if (HoodieStreamerUtils.isDropPartitionColumns(props)) {
- targetSchema = HoodieSchema.fromAvroSchema(HoodieAvroUtils.removeFields(targetSchema.toAvroSchema(), HoodieStreamerUtils.getPartitionColumns(props)));
+ targetSchema = HoodieSchema.fromAvroSchema(/*~~>*//*~~>*/HoodieAvroUtils.removeFields(targetSchema.toAvroSchema(), HoodieStreamerUtils.getPartitionColumns(props)));
}
final Pair<HoodieWriteConfig, HoodieSchema> initialWriteConfigAndSchema = getHoodieClientConfigAndWriterSchema(targetSchema, true, metaClient);
final HoodieWriteConfig initialWriteConfig = initialWriteConfigAndSchema.getLeft();
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
index 9d92cb84ed47..f1a9bd4b2aab 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
@@ -252,7 +252,7 @@ public class TestHoodieDeltaStreamerSchemaEvolutionBase extends HoodieDeltaStrea
List<GenericRecord> records = HoodieSparkUtils.createRdd(df, HOODIE_RECORD_STRUCT_NAME, HOODIE_RECORD_NAMESPACE, false, Option.empty()).toJavaRDD().collect();
try (Producer<String, byte[]> producer = new KafkaProducer<>(getProducerProperties())) {
for (GenericRecord record : records) {
- producer.send(new ProducerRecord<>(topicName, 0, "key", HoodieAvroUtils.avroToBytes(record)));
+ producer.send(new ProducerRecord<>(topicName, 0, "key", /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(record)));
}
}
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroKafkaSource.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroKafkaSource.java
index c3c938e452c8..ef15f5ed8c79 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroKafkaSource.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestAvroKafkaSource.java
@@ -124,7 +124,7 @@ public class TestAvroKafkaSource extends SparkClientFunctionalTestHarness {
try (Producer<String, byte[]> producer = new KafkaProducer<>(config)) {
for (int i = 0; i < genericRecords.size(); i++) {
// use consistent keys to get even spread over partitions for test expectations
- producer.send(new ProducerRecord<>(topic, i % numPartitions, "key", HoodieAvroUtils.avroToBytes(genericRecords.get(i))));
+ producer.send(new ProducerRecord<>(topic, i % numPartitions, "key", /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(genericRecords.get(i))));
}
}
}
@@ -135,7 +135,7 @@ public class TestAvroKafkaSource extends SparkClientFunctionalTestHarness {
try (Producer<String, byte[]> producer = new KafkaProducer<>(config)) {
for (int i = 0; i < genericRecords.size(); i++) {
// null kafka key
- producer.send(new ProducerRecord<>(topic, i % numPartitions, null, HoodieAvroUtils.avroToBytes(genericRecords.get(i))));
+ producer.send(new ProducerRecord<>(topic, i % numPartitions, null, /*~~>*//*~~>*/HoodieAvroUtils.avroToBytes(genericRecords.get(i))));
}
}
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGenericRddTransform.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGenericRddTransform.java
index 8adfdb4dc377..565cbf5bddfd 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGenericRddTransform.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestGenericRddTransform.java
@@ -63,7 +63,7 @@ public class TestGenericRddTransform extends SparkClientFunctionalTestHarness {
@Test
public void testGenericRddConvert() {
String fieldToNull = "partition_path";
- String schemaStr = makeFieldNonNull(HoodieTestDataGenerator.AVRO_SCHEMA, fieldToNull, "").toString();
+ String schemaStr = /*~~>*/makeFieldNonNull(HoodieTestDataGenerator.AVRO_SCHEMA, fieldToNull, "").toString();
HoodieTestDataGenerator datagen = new HoodieTestDataGenerator();
List<GenericRecord> recs = datagen.generateGenericRecords(10);
for (int i = 0; i < recs.size(); i++) {
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SanitizationTestUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SanitizationTestUtils.java
index e2a2b53bdd38..25cc0255b324 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SanitizationTestUtils.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/SanitizationTestUtils.java
@@ -43,7 +43,7 @@ public class SanitizationTestUtils {
public static String invalidCharMask = SCHEMA_FIELD_NAME_INVALID_CHAR_MASK.defaultValue();
private static String sanitizeIfNeeded(String src, boolean shouldSanitize) {
- return shouldSanitize ? HoodieAvroUtils.sanitizeName(src, invalidCharMask) : src;
+ return shouldSanitize ? /*~~>*//*~~>*/HoodieAvroUtils.sanitizeName(src, invalidCharMask) : src;
}
protected static StructType getSchemaWithProperNaming() {