You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/09/17 11:37:11 UTC
(hudi) branch release-1.2.1 updated (609be82e4e55 -> c14a871d57e8)
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a change to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 609be82e4e55 fix(build): exclude nested .mvn directories from the RAT check (#19462)
new a9b96378ecc8 fix(common): resolve nested Avro records by class built by Hudi (#19956)
new c14a871d57e8 perf(common): cache nested Avro record class lookup (#19972)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 30 +++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
(hudi) 02/02: perf(common): cache nested Avro record class lookup (#19972)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c14a871d57e80493cc9e336a044a25c6000d977b
Author: voonhous <vo...@gmail.com>
AuthorDate: Thu Sep 17 18:45:36 2026 +0800
perf(common): cache nested Avro record class lookup (#19972)
#19956 replaced SpecificData#getClass with an uncached Class.forName
per nested record, dropping the class cache Avro keeps by schema full
name. Delete block reads hit it twice per delete record (the record
and its ordering-value wrapper), costing ~450-880 ns per lookup vs
~3-5 ns before.
Cache the resolved class by schema full name, as SpecificData does.
(cherry picked from commit 5bccaabf56b60a0efd9665b5b6c9d115993f1e8e)
---
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 24 ++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
index d97fe8fdcb60..28d08c25ae86 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -98,6 +98,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.TimeZone;
import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import static org.apache.avro.Schema.Type.ARRAY;
@@ -124,6 +125,13 @@ public class HoodieAvroUtils {
private static final Properties PROPERTIES = new Properties();
+ /**
+ * Generated SpecificRecord classes keyed by schema full name, mirroring the class cache inside
+ * {@link SpecificData#getClass(Schema)} that the direct {@code Class.forName} lookup in
+ * {@link #getSpecificRecordClass(Schema, SpecificData)} bypasses. Bounded because only compiled SCHEMA$ schemas reach it.
+ */
+ private static final Map<String, Class<? extends SpecificRecordBase>> SPECIFIC_RECORD_CLASS_CACHE = new ConcurrentHashMap<>();
+
/**
* Resolves the Avro library version, preferring Maven's generated pom.properties over
* {@link Package#getImplementationVersion()}. The latter comes from whatever manifest happens to
@@ -1563,14 +1571,18 @@ public class HoodieAvroUtils {
* {@link SpecificData#getClass(Schema)}), which rejects Hudi's generated classes.
*
* <p>Only pass schemas taken from a compiled SCHEMA$, never a schema read from storage, which is what the validation guards against.
+ *
+ * <p>The result is cached by schema full name so the per-record path does not pay {@code Class.forName}.
*/
private static Class<? extends SpecificRecordBase> getSpecificRecordClass(Schema recordSchema, SpecificData specificData) {
- String className = SpecificData.getClassName(recordSchema);
- try {
- return Class.forName(className, false, specificData.getClassLoader()).asSubclass(SpecificRecordBase.class);
- } catch (ClassNotFoundException e) {
- throw new HoodieException("Failed to load SpecificRecord class " + className + " for Avro schema " + recordSchema.getFullName(), e);
- }
+ return SPECIFIC_RECORD_CLASS_CACHE.computeIfAbsent(recordSchema.getFullName(), fullName -> {
+ String className = SpecificData.getClassName(recordSchema);
+ try {
+ return Class.forName(className, false, specificData.getClassLoader()).asSubclass(SpecificRecordBase.class);
+ } catch (ClassNotFoundException e) {
+ throw new HoodieException("Failed to load SpecificRecord class " + className + " for Avro schema " + fullName, e);
+ }
+ });
}
private static Object convertFieldToSpecificRecordValue(Schema fieldSchema, Object value, SpecificData specificData) {
(hudi) 01/02: fix(common): resolve nested Avro records by class built by Hudi (#19956)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a9b96378ecc8e87bfc700de5d60d8e48ef76f472
Author: voonhous <vo...@gmail.com>
AuthorDate: Wed Sep 16 04:17:25 2026 +0800
fix(common): resolve nested Avro records by class built by Hudi (#19956)
(cherry picked from commit e19a990e3dec6820453e22914c2c5ce4bd545dab)
---
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 18 +++++++++++++++++-
1 file changed, 17 insertions(+), 1 deletion(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
index 4077e1788e90..d97fe8fdcb60 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -1557,11 +1557,27 @@ public class HoodieAvroUtils {
}
}
+ /**
+ * Loads the generated class for a nested RECORD schema with {@code Class.forName}, bypassing the
+ * {@code ClassSecurityValidator} check that Avro 1.12.2+ runs in {@code ClassUtils.forName} (reached from
+ * {@link SpecificData#getClass(Schema)}), which rejects Hudi's generated classes.
+ *
+ * <p>Only pass schemas taken from a compiled SCHEMA$, never a schema read from storage, which is what the validation guards against.
+ */
+ private static Class<? extends SpecificRecordBase> getSpecificRecordClass(Schema recordSchema, SpecificData specificData) {
+ String className = SpecificData.getClassName(recordSchema);
+ try {
+ return Class.forName(className, false, specificData.getClassLoader()).asSubclass(SpecificRecordBase.class);
+ } catch (ClassNotFoundException e) {
+ throw new HoodieException("Failed to load SpecificRecord class " + className + " for Avro schema " + recordSchema.getFullName(), e);
+ }
+ }
+
private static Object convertFieldToSpecificRecordValue(Schema fieldSchema, Object value, SpecificData specificData) {
Schema resolvedFieldSchema = getActualSchemaFromUnion(fieldSchema, value);
switch (resolvedFieldSchema.getType()) {
case RECORD:
- value = convertToSpecificRecord(specificData.getClass(resolvedFieldSchema), (GenericRecord) value, specificData);
+ value = convertToSpecificRecord(getSpecificRecordClass(resolvedFieldSchema, specificData), (GenericRecord) value, specificData);
break;
case ARRAY:
value = ((List<?>) value).stream().map(element -> convertFieldToSpecificRecordValue(resolvedFieldSchema.getElementType(), element, specificData)).collect(Collectors.toList());