You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fury.apache.org by ch...@apache.org on 2025/05/29 03:38:18 UTC
(fury) branch main updated: fix(java): fix generics when write/read null map kv chunk (#2261)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fury.git
The following commit(s) were added to refs/heads/main by this push:
new 3dd49c91 fix(java): fix generics when write/read null map kv chunk (#2261)
3dd49c91 is described below
commit 3dd49c91e743d2b7bf99fd16fa8ef9b70733d17c
Author: Shawn Yang <sh...@gmail.com>
AuthorDate: Thu May 29 11:38:14 2025 +0800
fix(java): fix generics when write/read null map kv chunk (#2261)
## What does this PR do?
1. Fix generics when write/read null map kv chunk
2. Fix some race condition in async jit mode
## Related issues
Closes #2247
#2257
#2253
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an
issue](https://github.com/apache/fury/issues/new/choose) describing the
need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the
PR will have an impact on performance, you can submit the PR first, and
if it will have impact on performance, the code reviewer will explain
it), be sure to attach a benchmark data here.
-->
---
.../fury/builder/BaseObjectCodecBuilder.java | 131 ++++++++++++------
.../java/org/apache/fury/builder/CodecBuilder.java | 2 +-
.../fury/builder/CompatibleCodecBuilder.java | 8 +-
.../fury/builder/MetaSharedCodecBuilder.java | 2 +-
.../apache/fury/builder/ObjectCodecBuilder.java | 9 +-
.../org/apache/fury/resolver/ClassResolver.java | 17 +++
.../collection/AbstractMapSerializer.java | 148 ++++++++++++++++++---
.../java/org/apache/fury/type/GenericType.java | 6 +-
.../serializer/collection/MapSerializersTest.java | 20 +++
.../DifferentPOJOCompatibleSerializerTest.java | 3 +-
...JOCompatibleSerializerWithRegistrationTest.java | 3 +-
11 files changed, 283 insertions(+), 66 deletions(-)
diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
index 7a011cf8..1f75a033 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/BaseObjectCodecBuilder.java
@@ -22,6 +22,7 @@ package org.apache.fury.builder;
import static org.apache.fury.codegen.CodeGenerator.getPackage;
import static org.apache.fury.codegen.Expression.Invoke.inlineInvoke;
import static org.apache.fury.codegen.Expression.Literal.ofInt;
+import static org.apache.fury.codegen.Expression.Literal.ofString;
import static org.apache.fury.codegen.Expression.Reference.fieldRef;
import static org.apache.fury.codegen.ExpressionOptimizer.invokeGenerated;
import static org.apache.fury.codegen.ExpressionUtils.add;
@@ -118,6 +119,7 @@ import org.apache.fury.serializer.StringSerializer;
import org.apache.fury.serializer.collection.AbstractCollectionSerializer;
import org.apache.fury.serializer.collection.AbstractMapSerializer;
import org.apache.fury.serializer.collection.CollectionFlags;
+import org.apache.fury.type.GenericType;
import org.apache.fury.type.TypeUtils;
import org.apache.fury.util.GraalvmSupport;
import org.apache.fury.util.Preconditions;
@@ -141,12 +143,12 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
private static final TypeRef<?> COLLECTION_SERIALIZER_TYPE =
TypeRef.of(AbstractCollectionSerializer.class);
private static final TypeRef<?> MAP_SERIALIZER_TYPE = TypeRef.of(AbstractMapSerializer.class);
+ private static final TypeRef<?> GENERIC_TYPE = TypeRef.of(GenericType.class);
protected final Reference refResolverRef;
protected final Reference classResolverRef =
fieldRef(CLASS_RESOLVER_NAME, CLASS_RESOLVER_TYPE_TOKEN);
protected final Fury fury;
- protected final ClassResolver classResolver;
protected final Reference stringSerializerRef;
private final Map<Class<?>, Reference> serializerMap = new HashMap<>();
private final Map<String, Object> sharedFieldMap = new HashMap<>();
@@ -157,7 +159,6 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
public BaseObjectCodecBuilder(TypeRef<?> beanType, Fury fury, Class<?> parentSerializerClass) {
super(new CodegenContext(), beanType);
this.fury = fury;
- this.classResolver = fury.getClassResolver();
this.parentSerializerClass = parentSerializerClass;
addCommonImports();
ctx.reserveName(REF_RESOLVER_NAME);
@@ -220,12 +221,12 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
protected abstract String codecSuffix();
- protected <T> T visitFury(Function<Fury, T> function) {
+ protected <T> T fury(Function<Fury, T> function) {
return fury.getJITContext().asyncVisitFury(function);
}
private boolean needWriteRef(TypeRef<?> type) {
- return visitFury(fury -> fury.getClassResolver().needToWriteRef(type));
+ return fury(fury -> fury.getClassResolver().needToWriteRef(type));
}
@Override
@@ -483,19 +484,19 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
}
protected boolean useCollectionSerialization(TypeRef<?> typeRef) {
- return visitFury(f -> f.getClassResolver().isCollection(TypeUtils.getRawType(typeRef)));
+ return fury(f -> f.getClassResolver().isCollection(TypeUtils.getRawType(typeRef)));
}
protected boolean useCollectionSerialization(Class<?> type) {
- return visitFury(f -> f.getClassResolver().isCollection(TypeUtils.getRawType(type)));
+ return fury(f -> f.getClassResolver().isCollection(TypeUtils.getRawType(type)));
}
protected boolean useMapSerialization(TypeRef<?> typeRef) {
- return visitFury(f -> f.getClassResolver().isMap(TypeUtils.getRawType(typeRef)));
+ return fury(f -> f.getClassResolver().isMap(TypeUtils.getRawType(typeRef)));
}
protected boolean useMapSerialization(Class<?> type) {
- return visitFury(f -> f.getClassResolver().isMap(TypeUtils.getRawType(type)));
+ return fury(f -> f.getClassResolver().isMap(TypeUtils.getRawType(type)));
}
/**
@@ -539,7 +540,8 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
classInfo,
inlineInvoke(classResolverRef, "getClassInfo", classInfoTypeRef, clsExpr))));
}
- writeClassAndObject.add(classResolver.writeClassExpr(classResolverRef, buffer, classInfo));
+ writeClassAndObject.add(
+ fury(f -> f.getClassResolver().writeClassExpr(classResolverRef, buffer, classInfo)));
writeClassAndObject.add(
new Invoke(
invokeInline(classInfo, "getSerializer", getSerializerType(clz)),
@@ -562,7 +564,8 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
new Assign(
classInfo,
inlineInvoke(classResolverRef, "getClassInfo", classInfoTypeRef, clsExpr))));
- writeClassAction.add(classResolver.writeClassExpr(classResolverRef, buffer, classInfo));
+ writeClassAction.add(
+ fury(f -> f.getClassResolver().writeClassExpr(classResolverRef, buffer, classInfo)));
if (returnSerializer) {
writeClassAction.add(
invoke(classInfo, "getSerializer", "serializer", getSerializerType(declaredClass)));
@@ -582,10 +585,10 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
Class<? extends Serializer> serializerClass;
if (fury.isCrossLanguage()) {
// xlang will take all map/collection interface as monomorphic
- serializerClass = visitFury(f -> f.getXtypeResolver().getSerializer(cls)).getClass();
+ serializerClass = fury(f -> f.getXtypeResolver().getSerializer(cls)).getClass();
} else {
// potential recursive call for seq codec generation is handled in `getSerializerClass`.
- serializerClass = visitFury(f -> f.getClassResolver().getSerializerClass(cls));
+ serializerClass = fury(f -> f.getClassResolver().getSerializerClass(cls));
}
Preconditions.checkNotNull(serializerClass, "Unsupported for class " + cls);
if (!ReflectionUtils.isPublic(serializerClass)) {
@@ -655,6 +658,31 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
}
}
+ private final Map<TypeRef<?>, String> namesForSharedGenericTypeFields = new HashMap<>();
+
+ protected Expression getGenericTypeField(TypeRef<?> typeRef) {
+ // create a field name from generic type, so multiple call of same generic type will reuse the
+ // same field.
+ String name =
+ namesForSharedGenericTypeFields.computeIfAbsent(
+ typeRef,
+ k ->
+ StringUtils.uncapitalize(typeRef.getRawType().getSimpleName())
+ + namesForSharedGenericTypeFields.size());
+
+ return getOrCreateField(
+ false,
+ GenericType.class,
+ name,
+ () ->
+ new Invoke(
+ classResolverRef,
+ "getGenericTypeInStruct",
+ GENERIC_TYPE,
+ beanClassExpr(),
+ ofString(typeRef.getType().getTypeName())));
+ }
+
/**
* The boolean value in tuple indicates whether the classinfo needs update.
*
@@ -743,9 +771,9 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
}
protected TypeRef<?> getSerializerType(Class<?> objType) {
- if (classResolver.isCollection(objType)) {
+ if (fury(f -> f.getClassResolver().isCollection(objType))) {
return COLLECTION_SERIALIZER_TYPE;
- } else if (classResolver.isMap(objType)) {
+ } else if (fury(f -> f.getClassResolver().isMap(objType))) {
return MAP_SERIALIZER_TYPE;
}
return SERIALIZER_TYPE;
@@ -781,7 +809,8 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
new Assign(
classInfo,
inlineInvoke(classResolverRef, "getClassInfo", classInfoTypeRef, clsExpr))));
- writeClassAction.add(classResolver.writeClassExpr(classResolverRef, buffer, classInfo));
+ writeClassAction.add(
+ fury(f -> f.getClassResolver().writeClassExpr(classResolverRef, buffer, classInfo)));
writeClassAction.add(
new Return(invokeInline(classInfo, "getSerializer", getSerializerType(typeRef))));
// Spit this into a separate method to avoid method too big to inline.
@@ -853,7 +882,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
Literal notDeclTypeFlag = ofInt(CollectionFlags.NOT_DECL_ELEMENT_TYPE);
Expression isDeclType = neq(new BitAnd(flags, notDeclTypeFlag), notDeclTypeFlag);
Expression elemSerializer; // make it in scope of `if(sameElementClass)`
- boolean maybeDecl = visitFury(f -> f.getClassResolver().isSerializable(elemClass));
+ boolean maybeDecl = fury(f -> f.getClassResolver().isSerializable(elemClass));
TypeRef<?> serializerType = getSerializerType(elementType);
if (maybeDecl) {
elemSerializer =
@@ -1085,7 +1114,8 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
classInfo,
inlineInvoke(classResolverRef, "getClassInfo", classInfoTypeRef, clsExpr))));
// Note: writeClassExpr is thread safe.
- writeClassAction.add(classResolver.writeClassExpr(classResolverRef, buffer, classInfo));
+ writeClassAction.add(
+ fury(f -> f.getClassResolver().writeClassExpr(classResolverRef, buffer, classInfo)));
writeClassAction.add(
new Return(invokeInline(classInfo, "getSerializer", MAP_SERIALIZER_TYPE)));
// Spit this into a separate method to avoid method too big to inline.
@@ -1121,8 +1151,8 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
boolean inline = keyMonomorphic && valueMonomorphic;
Class<?> keyTypeRawType = keyType.getRawType();
Class<?> valueTypeRawType = valueType.getRawType();
- boolean trackingKeyRef = visitFury(fury -> fury.getClassResolver().needToWriteRef(keyType));
- boolean trackingValueRef = visitFury(fury -> fury.getClassResolver().needToWriteRef(valueType));
+ boolean trackingKeyRef = fury(fury -> fury.getClassResolver().needToWriteRef(keyType));
+ boolean trackingValueRef = fury(fury -> fury.getClassResolver().needToWriteRef(valueType));
Tuple2<Expression, Expression> mapKVSerializer =
getMapKVSerializer(keyTypeRawType, valueTypeRawType);
Expression keySerializer = mapKVSerializer.f0;
@@ -1131,25 +1161,50 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
new While(
neqNull(entry),
() -> {
- String method = "writeJavaNullChunk";
- if (keyMonomorphic && valueMonomorphic) {
+ boolean hasGenerics =
+ keyTypeRawType != Object.class || valueTypeRawType != Object.class;
+ String method = hasGenerics ? "writeJavaNullChunkGeneric" : "writeJavaNullChunk";
+ GenericType keyGenericType =
+ fury(f -> f.getClassResolver().buildGenericType(keyType));
+ GenericType valueGenericType =
+ fury(f -> f.getClassResolver().buildGenericType(valueType));
+ if (keyGenericType.hasGenericParameters()
+ || valueGenericType.hasGenericParameters()) {
+ method = "writeJavaNullChunkGeneric";
+ } else if (keyMonomorphic && valueMonomorphic) {
if (!trackingKeyRef && !trackingValueRef) {
method = "writeNullChunkKVFinalNoRef";
+ } else {
+ method = "writeJavaNullChunk";
}
}
+ Expression writeNullChunk;
+ if (method.equals("writeJavaNullChunkGeneric")) {
+ writeNullChunk =
+ inlineInvoke(
+ serializer,
+ method,
+ MAP_ENTRY_TYPE,
+ buffer,
+ entry,
+ iterator,
+ getGenericTypeField(keyType),
+ getGenericTypeField(valueType));
+ } else {
+ writeNullChunk =
+ inlineInvoke(
+ serializer,
+ method,
+ MAP_ENTRY_TYPE,
+ buffer,
+ entry,
+ iterator,
+ keySerializer,
+ valueSerializer);
+ }
Expression writeChunk = writeChunk(buffer, entry, iterator, keyType, valueType);
return new ListExpression(
- new Assign(
- entry,
- inlineInvoke(
- serializer,
- method,
- MAP_ENTRY_TYPE,
- buffer,
- entry,
- iterator,
- keySerializer,
- valueSerializer)),
+ new Assign(entry, writeNullChunk),
new If(
neqNull(entry), inline ? writeChunk : new Assign(entry, inline(writeChunk))));
});
@@ -1214,8 +1269,8 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
Expression chunkHeader;
Expression keySerializer, valueSerializer;
- boolean trackingKeyRef = visitFury(fury -> fury.getClassResolver().needToWriteRef(keyType));
- boolean trackingValueRef = visitFury(fury -> fury.getClassResolver().needToWriteRef(valueType));
+ boolean trackingKeyRef = fury(fury -> fury.getClassResolver().needToWriteRef(keyType));
+ boolean trackingValueRef = fury(fury -> fury.getClassResolver().needToWriteRef(valueType));
Expression keyWriteRef = Literal.ofBoolean(trackingKeyRef);
Expression valueWriteRef = Literal.ofBoolean(trackingValueRef);
boolean inline = keyMonomorphic && valueMonomorphic;
@@ -1420,7 +1475,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
TypeRef<?> typeRef,
Function<Expression, Expression> callback,
InvokeHint invokeHint) {
- if (visitFury(f -> f.getClassResolver().needToWriteRef(typeRef))) {
+ if (fury(f -> f.getClassResolver().needToWriteRef(typeRef))) {
return readRef(buffer, callback, () -> deserializeForNotNull(buffer, typeRef, invokeHint));
} else {
if (typeRef.isPrimitive()) {
@@ -1438,7 +1493,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
TypeRef<?> typeRef,
Function<Expression, Expression> callback,
boolean nullable) {
- if (visitFury(f -> f.getClassResolver().needToWriteRef(typeRef))) {
+ if (fury(f -> f.getClassResolver().needToWriteRef(typeRef))) {
return readRef(buffer, callback, () -> deserializeForNotNull(buffer, typeRef, null));
} else {
if (typeRef.isPrimitive()) {
@@ -1631,7 +1686,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
Class<?> elemClass = TypeUtils.getRawType(elementType);
walkPath.add(elementType.toString());
boolean finalType = isMonomorphic(elemClass);
- boolean trackingRef = visitFury(fury -> fury.getClassResolver().needToWriteRef(elementType));
+ boolean trackingRef = fury(fury -> fury.getClassResolver().needToWriteRef(elementType));
if (finalType) {
if (trackingRef) {
builder.add(readContainerElements(elementType, true, null, null, buffer, collection, size));
@@ -1653,7 +1708,7 @@ public abstract class BaseObjectCodecBuilder extends CodecBuilder {
inlineInvoke(readClassInfo(elemClass, buffer), "getSerializer", SERIALIZER_TYPE);
TypeRef<?> serializerType = getSerializerType(elementType);
Expression elemSerializer; // make it in scope of `if(sameElementClass)`
- boolean maybeDecl = visitFury(f -> f.getClassResolver().isSerializable(elemClass));
+ boolean maybeDecl = fury(f -> f.getClassResolver().isSerializable(elemClass));
if (maybeDecl) {
elemSerializer =
new If(
diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
index bbda1ff0..4002865c 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/CodecBuilder.java
@@ -449,7 +449,7 @@ public abstract class CodecBuilder {
});
}
- private Reference getOrCreateField(
+ protected Reference getOrCreateField(
boolean isStatic, Class<?> type, String fieldName, Supplier<Expression> value) {
Reference fieldRef = fieldMap.get(fieldName);
if (fieldRef == null) {
diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
index 33b8bcc0..5bd35579 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/CompatibleCodecBuilder.java
@@ -878,9 +878,9 @@ public class CompatibleCodecBuilder extends BaseObjectCodecBuilder {
protected Expression writeFinalClassInfo(Expression buffer, Class<?> cls) {
Preconditions.checkArgument(ReflectionUtils.isMonomorphic(cls));
- ClassInfo classInfo = visitFury(f -> f.getClassResolver().getClassInfo(cls, false));
+ ClassInfo classInfo = fury(f -> f.getClassResolver().getClassInfo(cls, false));
if (classInfo != null && classInfo.getClassId() != ClassResolver.NO_CLASS_ID) {
- return classResolver.writeClassExpr(buffer, classInfo.getClassId());
+ return fury(f -> f.getClassResolver().writeClassExpr(buffer, classInfo.getClassId()));
}
Expression classInfoExpr = getFinalClassInfo(cls);
return new Invoke(classResolverRef, "writeClassInfo", buffer, classInfoExpr);
@@ -888,9 +888,9 @@ public class CompatibleCodecBuilder extends BaseObjectCodecBuilder {
protected Expression skipFinalClassInfo(Class<?> cls, Expression buffer) {
Preconditions.checkArgument(ReflectionUtils.isMonomorphic(cls));
- ClassInfo classInfo = visitFury(f -> f.getClassResolver().getClassInfo(cls, false));
+ ClassInfo classInfo = fury(f -> f.getClassResolver().getClassInfo(cls, false));
if (classInfo != null && classInfo.getClassId() != ClassResolver.NO_CLASS_ID) {
- return classResolver.skipRegisteredClassExpr(buffer);
+ return fury(f -> f.getClassResolver().skipRegisteredClassExpr(buffer));
}
// read `ClassInfo` is not used, set `inlineReadClassInfo` false,
// to avoid read doesn't happen in generated code.
diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
index 91a6413e..424cff77 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/MetaSharedCodecBuilder.java
@@ -75,7 +75,7 @@ public class MetaSharedCodecBuilder extends ObjectCodecBuilder {
"Class version check should be disabled when compatible mode is enabled.");
this.classDef = classDef;
Collection<Descriptor> descriptors =
- visitFury(
+ fury(
f -> MetaSharedSerializer.consolidateFields(f.getClassResolver(), beanClass, classDef));
DescriptorGrouper grouper = fury.getClassResolver().createDescriptorGrouper(descriptors, false);
objectCodecOptimizer =
diff --git a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
index eb0a837f..54672e85 100644
--- a/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
+++ b/java/fury-core/src/main/java/org/apache/fury/builder/ObjectCodecBuilder.java
@@ -90,15 +90,16 @@ public class ObjectCodecBuilder extends BaseObjectCodecBuilder {
boolean shareMeta = fury.getConfig().isMetaShareEnabled();
if (shareMeta) {
descriptors =
- visitFury(
+ fury(
f ->
f.getClassResolver()
.getClassDef(beanClass, true)
- .getDescriptors(classResolver, beanClass));
+ .getDescriptors(f.getClassResolver(), beanClass));
} else {
descriptors = fury.getClassResolver().getFieldDescriptors(beanClass, true);
}
- DescriptorGrouper grouper = classResolver.createDescriptorGrouper(descriptors, false);
+ Collection<Descriptor> p = descriptors;
+ DescriptorGrouper grouper = fury(f -> f.getClassResolver().createDescriptorGrouper(p, false));
descriptors = grouper.getSortedDescriptors();
classVersionHash =
new Literal(ObjectSerializer.computeStructHash(fury, descriptors), PRIMITIVE_INT_TYPE);
@@ -137,7 +138,7 @@ public class ObjectCodecBuilder extends BaseObjectCodecBuilder {
/** Mark non-inner registered final types as non-final to write class def for those types. */
@Override
protected boolean isMonomorphic(Class<?> clz) {
- return visitFury(f -> f.getClassResolver().isMonomorphic(clz));
+ return fury(f -> f.getClassResolver().isMonomorphic(clz));
}
/**
diff --git a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
index f4661179..63902c2d 100644
--- a/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
+++ b/java/fury-core/src/main/java/org/apache/fury/resolver/ClassResolver.java
@@ -268,6 +268,7 @@ public class ClassResolver implements TypeResolver {
private ClassChecker classChecker = (classResolver, className) -> true;
private GenericType objectGenericType;
private final IdentityMap<Type, GenericType> genericTypes = new IdentityMap<>();
+ private final Map<Class, Map<String, GenericType>> classGenericTypes = new HashMap<>();
private final Map<List<ClassLoader>, CodeGenerator> codeGeneratorMap = new HashMap<>();
}
@@ -2004,6 +2005,22 @@ public class ClassResolver implements TypeResolver {
public void resetWrite() {}
+ @CodegenInvoke
+ public GenericType getGenericTypeInStruct(Class<?> cls, String genericTypeStr) {
+ Map<String, GenericType> map =
+ extRegistry.classGenericTypes.computeIfAbsent(cls, k -> new HashMap<>());
+ GenericType genericType = map.get(genericTypeStr);
+ if (genericType == null) {
+ for (Field field : ReflectionUtils.getFields(cls, true)) {
+ Type type = field.getGenericType();
+ TypeRef<Object> typeRef = TypeRef.of(type);
+ genericType = buildGenericType(typeRef);
+ map.put(type.getTypeName(), genericType);
+ }
+ }
+ return genericType;
+ }
+
@Override
public GenericType buildGenericType(TypeRef<?> typeRef) {
return GenericType.build(
diff --git a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
index f392fe3a..5fa74580 100644
--- a/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
+++ b/java/fury-core/src/main/java/org/apache/fury/serializer/collection/AbstractMapSerializer.java
@@ -138,18 +138,30 @@ public abstract class AbstractMapSerializer<T> extends Serializer<T> {
Iterator<Entry<Object, Object>> iterator = map.entrySet().iterator();
Entry<Object, Object> entry = iterator.next();
while (entry != null) {
- entry = writeJavaNullChunk(buffer, entry, iterator, keySerializer, valueSerializer);
- if (entry != null) {
- if (keySerializer != null || valueSerializer != null) {
+ if (keySerializer != null || valueSerializer != null) {
+ entry = writeJavaNullChunk(buffer, entry, iterator, keySerializer, valueSerializer);
+ if (entry != null) {
entry =
writeJavaChunk(
classResolver, buffer, entry, iterator, keySerializer, valueSerializer);
- } else {
- Generics generics = fury.getGenerics();
- GenericType genericType = generics.nextGenericType();
- if (genericType == null) {
+ }
+ } else {
+ Generics generics = fury.getGenerics();
+ GenericType genericType = generics.nextGenericType();
+ if (genericType == null) {
+ entry = writeJavaNullChunk(buffer, entry, iterator, null, null);
+ if (entry != null) {
entry = writeJavaChunk(classResolver, buffer, entry, iterator, null, null);
- } else {
+ }
+ } else {
+ if (genericType.getTypeParametersCount() < 2) {
+ genericType = getKVGenericType(genericType);
+ }
+ GenericType keyGenericType = genericType.getTypeParameter0();
+ GenericType valueGenericType = genericType.getTypeParameter1();
+ entry =
+ writeJavaNullChunkGeneric(buffer, entry, iterator, keyGenericType, valueGenericType);
+ if (entry != null) {
entry =
writeJavaChunkGeneric(
classResolver, generics, genericType, buffer, entry, iterator);
@@ -257,6 +269,81 @@ public abstract class AbstractMapSerializer<T> extends Serializer<T> {
}
}
+ public final Entry writeJavaNullChunkGeneric(
+ MemoryBuffer buffer,
+ Entry entry,
+ Iterator<Entry<Object, Object>> iterator,
+ GenericType keyType,
+ GenericType valueType) {
+ while (true) {
+ Object key = entry.getKey();
+ Object value = entry.getValue();
+ if (key != null) {
+ if (value != null) {
+ return entry;
+ }
+ writeKeyForNullValueChunkGeneric(buffer, key, keyType);
+ } else {
+ writeValueForNullKeyChunkGeneric(buffer, value, valueType);
+ }
+ if (iterator.hasNext()) {
+ entry = iterator.next();
+ } else {
+ return null;
+ }
+ }
+ }
+
+ private void writeKeyForNullValueChunkGeneric(
+ MemoryBuffer buffer, Object key, GenericType keyType) {
+ if (!keyType.isMonomorphic()) {
+ buffer.writeByte(VALUE_HAS_NULL | TRACKING_KEY_REF);
+ binding.writeRef(buffer, key, keyClassInfoWriteCache);
+ return;
+ }
+ Serializer serializer = keyType.getSerializer(typeResolver);
+ if (keyType.hasGenericParameters()) {
+ fury.getGenerics().pushGenericType(keyType);
+ fury.incDepth(1);
+ }
+ if (serializer.needToWriteRef()) {
+ buffer.writeByte(NULL_VALUE_KEY_DECL_TYPE_TRACKING_REF);
+ binding.writeRef(buffer, key, serializer);
+ } else {
+ buffer.writeByte(NULL_VALUE_KEY_DECL_TYPE);
+ binding.write(buffer, serializer, key);
+ }
+ if (keyType.hasGenericParameters()) {
+ fury.incDepth(-1);
+ fury.getGenerics().popGenericType();
+ }
+ }
+
+ private void writeValueForNullKeyChunkGeneric(
+ MemoryBuffer buffer, Object value, GenericType valueType) {
+ if (!valueType.isMonomorphic()) {
+ buffer.writeByte(KEY_HAS_NULL | TRACKING_VALUE_REF);
+ binding.writeRef(buffer, value, valueClassInfoWriteCache);
+ return;
+ }
+ Serializer serializer = valueType.getSerializer(typeResolver);
+ if (valueType.hasGenericParameters()) {
+ fury.getGenerics().pushGenericType(valueType);
+ fury.incDepth(1);
+ }
+ if (serializer.needToWriteRef()) {
+ buffer.writeByte(NULL_KEY_VALUE_DECL_TYPE_TRACKING_REF);
+ binding.writeRef(buffer, value, serializer);
+ } else {
+ buffer.writeByte(NULL_KEY_VALUE_DECL_TYPE);
+ binding.write(buffer, serializer, value);
+ }
+ if (valueType.hasGenericParameters()) {
+ fury.incDepth(-1);
+ fury.getGenerics().popGenericType();
+ }
+ }
+
// Make byte code of this method smaller than 325 for better jit inline
private Entry writeJavaChunk(
TypeResolver classResolver,
@@ -341,7 +428,8 @@ public abstract class AbstractMapSerializer<T> extends Serializer<T> {
return classInfo.getSerializer();
}
- private Entry writeJavaChunkGeneric(
+ @CodegenInvoke
+ public Entry writeJavaChunkGeneric(
TypeResolver classResolver,
Generics generics,
GenericType genericType,
@@ -599,10 +687,14 @@ public abstract class AbstractMapSerializer<T> extends Serializer<T> {
boolean trackKeyRef = (chunkHeader & TRACKING_KEY_REF) != 0;
Object key;
if ((chunkHeader & KEY_DECL_TYPE) != 0) {
- if (trackKeyRef) {
- key = binding.readRef(buffer, keySerializer);
+ if (keySerializer == null) {
+ key = readNonEmptyValueFromNullChunk(buffer, trackKeyRef, true);
} else {
- key = binding.read(buffer, keySerializer);
+ if (trackKeyRef) {
+ key = binding.readRef(buffer, keySerializer);
+ } else {
+ key = binding.read(buffer, keySerializer);
+ }
}
} else {
key = binding.readRef(buffer, keyClassInfoReadCache);
@@ -635,10 +727,14 @@ public abstract class AbstractMapSerializer<T> extends Serializer<T> {
Object value;
boolean trackValueRef = (chunkHeader & TRACKING_VALUE_REF) != 0;
if ((chunkHeader & VALUE_DECL_TYPE) != 0) {
- if (trackValueRef) {
- value = binding.readRef(buffer, valueSerializer);
+ if (valueSerializer == null) {
+ value = readNonEmptyValueFromNullChunk(buffer, trackValueRef, false);
} else {
- value = binding.read(buffer, valueSerializer);
+ if (trackValueRef) {
+ value = binding.readRef(buffer, valueSerializer);
+ } else {
+ value = binding.read(buffer, valueSerializer);
+ }
}
} else {
value = binding.readRef(buffer, valueClassInfoReadCache);
@@ -649,6 +745,28 @@ public abstract class AbstractMapSerializer<T> extends Serializer<T> {
}
}
+ private Object readNonEmptyValueFromNullChunk(
+ MemoryBuffer buffer, boolean trackRef, boolean isKey) {
+ Generics generics = fury.getGenerics();
+ GenericType genericType = generics.nextGenericType();
+ if (genericType.getTypeParametersCount() < 2) {
+ genericType = getKVGenericType(genericType);
+ }
+ GenericType type = isKey ? genericType.getTypeParameter0() : genericType.getTypeParameter1();
+ generics.pushGenericType(type);
+ fury.incDepth(1);
+ Serializer<?> serializer = type.getSerializer(typeResolver);
+ Object v;
+ if (trackRef) {
+ v = binding.readRef(buffer, serializer);
+ } else {
+ v = binding.read(buffer, serializer);
+ }
+ fury.incDepth(-1);
+ generics.popGenericType();
+ return v;
+ }
+
@CodegenInvoke
public long readNullChunkKVFinalNoRef(
MemoryBuffer buffer,
diff --git a/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java b/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
index 8a6bb6f6..eb045109 100644
--- a/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
+++ b/java/fury-core/src/main/java/org/apache/fury/type/GenericType.java
@@ -34,7 +34,11 @@ import org.apache.fury.reflect.TypeRef;
import org.apache.fury.resolver.TypeResolver;
import org.apache.fury.serializer.Serializer;
-/** GenericType for building java generics as a tree and binding with fury serializers. */
+/**
+ * GenericType for building java generics as a tree and binding with fury serializers. Note
+ * GenericType for specific types such as Object.class can't be singleton, because GenericType has
+ * some mutable fields
+ */
// TODO(chaokunyang) refine generics which can be inspired by spring ResolvableType.
@SuppressWarnings("rawtypes")
public class GenericType {
diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java
index 2e38f728..ff4e4704 100644
--- a/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/serializer/collection/MapSerializersTest.java
@@ -1008,4 +1008,24 @@ public class MapSerializersTest extends FuryTestBase {
serDeCheck(fury, pojo);
fury.serialize(pojo);
}
+
+ @Data
+ @AllArgsConstructor
+ public static class NullChunkGeneric {
+ public Map<String, Integer> map;
+ public Map<String, List<Integer>> map1;
+ }
+
+ @Test
+ public void testNullChunkGeneric() {
+ Fury fury1 = builder().withCodegen(true).build();
+ Map<String, Integer> map = ofHashMap(null, 1, "k1", null, "k2", 2);
+ Map<String, List<Integer>> map1 =
+ ofHashMap(null, ofArrayList(1), "k1", null, "k2", ofArrayList(2));
+ NullChunkGeneric o = new NullChunkGeneric(map, map1);
+ byte[] bytes = fury1.serialize(o);
+ Fury fury2 = builder().withCodegen(false).build();
+ Object object = fury2.deserialize(bytes);
+ assertEquals(object, o);
+ }
}
diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerTest.java
index 59dd15d4..463113f3 100644
--- a/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerTest.java
@@ -19,6 +19,7 @@
package org.apache.fury.serializer.compatible;
+import java.util.Arrays;
import org.apache.fury.Fury;
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.config.Language;
@@ -44,13 +45,13 @@ public class DifferentPOJOCompatibleSerializerTest extends Assert {
.requireClassRegistration(false)
.withAsyncCompilation(true)
.serializeEnumByName(true)
+ .withName(getClass() + Arrays.toString(classes))
.build();
if (classes != null) {
for (Class<?> clazz : classes) {
instance.register(clazz);
}
}
- ;
return instance;
}
diff --git a/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerWithRegistrationTest.java b/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerWithRegistrationTest.java
index c5c8d15e..aacd6d0e 100644
--- a/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerWithRegistrationTest.java
+++ b/java/fury-core/src/test/java/org/apache/fury/serializer/compatible/DifferentPOJOCompatibleSerializerWithRegistrationTest.java
@@ -19,6 +19,7 @@
package org.apache.fury.serializer.compatible;
+import java.util.Arrays;
import org.apache.fury.Fury;
import org.apache.fury.config.CompatibleMode;
import org.apache.fury.config.Language;
@@ -44,13 +45,13 @@ public class DifferentPOJOCompatibleSerializerWithRegistrationTest extends Asser
.requireClassRegistration(false)
.withAsyncCompilation(true)
.serializeEnumByName(true)
+ .withName(getClass() + Arrays.toString(classes))
.build();
if (classes != null) {
for (Class<?> clazz : classes) {
instance.register(clazz);
}
}
- ;
return instance;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org