You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fury.apache.org by "chaokunyang (via GitHub)" <gi...@apache.org> on 2025/01/26 15:24:34 UTC
[PR] feat(java): jit support for chunk based map serialization [fury]
chaokunyang opened a new pull request, #2027:
URL: https://github.com/apache/fury/pull/2027
## What does this PR do?
<!-- Describe the purpose of this PR. -->
## Related issues
<!--
Is there any related issue? Please attach here.
- #xxxx0
- #xxxx1
- #xxxx2
-->
## Does this PR introduce any user-facing change?
<!--
If any user-facing interface changes, please [open an issue](https://github.com/apache/fury/issues/new/choose) describing the need to do so and update the document if necessary.
-->
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
## Benchmark
<!--
When the PR has an impact on performance (if you don't know whether the PR will have an impact on performance, you can submit the PR first, and if it will have impact on performance, the code reviewer will explain it), be sure to attach a benchmark data here.
-->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org
Re: [PR] feat(java): jit support for chunk based map serialization [fury]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on PR #2027:
URL: https://github.com/apache/fury/pull/2027#issuecomment-2618222664
For following struct with map field:
```java
public class SimpleMapFieldStruct {
public Map<String, String> map;
}
```
This PR will generate code like:
```java
package org.apache.fury.serializer.collection;
import org.apache.fury.Fury;
import org.apache.fury.builder.Generated.GeneratedObjectSerializer;
import org.apache.fury.memory.MemoryBuffer;
import org.apache.fury.resolver.ClassInfo;
import org.apache.fury.resolver.ClassInfoHolder;
import org.apache.fury.resolver.ClassResolver;
import org.apache.fury.resolver.NoRefResolver;
import org.apache.fury.serializer.Serializer;
import org.apache.fury.serializer.StringSerializer;
public final class MapSerializersTest_SimpleMapFieldStructFuryCodec_0 extends GeneratedObjectSerializer {
private final NoRefResolver refResolver;
private final ClassResolver classResolver;
private final StringSerializer strSerializer;
private Fury fury;
private ClassInfo mapClassInfo;
private final StringSerializer stringSerializer;
private final ClassInfoHolder map3ClassInfoHolder;
public MapSerializersTest_SimpleMapFieldStructFuryCodec_0(Fury fury, Class classType) {
super(fury, classType);
this.fury = fury;
fury.getClassResolver().setSerializerIfAbsent(classType, this);
org.apache.fury.resolver.RefResolver refResolver0 = fury.getRefResolver();
refResolver = ((NoRefResolver)refResolver0);
classResolver = fury.getClassResolver();
strSerializer = fury.getStringSerializer();
mapClassInfo = classResolver.nilClassInfo();
stringSerializer = ((StringSerializer)classResolver.getRawSerializer(java.lang.String.class));
map3ClassInfoHolder = classResolver.nilClassInfoHolder();
}
private AbstractMapSerializer writeMapClassInfo(java.util.Map map1, MemoryBuffer memoryBuffer) {
ClassResolver classResolver = this.classResolver;
Class value = mapClassInfo.getCls();
Class cls = map1.getClass();
if ((value != cls)) {
mapClassInfo = classResolver.getClassInfo(cls);
}
classResolver.writeClass(memoryBuffer, mapClassInfo);
Serializer serializer = mapClassInfo.getSerializer();
return ((AbstractMapSerializer)serializer);
}
private void writeFields(org.apache.fury.serializer.collection.MapSerializersTest.SimpleMapFieldStruct simpleMapFieldStruct1, MemoryBuffer memoryBuffer1) {
StringSerializer stringSerializer = this.stringSerializer;
java.util.Map map0 = simpleMapFieldStruct1.map;
if ((map0 == null)) {
memoryBuffer1.writeByte(((byte)-3));
} else {
memoryBuffer1.writeByte(((byte)0));
AbstractMapSerializer abstractMapSerializer = this.writeMapClassInfo(map0, memoryBuffer1);
if (abstractMapSerializer.supportCodegenHook()) {
java.util.Map map2 = abstractMapSerializer.onMapWrite(memoryBuffer1, map0);
if ((!map2.isEmpty())) {
java.util.Iterator iterator = map2.entrySet().iterator();
java.util.Map.Entry entry = (java.util.Map.Entry)iterator.next();
while ((entry != null)) {
entry = abstractMapSerializer.writeNullChunkKVNoRef(memoryBuffer1, entry, iterator, stringSerializer, stringSerializer);
if ((entry != null)) {
String key = (String)entry.getKey();
String value0 = (String)entry.getValue();
memoryBuffer1.writeInt16(((short)-1));
int chunkSizeOffset = memoryBuffer1.writerIndex() - 1;
memoryBuffer1.putByte((chunkSizeOffset - 1), 36);
int chunkSize = 0;
while (true) {
if (((key == null) || (value0 == null))) {
break;
}
stringSerializer.write(memoryBuffer1, key);
stringSerializer.write(memoryBuffer1, value0);
chunkSize = (chunkSize + 1);
if ((chunkSize == 255)) {
break;
}
if (iterator.hasNext()) {
entry = ((java.util.Map.Entry)iterator.next());
key = ((String)entry.getKey());
value0 = ((String)entry.getValue());
} else {
entry = null;
break;
}
}
memoryBuffer1.putByte(chunkSizeOffset, chunkSize);
}
}
}
} else {
abstractMapSerializer.write(memoryBuffer1, map0);
}
}
}
private void readFields(MemoryBuffer memoryBuffer2, org.apache.fury.serializer.collection.MapSerializersTest.SimpleMapFieldStruct simpleMapFieldStruct2) {
StringSerializer strSerializer = this.strSerializer;
StringSerializer stringSerializer = this.stringSerializer;
if ((memoryBuffer2.readByte() != ((byte)-3))) {
Serializer serializer0 = classResolver.readClassInfo(memoryBuffer2, map3ClassInfoHolder).getSerializer();
AbstractMapSerializer mapSerializer = (AbstractMapSerializer)serializer0;
Object object1;
if (mapSerializer.supportCodegenHook()) {
java.util.Map map4 = mapSerializer.newMap(memoryBuffer2);
int size = mapSerializer.getAndClearNumElements();
if ((size == 0)) {
return;
}
int chunkHeader = memoryBuffer2.readUnsignedByte();
while ((size > 0)) {
long sizeAndHeader = mapSerializer.readJavaNullChunk(memoryBuffer2, map4, chunkHeader, size, stringSerializer, stringSerializer);
chunkHeader = ((int)(sizeAndHeader & 255));
size = ((int)(sizeAndHeader >>> 8));
int chunkSize0 = memoryBuffer2.readUnsignedByte();
for (int i = 0; i < chunkSize0; i+=1) {
String string = strSerializer.readCharsString(memoryBuffer2);
String string1 = strSerializer.readCharsString(memoryBuffer2);
map4.put(string, string1);
size = (size - 1);
}
if ((size > 0)) {
chunkHeader = memoryBuffer2.readUnsignedByte();
}
}
object1 = mapSerializer.onMapRead(map4);
} else {
Object object = mapSerializer.read(memoryBuffer2);
object1 = object;
}
simpleMapFieldStruct2.map = ((java.util.Map)object1);
} else {
simpleMapFieldStruct2.map = null;
}
}
@Override public final void write(MemoryBuffer buffer, Object obj) {
org.apache.fury.serializer.collection.MapSerializersTest.SimpleMapFieldStruct simpleMapFieldStruct3 = (org.apache.fury.serializer.collection.MapSerializersTest.SimpleMapFieldStruct)obj;
this.writeFields(simpleMapFieldStruct3, buffer);
}
@Override public final Object read(MemoryBuffer buffer) {
org.apache.fury.serializer.collection.MapSerializersTest.SimpleMapFieldStruct simpleMapFieldStruct4 = new org.apache.fury.serializer.collection.MapSerializersTest.SimpleMapFieldStruct();
refResolver.reference(simpleMapFieldStruct4);
this.readFields(buffer, simpleMapFieldStruct4);
return simpleMapFieldStruct4;
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org
Re: [PR] feat(java): jit support for chunk based map serialization [fury]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2027:
URL: https://github.com/apache/fury/pull/2027#discussion_r1932520224
##########
java/benchmark/plot_map_benchmark.py:
##########
@@ -0,0 +1,160 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Load the CSV data
+no_chunk_file = (
+ "/Users/chaokunyang/Desktop/chaokun/fury_open_source/nochunk-jmh-result.csv"
Review Comment:
Good catch, fixed now
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org
Re: [PR] feat(java): jit support for chunk based map serialization [fury]
Posted by "pandalee99 (via GitHub)" <gi...@apache.org>.
pandalee99 commented on code in PR #2027:
URL: https://github.com/apache/fury/pull/2027#discussion_r1932510037
##########
java/benchmark/plot_map_benchmark.py:
##########
@@ -0,0 +1,160 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Load the CSV data
+no_chunk_file = (
+ "/Users/chaokunyang/Desktop/chaokun/fury_open_source/nochunk-jmh-result.csv"
Review Comment:
The address doesn't seem very suitable
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org
Re: [PR] feat(java): jit support for chunk based map serialization [fury]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang merged PR #2027:
URL: https://github.com/apache/fury/pull/2027
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org
Re: [PR] feat(java): jit support for chunk based map serialization [fury]
Posted by "pandalee99 (via GitHub)" <gi...@apache.org>.
pandalee99 commented on code in PR #2027:
URL: https://github.com/apache/fury/pull/2027#discussion_r1932510037
##########
java/benchmark/plot_map_benchmark.py:
##########
@@ -0,0 +1,160 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+
+# Load the CSV data
+no_chunk_file = (
+ "/Users/chaokunyang/Desktop/chaokun/fury_open_source/nochunk-jmh-result.csv"
Review Comment:
Maybe this isn't the right address
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fury.apache.org
For additional commands, e-mail: commits-help@fury.apache.org