You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2020/12/15 22:19:30 UTC
[ignite-3] branch ignite-13618-javapoet updated: WIP. Implement
classloader for dynamic classes. Fix tests.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch ignite-13618-javapoet
in repository https://gitbox.apache.org/repos/asf/ignite-3.git
The following commit(s) were added to refs/heads/ignite-13618-javapoet by this push:
new 3c7ef24 WIP. Implement classloader for dynamic classes. Fix tests.
3c7ef24 is described below
commit 3c7ef24763bf1df88c244736cbecd6e704ffbfd5
Author: Andrew Mashenkov <an...@gmail.com>
AuthorDate: Wed Dec 16 01:19:19 2020 +0300
WIP.
Implement classloader for dynamic classes. Fix tests.
---
.../internal/schema/marshaller/MarshallerUtil.java | 63 ++++++++++++++++------
.../schema/marshaller/SerializerFactory.java | 4 +-
...izerGenerator.java => SerializerGenerator.java} | 20 ++-----
.../benchmarks/SerializerBenchmarkTest.java | 2 +
.../schema/marshaller/JavaSerializerTest.java | 60 ++++++++++-----------
5 files changed, 85 insertions(+), 64 deletions(-)
diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java
index e04fc64..4f36129 100644
--- a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java
+++ b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/MarshallerUtil.java
@@ -144,21 +144,19 @@ public final class MarshallerUtil {
public static ClassLoader compileCode(JavaFile javafile, Map<String, byte[]> classes) {
final JavaCompiler cmp = ToolProvider.getSystemJavaCompiler();
- final Map<String, byte[]> classes0 = classes == null ? new HashMap<>() : classes;
+ final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- final MemoryClassLoader loader = new MemoryClassLoader(classes0, ClassLoader.getSystemClassLoader());
+ Map<String, byte[]> classes = classLoader instanceof MemoryClassLoader ?
+ ((MemoryClassLoader)classLoader).classBytes : new ConcurrentHashMap<>();
- try (final MemoryJavaFileManager fileManager = new MemoryJavaFileManager(cmp.getStandardFileManager(null, null, null), classes0) {
- @Override public ClassLoader getClassLoader(Location location) {
- return loader;
- }
- }) {
+ try (final MemoryJavaFileManager fileManager = new MemoryJavaFileManager(cmp.getStandardFileManager(null, null, null), classes)) {
DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();
JavaCompiler.CompilationTask task = cmp.getTask(null, fileManager, diagnostics, null, null, Collections.singletonList(javafile.toJavaFileObject()));
if (task.call())
- return loader;
+ return classLoader instanceof MemoryClassLoader ? classLoader :
+ new MemoryClassLoader(classes, ClassLoader.getSystemClassLoader());
// TODO: write to log.
for (Diagnostic diagnostic : diagnostics.getDiagnostics()) {
@@ -187,8 +185,17 @@ public final class MarshallerUtil {
this.classes = classes;
}
- Map<String, byte[]> getClasses() {
- return classes;
+ private static class MemoryJavaFileObject extends SimpleJavaFileObject {
+ private final byte[] bytes;
+
+ public MemoryJavaFileObject(String className, byte[] bytes) {
+ super(URI.create(className + Kind.CLASS.extension), Kind.CLASS);
+ this.bytes = bytes;
+ }
+
+ @Override public InputStream openInputStream() {
+ return new ByteArrayInputStream(bytes);
+ }
}
/**
@@ -236,6 +243,32 @@ public final class MarshallerUtil {
return it;
}
+ @Override public String inferBinaryName(Location location, JavaFileObject jfo) {
+ if (!(jfo instanceof MemoryJavaFileObject)) {
+ String result = super.inferBinaryName(location, jfo);
+ assert result != null;
+ return result;
+ }
+
+ // A [Java]FileObject's "name" looks like this: "/orc/codehaus/commons/compiler/Foo.java".
+ // A [Java]FileObject's "binary name" looks like "java.lang.annotation.Retention".
+
+ String bn = jfo.getName();
+ if (bn.startsWith("/"))
+ bn = bn.substring(1);
+
+ if (!bn.endsWith(jfo.getKind().extension)) {
+ throw new AssertionError(
+ "Name \"" + jfo.getName() + "\" does not match kind \"" + jfo.getKind() + "\""
+ );
+ }
+ bn = bn.substring(0, bn.length() - jfo.getKind().extension.length());
+
+ bn = bn.replace('/', '.');
+
+ return bn;
+ }
+
@Override public JavaFileObject getJavaFileForInput(Location location, String className, Kind kind) throws IOException {
if (location == StandardLocation.CLASS_OUTPUT) {
JavaFileObject javaFileObject = getJavaFileObjectByName(className);
@@ -250,13 +283,9 @@ public final class MarshallerUtil {
@Nullable private JavaFileObject getJavaFileObjectByName(String className) {
final byte[] bytes = classes.get(className);
- if (bytes != null) {
- return new SimpleJavaFileObject(URI.create(className), JavaFileObject.Kind.CLASS) {
- @Override public InputStream openInputStream() {
- return new ByteArrayInputStream(bytes);
- }
- };
- }
+ if (bytes != null)
+ return new MemoryJavaFileObject(className, bytes);
+
return null;
}
diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java
index d9a8295..ce88b18 100644
--- a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java
+++ b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/SerializerFactory.java
@@ -18,7 +18,7 @@
package org.apache.ignite.internal.schema.marshaller;
import org.apache.ignite.internal.schema.SchemaDescriptor;
-import org.apache.ignite.internal.schema.marshaller.generator.JaninoSerializerGenerator;
+import org.apache.ignite.internal.schema.marshaller.generator.SerializerGenerator;
import org.apache.ignite.internal.schema.marshaller.reflection.JavaSerializerFactory;
/**
@@ -30,7 +30,7 @@ public interface SerializerFactory {
* @return Serializer factory back by code generator.
*/
public static SerializerFactory createJaninoSerializerFactory() {
- return new JaninoSerializerGenerator();
+ return new SerializerGenerator();
}
/**
diff --git a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/JaninoSerializerGenerator.java b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/SerializerGenerator.java
similarity index 96%
rename from modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/JaninoSerializerGenerator.java
rename to modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/SerializerGenerator.java
index 9de4011..42786bf 100644
--- a/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/JaninoSerializerGenerator.java
+++ b/modules/commons/src/main/java/org/apache/ignite/internal/schema/marshaller/generator/SerializerGenerator.java
@@ -26,11 +26,8 @@ import com.squareup.javapoet.TypeName;
import com.squareup.javapoet.TypeSpec;
import java.lang.invoke.MethodHandles;
import java.lang.reflect.InvocationTargetException;
-import java.util.HashMap;
-import java.util.Map;
import javax.annotation.processing.Generated;
import javax.lang.model.element.Modifier;
-import javax.tools.JavaFileManager;
import org.apache.ignite.internal.schema.ByteBufferTuple;
import org.apache.ignite.internal.schema.Columns;
import org.apache.ignite.internal.schema.SchemaDescriptor;
@@ -44,19 +41,16 @@ import org.apache.ignite.internal.schema.marshaller.SerializerFactory;
import org.apache.ignite.internal.util.ObjectFactory;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
-import org.jetbrains.annotations.TestOnly;
/**
* {@link Serializer} code generator backed with Janino.
*/
-public class JaninoSerializerGenerator implements SerializerFactory {
+public class SerializerGenerator implements SerializerFactory {
public static final String SERIALIZER_PACKAGE_NAME = "org.apache.ignite.internal.schema.marshaller";
public static final String SERIALIZER_CLASS_NAME_PREFIX = "JaninoSerializerForSchema_";
- private Map<String, byte[]> classes;
-
/** {@inheritDoc} */
@Override public Serializer create(
SchemaDescriptor schema,
@@ -71,7 +65,7 @@ public class JaninoSerializerGenerator implements SerializerFactory {
//TODO: pass code to logger on trace level.
System.out.println(javaFile.toString());
- final ClassLoader loader = MarshallerUtil.compileCode(javaFile, classes);
+ final ClassLoader loader = MarshallerUtil.compileCode(javaFile);
try {
final Class<?> aClass = loader.loadClass(javaFile.packageName + '.' + className);
@@ -86,11 +80,6 @@ public class JaninoSerializerGenerator implements SerializerFactory {
}
}
- @TestOnly
- public void setClasses(Map<String, byte[]> classes) {
- this.classes = classes;
- }
-
/**
* Generates serializer code.
*
@@ -103,7 +92,7 @@ public class JaninoSerializerGenerator implements SerializerFactory {
try {
// Build field accessor generators.
final MarshallerExprGenerator keyMarsh = createObjectMarshaller(keyClass, "keyFactory", schema.keyColumns(), 0);
- final MarshallerExprGenerator valMarsh = createObjectMarshaller(valClass, "valFactory", schema.valueColumns(), schema.valueColumns().length());
+ final MarshallerExprGenerator valMarsh = createObjectMarshaller(valClass, "valFactory", schema.valueColumns(), schema.keyColumns().length());
final TypeSpec.Builder classBuilder = TypeSpec.classBuilder(className)
.addSuperinterface(Serializer.class)
@@ -171,7 +160,8 @@ public class JaninoSerializerGenerator implements SerializerFactory {
classBuilder.addStaticBlock(staticInitBuilder.build());
}
- private MethodSpec generateHelpersMetod(SchemaDescriptor schema, MarshallerExprGenerator keyMarsh, MarshallerExprGenerator valMarsh) {
+ private MethodSpec generateHelpersMetod(SchemaDescriptor schema, MarshallerExprGenerator keyMarsh,
+ MarshallerExprGenerator valMarsh) {
final MethodSpec.Builder builder = MethodSpec
.methodBuilder("createAssembler")
.addModifiers(Modifier.PRIVATE)
diff --git a/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java b/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java
index e84e73b..f574dec 100644
--- a/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java
+++ b/modules/commons/src/test/java/org/apache/ignite/internal/benchmarks/SerializerBenchmarkTest.java
@@ -94,6 +94,8 @@ public class SerializerBenchmarkTest {
*/
@Setup
public void init() throws Exception {
+ Thread.currentThread().setContextClassLoader(MarshallerUtil.dynamicClassLoader());
+
long seed = System.currentTimeMillis();
rnd = new Random(seed);
diff --git a/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java b/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java
index a9589e9..d10fdf7 100644
--- a/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java
+++ b/modules/commons/src/test/java/org/apache/ignite/internal/schema/marshaller/JavaSerializerTest.java
@@ -37,7 +37,7 @@ import org.apache.ignite.internal.schema.NativeType;
import org.apache.ignite.internal.schema.NativeTypeSpec;
import org.apache.ignite.internal.schema.SchemaDescriptor;
import org.apache.ignite.internal.schema.TestUtils;
-import org.apache.ignite.internal.schema.marshaller.generator.JaninoSerializerGenerator;
+import org.apache.ignite.internal.schema.marshaller.generator.SerializerGenerator;
import org.apache.ignite.internal.schema.marshaller.reflection.JavaSerializerFactory;
import org.apache.ignite.internal.util.ObjectFactory;
import org.junit.jupiter.api.BeforeEach;
@@ -71,7 +71,7 @@ public class JavaSerializerTest {
*/
private static List<SerializerFactory> serializerFactoryProvider() {
return Arrays.asList(
- new JaninoSerializerGenerator(),
+ new SerializerGenerator(),
new JavaSerializerFactory()
);
}
@@ -265,46 +265,49 @@ public class JavaSerializerTest {
assertThrows(IllegalStateException.class,
() -> factory.create(schema, key.getClass(), val.getClass()),
"Class has no default constructor: class=org.apache.ignite.internal.schema.marshaller.JavaSerializerTest$WrongTestObject"
- );
+ );
}
-
/**
*
*/
@ParameterizedTest
@MethodSource("serializerFactoryProvider")
public void testClassLoader(SerializerFactory factory) throws SerializationException {
- Column[] keyCols = new Column[] {
- new Column("key", LONG, false)
- };
+ final ClassLoader prevClassLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(MarshallerUtil.dynamicClassLoader());
- Column[] valCols = new Column[] {
- new Column("col0", LONG, false),
- new Column("col1", LONG, false),
- new Column("col2", LONG, false),
- };
+ Column[] keyCols = new Column[] {
+ new Column("key", LONG, false)
+ };
- SchemaDescriptor schema = new SchemaDescriptor(1, new Columns(keyCols), new Columns(valCols));
+ Column[] valCols = new Column[] {
+ new Column("col0", LONG, false),
+ new Column("col1", LONG, false),
+ new Column("col2", LONG, false),
+ };
- final Map<String, byte[]> classes = new HashMap<>();
- final Class<?> valClass = createGeneratedObjectClass(Long.class, classes);
- final ObjectFactory<?> objFactory = new ObjectFactory<>(valClass);
+ SchemaDescriptor schema = new SchemaDescriptor(1, new Columns(keyCols), new Columns(valCols));
- final Long key = rnd.nextLong();
+ final Class<?> valClass = createGeneratedObjectClass(Long.class);
+ final ObjectFactory<?> objFactory = new ObjectFactory<>(valClass);
- if (factory instanceof JaninoSerializerGenerator)
- ((JaninoSerializerGenerator)factory).setClasses(classes);
+ final Long key = rnd.nextLong();
- Serializer serializer = factory.create(schema, key.getClass(), valClass);
+ Serializer serializer = factory.create(schema, key.getClass(), valClass);
- byte[] bytes = serializer.serialize(key, objFactory.create());
+ byte[] bytes = serializer.serialize(key, objFactory.create());
- Object key1 = serializer.deserializeKey(bytes);
- Object val1 = serializer.deserializeValue(bytes);
+ Object key1 = serializer.deserializeKey(bytes);
+ Object val1 = serializer.deserializeValue(bytes);
- assertTrue(key.getClass().isInstance(key1));
- assertTrue(valClass.isInstance(val1));
+ assertTrue(key.getClass().isInstance(key1));
+ assertTrue(valClass.isInstance(val1));
+ }
+ finally {
+ Thread.currentThread().setContextClassLoader(prevClassLoader);
+ }
}
/**
@@ -367,10 +370,9 @@ public class JavaSerializerTest {
* Generate class for test objects.
*
* @param fieldType Field type.
- * @param classes
* @return Generated test object class.
*/
- private Class<?> createGeneratedObjectClass(Class<?> fieldType, Map<String, byte[]> classes) {
+ private Class<?> createGeneratedObjectClass(Class<?> fieldType) {
final String packageName = getClass().getPackageName();
final String className = "GeneratedTestObject";
@@ -392,10 +394,8 @@ public class JavaSerializerTest {
final JavaFile javaFile = JavaFile.builder(packageName, classBuilder.build()).build();
- final ClassLoader loader = MarshallerUtil.compileCode(javaFile, classes);
-
try {
- return loader.loadClass(packageName + '.' + className);
+ return MarshallerUtil.compileCode(javaFile).loadClass(packageName + '.' + className);
}
catch (Exception ex) {
throw new IllegalStateException("Failed to compile/instantiate generated Serializer.", ex);