You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ik...@apache.org on 2017/01/13 18:49:18 UTC
git commit: updated refs/heads/trunk to ad84a99
Repository: giraph
Updated Branches:
refs/heads/trunk 3b53360ab -> ad84a99e2
GIRAPH-1129
closes #13
Project: http://git-wip-us.apache.org/repos/asf/giraph/repo
Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/ad84a99e
Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/ad84a99e
Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/ad84a99e
Branch: refs/heads/trunk
Commit: ad84a99e2ba630fa81aa6823515bfdeac1f3fe8a
Parents: 3b53360
Author: Igor Kabiljo <ik...@fb.com>
Authored: Fri Jan 13 10:48:45 2017 -0800
Committer: Igor Kabiljo <ik...@fb.com>
Committed: Fri Jan 13 10:48:45 2017 -0800
----------------------------------------------------------------------
giraph-block-app-8/pom.xml | 4 +
.../giraph/compiling/ClassCodeGenerator.java | 35 +++++
.../giraph/compiling/FactoryCodeGenerator.java | 115 ++++++++++++++
.../giraph/compiling/LambdaConfOption.java | 103 +++++++++++++
.../compiling/ObjectFactoryConfOption.java | 134 ++++++++++++++++
.../compiling/ObjectInitializerConfOption.java | 39 +++++
.../giraph/compiling/RuntimeClassGenerator.java | 113 ++++++++++++++
.../RuntimeObjectFactoryGenerator.java | 68 +++++++++
.../apache/giraph/compiling/package-info.java | 21 +++
.../giraph/compiling/TestLambdaConfOption.java | 109 +++++++++++++
.../TestRuntimeObjectFactoryCreationUtils.java | 151 +++++++++++++++++++
pom.xml | 12 +-
12 files changed, 901 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/pom.xml
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/pom.xml b/giraph-block-app-8/pom.xml
index 82885be..96035e7 100644
--- a/giraph-block-app-8/pom.xml
+++ b/giraph-block-app-8/pom.xml
@@ -119,6 +119,10 @@ under the License.
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>compiler</artifactId>
+ </dependency>
<!-- runtime dependency -->
<dependency>
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
new file mode 100644
index 0000000..a2dcaee
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ClassCodeGenerator.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+/**
+ * Interface which can generate code from code snippet
+ */
+public interface ClassCodeGenerator {
+ /**
+ * Generate code for creating a class inside of specified package with given
+ * class name, using provided code snippet
+ *
+ * @param packageName Package to create class in
+ * @param className Name of the class
+ * @param codeSnippet Cod snippet for this class
+ * @return Code used to generate the class
+ */
+ String generateCode(
+ String packageName, String className, String codeSnippet);
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
new file mode 100644
index 0000000..dab03d3
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/FactoryCodeGenerator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+
+import org.apache.giraph.utils.Factory;
+
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Class which creates object factories from code snippet to create and return
+ * object
+ */
+public class FactoryCodeGenerator implements ClassCodeGenerator {
+ private final Type objectType;
+
+ public FactoryCodeGenerator(Type objectType) {
+ this.objectType = objectType;
+ }
+
+ @Override
+ public String generateCode(
+ String packageName, String className, String creationCode) {
+ StringBuilder src = new StringBuilder();
+ src.append("package ").append(packageName).append(";\n");
+ // Import Factory
+ src.append("import ").append(Factory.class.getName()).append(";\n");
+
+ appendTypeImport(src, objectType);
+ String simpleName = getSimpleName(objectType);
+
+ // Allow classes from java.util to be used
+ src.append("import java.util.*;\n");
+ // Object compiled in one VM cannot be deserialized in other,
+ // so mark generated class as NonKryoWritable
+ src.append(
+ "import org.apache.giraph.writable.kryo.markers.NonKryoWritable;\n");
+ src.append("public class ").append(className).append(
+ " implements ").append(Factory.class.getSimpleName()).append(
+ "<").append(simpleName).append(">, NonKryoWritable {\n");
+ src.append("public ").append(simpleName).append(" create() {\n");
+ src.append(creationCode).append("\n").append("}\n").append("}\n");
+ return src.toString();
+ }
+
+ public static String getSimpleName(TypeToken<?> typeToken) {
+ return getSimpleName(typeToken.getType());
+ }
+
+ public static String getSimpleName(Type type) {
+ return (type instanceof Class) ?
+ ((Class<?>) type).getCanonicalName() :
+ type.toString();
+ }
+
+ private static void appendTypeImport(StringBuilder src, Type objectType) {
+ if (objectType == null) {
+ return;
+ }
+
+ if (objectType instanceof Class) {
+ appendClassImport(src, (Class<?>) objectType);
+ } else if (objectType instanceof ParameterizedType) {
+ appendTypeImport(src, ((ParameterizedType) objectType).getRawType());
+ appendTypeImport(src, ((ParameterizedType) objectType).getOwnerType());
+
+ for (Type innerType :
+ ((ParameterizedType) objectType).getActualTypeArguments()) {
+ appendTypeImport(src, innerType);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported " + objectType + " of type " + objectType.getClass());
+ }
+ }
+
+ private static void appendClassImport(
+ StringBuilder src, Class<?> objectClass) {
+ // Allow classes from the same package to be used
+ src.append("import ").append(
+ objectClass.getPackage().getName()).append(".*;\n");
+ // If objectClass is inner class then above import is not enough
+ src.append("import ").append(objectClass.getCanonicalName()).append(";\n");
+ }
+
+ @Override
+ public int hashCode() {
+ return objectType.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof FactoryCodeGenerator)) {
+ return false;
+ }
+ return objectType.equals(((FactoryCodeGenerator) obj).objectType);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
new file mode 100644
index 0000000..84e3407
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/LambdaConfOption.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import static org.apache.giraph.compiling.FactoryCodeGenerator.getSimpleName;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.lang.reflect.Type;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+
+import com.google.common.base.Preconditions;
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Conf option which is configured by specifying lambda body, and it returns
+ * compiled instance of wanted functional interface with given body.
+ * See TestLambdaConfOption for example.
+ *
+ * @param <T> Functional interface type
+ */
+public class LambdaConfOption<T> extends ObjectFactoryConfOption<T> {
+ private final String[] argumentNames;
+
+ public LambdaConfOption(String key, Class<T> classType, String codeSnippet,
+ String description, String... argumentNames) {
+ super(key, classType, codeSnippet, description);
+ this.argumentNames = argumentNames;
+ }
+
+ public LambdaConfOption(
+ String key, TypeToken<T> interfaceTypeToken, String codeSnippet,
+ String description, String... arguments) {
+ super(key, interfaceTypeToken, codeSnippet, description);
+ this.argumentNames = arguments;
+ }
+
+ @Override
+ protected String createFullCodeSnippet(String codeSnippet) {
+ if (codeSnippet == null || codeSnippet.isEmpty()) {
+ return null;
+ }
+
+ return createLambdaCode(codeSnippet, getInterfaceType(), argumentNames);
+ }
+
+ public static String createLambdaCode(
+ String body, Type type, String[] argumentNames) {
+ TypeToken<?> typeToken = TypeToken.of(type);
+ Class<?> rawType = typeToken.getRawType();
+
+ if (rawType.isInterface()) {
+ return "return (" +
+ Stream.of(argumentNames).collect(Collectors.joining(", ")) +
+ ") -> " + body + ";";
+ } else {
+ // Long workaround for abstract classes
+ Method sam = null;
+
+ for (Method m : rawType.getMethods()) {
+ if (Modifier.isAbstract(m.getModifiers())) {
+ Preconditions.checkState(sam == null);
+ sam = m;
+ }
+ }
+ Preconditions.checkState(sam != null);
+
+ Type[] params = sam.getGenericParameterTypes();
+ Preconditions.checkState(params.length == argumentNames.length);
+
+ String argList = IntStream.range(0, params.length).mapToObj(
+ (i) -> getSimpleName(typeToken.resolveType(params[i])) + " " +
+ argumentNames[i]
+ ).collect(Collectors.joining(", "));
+
+ return " return new " + getSimpleName(type) + "() {\n" +
+ " @Override\n" +
+ " public " +
+ getSimpleName(typeToken.resolveType(sam.getGenericReturnType())) +
+ " " + sam.getName() + "(" + argList + ") {\n" +
+ " return " + body + ";\n" +
+ " }\n" +
+ " };";
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
new file mode 100644
index 0000000..412763b
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectFactoryConfOption.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.lang.reflect.Type;
+
+import org.apache.giraph.conf.StrConfOption;
+import org.apache.giraph.function.Supplier;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.reflect.TypeToken;
+
+/**
+ * Option in configuration which can create objects
+ *
+ * @param <T> Type of objects for this option
+ */
+public class ObjectFactoryConfOption<T> {
+ /** Conf option which holds code snippet for creating objects */
+ private final StrConfOption codeSnippetConfOption;
+ /** Base interface for class */
+ private final Type interfaceType;
+
+ public ObjectFactoryConfOption(
+ String key, Class<T> interfaceClass, String defaultValue,
+ String description) {
+ codeSnippetConfOption = new StrConfOption(key, defaultValue, description);
+ this.interfaceType = interfaceClass;
+ }
+
+ public ObjectFactoryConfOption(
+ String key, TypeToken<T> interfaceTypeToken, String defaultValue,
+ String description) {
+ codeSnippetConfOption = new StrConfOption(key, defaultValue, description);
+ this.interfaceType = interfaceTypeToken.getType();
+ }
+
+ /**
+ * Create object from configuration, or return null if conf option is empty.
+ *
+ * Return objects are NonKryoWritable.
+ *
+ * @param conf Configuration
+ * @return New object, or null if conf option is empty
+ */
+ public final T createObject(Configuration conf) {
+ final String codeSnippet = createFullCodeSnippet(
+ codeSnippetConfOption.get(conf));
+ if (codeSnippet == null || codeSnippet.isEmpty()) {
+ return null;
+ } else {
+ return RuntimeObjectFactoryGenerator.<T>createFactory(
+ codeSnippet, interfaceType).create();
+ }
+ }
+
+ /**
+ * Take user passed codeSnippet and create full code snippet to be given to
+ * RuntimeObjectFactoryGenerator.
+ *
+ * Subclasses can override this method to customize it's logic.
+ */
+ protected String createFullCodeSnippet(String codeSnippet) {
+ return codeSnippet;
+ }
+
+ /**
+ * Create object cached supplier from configuration, which stores snippet for
+ * serializaition, working around objects themselves being NonKryoWritable.
+ *
+ * Object is created only once per new or deserialized instance of returned
+ * supplier.
+ */
+ public final Supplier<T> createSupplier(Configuration conf) {
+ final String codeSnippet = createFullCodeSnippet(
+ codeSnippetConfOption.get(conf));
+ final Type type = this.interfaceType;
+ if (codeSnippet == null || codeSnippet.isEmpty()) {
+ return null;
+ }
+ return createNewSupplier(codeSnippet, type);
+ }
+
+ private static <T> Supplier<T> createNewSupplier(
+ final String codeSnippet, final Type type) {
+ return RuntimeObjectFactoryGenerator.createCachedSupplier(
+ codeSnippet, type);
+ }
+
+ public void setCodeSnippet(Configuration conf, String codeSnippet) {
+ codeSnippetConfOption.set(conf, codeSnippet);
+ }
+
+ public void setCodeSnippetIfUnset(Configuration conf, String codeSnippet) {
+ codeSnippetConfOption.setIfUnset(conf, codeSnippet);
+ }
+
+ public boolean isDefaultValue(Configuration conf) {
+ return codeSnippetConfOption.isDefaultValue(conf);
+ }
+
+ public boolean isEmpty(Configuration conf) {
+ final String codeSnippet = createFullCodeSnippet(
+ codeSnippetConfOption.get(conf));
+ return codeSnippet == null || codeSnippet.isEmpty();
+ }
+
+ public String getCodeSnippet(Configuration conf) {
+ return codeSnippetConfOption.get(conf);
+ }
+
+ public Type getInterfaceType() {
+ return interfaceType;
+ }
+
+ public String getInterfaceName() {
+ return FactoryCodeGenerator.getSimpleName(interfaceType);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
new file mode 100644
index 0000000..9568a59
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/ObjectInitializerConfOption.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+/**
+ * Conf option which can initialize an object of specific class with desired
+ * parameters set.
+ * See TestRuntimeObjectFactoryCreationUtils for example.
+ * Make sure fields which you want to set are protected.
+ *
+ * @param <T> Class type
+ */
+public class ObjectInitializerConfOption<T>
+ extends ObjectFactoryConfOption<T> {
+ public ObjectInitializerConfOption(
+ String key, Class<T> classType, String codeSnippet, String description) {
+ super(key, classType, codeSnippet, description);
+ }
+
+ @Override
+ protected String createFullCodeSnippet(String codeSnippet) {
+ return "return new " + getInterfaceName() + "(){{" + codeSnippet + "}};";
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
new file mode 100644
index 0000000..32681f2
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeClassGenerator.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Preconditions;
+
+import net.openhft.compiler.CachedCompiler;
+
+/**
+ * Helper class for creating classes at runtime
+ */
+public class RuntimeClassGenerator {
+ private static final Logger LOG = Logger.getLogger(
+ RuntimeClassGenerator.class);
+
+ /**
+ * In order not to create multiple classes for exactly same Code generator
+ * and String, this keeps all known classes
+ */
+ private static final
+ Map<ClassCodeGenerator, Map<String, Class<?>>> KNOWN_CLASSES =
+ new HashMap<>();
+
+ /** Counter for number of created classes */
+ private static final
+ AtomicInteger CREATED_CLASSES_COUNTER = new AtomicInteger();
+
+ protected RuntimeClassGenerator() {
+ }
+
+ public static synchronized <T> Class<T> getOrCreateClass(
+ ClassCodeGenerator classCodeGenerator, String codeSnippet) {
+ Map<String, Class<?>> knownClassCodeGenerators =
+ KNOWN_CLASSES.get(classCodeGenerator);
+ if (knownClassCodeGenerators == null) {
+ knownClassCodeGenerators = new HashMap<>();
+ KNOWN_CLASSES.put(classCodeGenerator, knownClassCodeGenerators);
+ } else {
+ Class<?> generatedClass = knownClassCodeGenerators.get(codeSnippet);
+ if (generatedClass != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Reusing class for " + classCodeGenerator + " " + codeSnippet);
+ }
+ return (Class<T>) generatedClass;
+ }
+ }
+ String packageName = classCodeGenerator.getClass().getPackage().getName();
+ String className = classCodeGenerator.getClass().getSimpleName() +
+ CREATED_CLASSES_COUNTER.getAndIncrement();
+ String fullClassName = packageName + "." + className;
+ String classCode = classCodeGenerator.generateCode(
+ packageName, className, codeSnippet);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating " + fullClassName + " with \n" + classCode);
+ }
+ Class<T> generatedClass = generateClass(fullClassName, classCode);
+ knownClassCodeGenerators.put(codeSnippet, generatedClass);
+ return generatedClass;
+ }
+
+ /**
+ * Create class with a name, in the package and with source code.
+ */
+ public static synchronized <T> Class<T> createClassOnce(
+ String className, String packageName, String classCode) {
+ return generateClass(packageName + "." + className, classCode);
+ }
+
+ private static <T> Class<T> generateClass(
+ String fullClassName, String classCode) {
+ try {
+ Class<T> generatedClass = new CachedCompiler(null, null).loadFromJava(
+ fullClassName, classCode);
+ // Confirm that class was loaded into default class loader
+ Preconditions.checkState(generatedClass == Class.forName(fullClassName));
+ return generatedClass;
+ } catch (ClassNotFoundException e) {
+ throw new IllegalArgumentException(
+ "Couldn't be compile class " + fullClassName + " , with code: " +
+ classCode,
+ e);
+ }
+ }
+
+ /**
+ * For test
+ */
+ static int getCreatedClassesCount() {
+ return CREATED_CLASSES_COUNTER.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
new file mode 100644
index 0000000..fe14fb0
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/RuntimeObjectFactoryGenerator.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.lang.reflect.Type;
+
+import org.apache.giraph.function.Supplier;
+import org.apache.giraph.utils.Factory;
+
+/**
+ * Helper class for creating object factories at runtime
+ */
+public class RuntimeObjectFactoryGenerator extends RuntimeClassGenerator {
+ private RuntimeObjectFactoryGenerator() {
+ }
+
+ public static <T> Factory<T> createFactory(
+ String creationCode, Class<T> objectClass) {
+ return createFactory(creationCode, (Type) objectClass);
+ }
+
+ public static <T> Factory<T> createFactory(
+ String creationCode, Type objectType) {
+ FactoryCodeGenerator factoryCodeGenerator =
+ new FactoryCodeGenerator(objectType);
+ Class<Factory<T>> factoryClass = getOrCreateClass(
+ factoryCodeGenerator, creationCode);
+ try {
+ return factoryClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ public static <T> Supplier<T> createCachedSupplier(
+ String creationCode, Type objectType) {
+ Supplier<T> supplier = new Supplier<T>() {
+ private transient T cachedValue;
+
+ @Override
+ public T get() {
+ if (cachedValue == null) {
+ cachedValue = RuntimeObjectFactoryGenerator.<T>createFactory(
+ creationCode, objectType).create();
+ }
+ return cachedValue;
+ }
+ };
+ // Compile in current scope, and fail fast if there are compile errors
+ supplier.get();
+ return supplier;
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
new file mode 100644
index 0000000..0b48bf3
--- /dev/null
+++ b/giraph-block-app-8/src/main/java/org/apache/giraph/compiling/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Utilities for compiling code at runtime.
+ */
+package org.apache.giraph.compiling;
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
new file mode 100644
index 0000000..59f688e
--- /dev/null
+++ b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestLambdaConfOption.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import java.io.Serializable;
+
+import org.apache.giraph.function.Function;
+import org.apache.giraph.function.primitive.Obj2LongFunction;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.reflect.TypeToken;
+
+public class TestLambdaConfOption {
+ @Before
+ public void setLogging() {
+ Logger.getLogger(RuntimeClassGenerator.class).setLevel(Level.DEBUG);
+ }
+
+ @Test
+ public void testLambdaConfOptionClass() throws NoSuchFieldException, SecurityException {
+ Configuration conf = new Configuration();
+ LambdaConfOption<ValueInIteration> testObjectFactoryConfOption =
+ new LambdaConfOption<>("testFactory", ValueInIteration.class, "", "", "iter");
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "");
+ Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "iter + 5");
+ ValueInIteration f = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(10, f.get(5), 0.0);
+ }
+
+ @Test
+ public void testLambdaConfOptionTypeToken() throws NoSuchFieldException, SecurityException {
+ Configuration conf = new Configuration();
+ LambdaConfOption<Obj2LongFunction<LongWritable>> testObjectFactoryConfOption =
+ new LambdaConfOption<>("testFactory", new TypeToken<Obj2LongFunction<LongWritable>>(){}, "", "", "id");
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "");
+ Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "id.get()");
+ Obj2LongFunction<LongWritable> f = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(10, f.apply(new LongWritable(10)));
+ }
+
+ @Test
+ public void testLambdaConfOptionTypeToken2() throws NoSuchFieldException, SecurityException {
+ Configuration conf = new Configuration();
+ LambdaConfOption<Function<LongWritable, DoubleWritable>> testObjectFactoryConfOption =
+ new LambdaConfOption<>("testFactory", new TypeToken<Function<LongWritable, DoubleWritable>>(){}, "", "", "id");
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "");
+ Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "new DoubleWritable(id.get())");
+ Function<LongWritable, DoubleWritable> f = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(10, f.apply(new LongWritable(10)).get(), 0.0);
+ }
+
+ /**
+ * Value that depends on current iteration.
+ */
+ public interface ValueInIteration extends Serializable {
+ float get(int iteration);
+ }
+
+ public static abstract class AbstractClass implements ValueInIteration {
+ public static int get5() {
+ return 5;
+ }
+ }
+
+ @Test
+ public void testLambdaConfOptionAbstractClass() throws NoSuchFieldException, SecurityException {
+ Configuration conf = new Configuration();
+ LambdaConfOption<AbstractClass> testObjectFactoryConfOption =
+ new LambdaConfOption<>("testFactory", AbstractClass.class, "", "", "iter");
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "");
+ Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "iter + get5()");
+ ValueInIteration f = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(10, f.get(5), 0.0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
----------------------------------------------------------------------
diff --git a/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
new file mode 100644
index 0000000..2be280b
--- /dev/null
+++ b/giraph-block-app-8/src/test/java/org/apache/giraph/compiling/TestRuntimeObjectFactoryCreationUtils.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.giraph.compiling;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestRuntimeObjectFactoryCreationUtils {
+ @Before
+ public void setLogging() {
+ Logger.getLogger(RuntimeClassGenerator.class).setLevel(Level.DEBUG);
+ }
+
+ @Test
+ public void testRuntimeObjectFactoryCreationUtils() {
+ int startCount = RuntimeObjectFactoryGenerator.getCreatedClassesCount();
+
+ TestClass1 testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+ "return new TestClass1(1, \"A\", 0.5);", TestClass1.class).create();
+ Assert.assertEquals(testClass1Object.x, 1);
+ Assert.assertEquals(testClass1Object.y, "A");
+ Assert.assertEquals(testClass1Object.z, 0.5, 0);
+ Assert.assertEquals(startCount + 1,
+ RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+ TestClass2 testClass2Object = RuntimeObjectFactoryGenerator.createFactory(
+ "return new TestClass2(5);", TestClass2.class).create();
+ Assert.assertEquals(testClass2Object.a, 5);
+ Assert.assertEquals(startCount + 2,
+ RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+ testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+ "return new TestClass1(5, \"XY\", 1.5).setZ(3.5);", TestClass1.class).create();
+ Assert.assertEquals(testClass1Object.x, 5);
+ Assert.assertEquals(testClass1Object.y, "XY");
+ Assert.assertEquals(testClass1Object.z, 3.5, 0);
+ Assert.assertEquals(startCount + 3,
+ RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+ testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+ "return new TestClass1(1, \"A\", 0.5);", TestClass1.class).create();
+ Assert.assertEquals(testClass1Object.x, 1);
+ Assert.assertEquals(testClass1Object.y, "A");
+ Assert.assertEquals(testClass1Object.z, 0.5, 0);
+ Assert.assertEquals(startCount + 3,
+ RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+ testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+ "TestClass1 ret = new TestClass1(5, \"XY\", 1.5); ret.setZ(3.5); return ret;",
+ TestClass1.class).create();
+ Assert.assertEquals(testClass1Object.x, 5);
+ Assert.assertEquals(testClass1Object.y, "XY");
+ Assert.assertEquals(testClass1Object.z, 3.5, 0);
+ Assert.assertEquals(startCount + 4,
+ RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+
+ testClass1Object = RuntimeObjectFactoryGenerator.createFactory(
+ "return new TestClass1() {{ x=1;y=\"A\";z=0.5; }};",
+ TestClass1.class).create();
+ Assert.assertEquals(testClass1Object.x, 1);
+ Assert.assertEquals(testClass1Object.y, "A");
+ Assert.assertEquals(testClass1Object.z, 0.5, 0);
+ Assert.assertEquals(startCount + 5,
+ RuntimeObjectFactoryGenerator.getCreatedClassesCount());
+ }
+
+ @Test
+ public void testObjectConfOption() {
+ Configuration conf = new Configuration();
+ ObjectFactoryConfOption<TestClass1> testObjectFactoryConfOption = new ObjectFactoryConfOption<>(
+ "testFactory", TestClass1.class, "return new TestClass1(50, \"XY\", 1.5);", "");
+ TestClass1 testClass1Object = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(testClass1Object.x, 50);
+ Assert.assertEquals(testClass1Object.y, "XY");
+ Assert.assertEquals(testClass1Object.z, 1.5, 0);
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "");
+ Assert.assertNull(testObjectFactoryConfOption.createObject(conf));
+
+ testObjectFactoryConfOption.setCodeSnippet(conf, "return new TestClass1(10, \"A\", 0.5);");
+ testClass1Object = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(testClass1Object.x, 10);
+ Assert.assertEquals(testClass1Object.y, "A");
+ Assert.assertEquals(testClass1Object.z, 0.5, 0);
+
+ TestClass1 testClass1OtherObject = testObjectFactoryConfOption.createObject(conf);
+ Assert.assertEquals(testClass1OtherObject.x, 10);
+ Assert.assertEquals(testClass1OtherObject.y, "A");
+ Assert.assertEquals(testClass1OtherObject.z, 0.5, 0);
+
+ // Verify that createObject returns different instance
+ testClass1OtherObject.setZ(5);
+ Assert.assertEquals(testClass1OtherObject.z, 5, 0);
+ Assert.assertEquals(testClass1Object.z, 0.5, 0);
+
+ ObjectInitializerConfOption<TestClass1> testObjectInitializerConfOption =
+ new ObjectInitializerConfOption<>("testInitializer", TestClass1.class,
+ "x=3;y=\"Y\";z=3.5;", "");
+ testClass1Object = testObjectInitializerConfOption.createObject(conf);
+ Assert.assertEquals(testClass1Object.x, 3);
+ Assert.assertEquals(testClass1Object.y, "Y");
+ Assert.assertEquals(testClass1Object.z, 3.5, 0);
+ }
+
+ public static class TestClass1 {
+ protected int x;
+ protected String y;
+ protected double z;
+
+ public TestClass1(int x, String y, double z) {
+ this.x = x;
+ this.y = y;
+ this.z = z;
+ }
+
+ public TestClass1() {
+ }
+
+ public TestClass1 setZ(double z) {
+ this.z = z;
+ return this;
+ }
+ }
+
+ public static class TestClass2 {
+ private final int a;
+
+ public TestClass2(int a) {
+ this.a = a;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/giraph/blob/ad84a99e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 017fd1f..91128f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -349,11 +349,12 @@ under the License.
<dep.log4j.version>1.2.17</dep.log4j.version>
<dep.mockito.version>1.9.5</dep.mockito.version>
<!-- note: old version of netty is required by hadoop_facebook for tests to succeed -->
- <dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
<dep.netty.version>4.0.14.Final</dep.netty.version>
+ <dep.oldnetty.version>3.2.2.Final</dep.oldnetty.version>
<dep.objenesis.version>2.1</dep.objenesis.version>
+ <dep.openhft-compiler.version>2.2.1</dep.openhft-compiler.version>
<dep.paranamer.version>2.5.2</dep.paranamer.version>
- <dep.slf4j.version>1.7.5</dep.slf4j.version>
+ <dep.slf4j.version>1.7.6</dep.slf4j.version>
<dep.tinkerpop.rexter.version>2.4.0</dep.tinkerpop.rexter.version>
<dep.typetools.version>0.2.1</dep.typetools.version>
<dep.yammer-metrics.version>2.2.0</dep.yammer-metrics.version>
@@ -2072,7 +2073,12 @@ under the License.
<artifactId>slf4j-log4j12</artifactId>
<version>${dep.slf4j.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>net.openhft</groupId>
+ <artifactId>compiler</artifactId>
+ <version>${dep.openhft-compiler.version}</version>
+ </dependency>
+
<!-- provided dependencies. sorted lexicographically. -->
<dependency>
<groupId>commons-collections</groupId>