You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by is...@apache.org on 2015/11/11 21:35:54 UTC
[1/2] incubator-apex-core git commit: APEX-188 #resolve #comment read
only class metadata(class signature with default constructor) when construct
the type graph and load the detail until it is required
Repository: incubator-apex-core
Updated Branches:
refs/heads/devel-3 b3863b24b -> 71ebf1724
APEX-188 #resolve #comment read only class metadata(class signature with default constructor) when construct the type graph and load the detail until it is required
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/0631c9b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/0631c9b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/0631c9b9
Branch: refs/heads/devel-3
Commit: 0631c9b98fc266d1f43ebfb21e378c3676a2af84
Parents: b3863b2
Author: siyuan <hs...@gmail.com>
Authored: Wed Nov 11 09:48:15 2015 -0800
Committer: siyuan <hs...@gmail.com>
Committed: Wed Nov 11 09:48:15 2015 -0800
----------------------------------------------------------------------
engine/pom.xml | 2 +-
.../stram/webapp/OperatorDiscoverer.java | 1 -
.../com/datatorrent/stram/webapp/TypeGraph.java | 460 ++++++++++---------
.../stram/webapp/TypeGraphFactory.java | 14 +-
.../datatorrent/stram/webapp/asm/ASMUtil.java | 5 +
.../stram/webapp/asm/ClassNodeType.java | 34 +-
.../stram/webapp/asm/CompactUtil.java | 20 +-
.../stram/webapp/asm/FastClassIndexReader.java | 340 ++++++++++++++
.../stram/webapp/OperatorDiscoveryTest.java | 1 +
.../webapp/asm/FastClassIndexReaderTest.java | 109 +++++
pom.xml | 1 +
11 files changed, 727 insertions(+), 260 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/pom.xml
----------------------------------------------------------------------
diff --git a/engine/pom.xml b/engine/pom.xml
index 791016d..644f1de 100644
--- a/engine/pom.xml
+++ b/engine/pom.xml
@@ -145,7 +145,7 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
- <maxAllowedViolations>2020</maxAllowedViolations>
+ <maxAllowedViolations>1869</maxAllowedViolations>
</configuration>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
index 55af0ae..7a24c51 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
@@ -375,7 +375,6 @@ public class OperatorDiscoverer
typeGraph.trim();
- typeGraph.updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles);
} finally {
for (Entry<String, JarFile> entry : openJarFiles.entrySet()) {
try {
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
index 2a55e8f..6cf36f3 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
@@ -22,27 +22,49 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+import java.util.Queue;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.regex.Pattern;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.text.WordUtils;
+
import org.codehaus.jackson.map.deser.std.FromStringDeserializer;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
-import org.apache.xbean.asm5.ClassReader;
-import org.apache.xbean.asm5.Opcodes;
-import org.apache.xbean.asm5.tree.ClassNode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.xbean.asm5.ClassReader;
+import org.apache.xbean.asm5.tree.ClassNode;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.primitives.Primitives;
+
import com.datatorrent.api.Component;
import com.datatorrent.api.InputOperator;
import com.datatorrent.api.Operator;
-
import com.datatorrent.common.util.BaseOperator;
import com.datatorrent.netlet.util.DTThrowable;
import com.datatorrent.stram.webapp.asm.ClassNodeType;
@@ -51,6 +73,7 @@ import com.datatorrent.stram.webapp.asm.CompactClassNode;
import com.datatorrent.stram.webapp.asm.CompactFieldNode;
import com.datatorrent.stram.webapp.asm.CompactMethodNode;
import com.datatorrent.stram.webapp.asm.CompactUtil;
+import com.datatorrent.stram.webapp.asm.FastClassIndexReader;
import com.datatorrent.stram.webapp.asm.MethodSignatureVisitor;
import com.datatorrent.stram.webapp.asm.Type;
import com.datatorrent.stram.webapp.asm.Type.ArrayTypeNode;
@@ -58,12 +81,6 @@ import com.datatorrent.stram.webapp.asm.Type.ParameterizedTypeNode;
import com.datatorrent.stram.webapp.asm.Type.TypeNode;
import com.datatorrent.stram.webapp.asm.Type.TypeVariableNode;
import com.datatorrent.stram.webapp.asm.Type.WildcardTypeNode;
-import com.esotericsoftware.kryo.Kryo;
-import com.esotericsoftware.kryo.Serializer;
-import com.esotericsoftware.kryo.io.Input;
-import com.esotericsoftware.kryo.io.Output;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.primitives.Primitives;
/**
* A graph data structure holds all type information and their relationship needed in app builder
@@ -103,8 +120,9 @@ public class TypeGraph
TypeGraphVertex parentVertex = typeGraph.get(parentClassName);
TypeGraphVertex classVertex = typeGraph.get(subClassName);
- if(parentVertex == null || classVertex == null)
+ if (parentVertex == null || classVertex == null) {
return false;
+ }
return TypeGraph.isAncestor(parentVertex, classVertex);
}
@@ -123,8 +141,9 @@ public class TypeGraph
return false;
}
for (TypeGraphVertex vertex : tgv.ancestors) {
- if (isAncestor(typeTgv, vertex))
+ if (isAncestor(typeTgv, vertex)) {
return true;
+ }
}
return false;
}
@@ -134,7 +153,8 @@ public class TypeGraph
return typeGraph.get(typeName);
}
- enum UI_TYPE {
+ enum UI_TYPE
+ {
LIST("List", Collection.class.getName()),
@@ -204,8 +224,8 @@ public class TypeGraph
{
List<String> allTypes = TypeGraph.getAllAncestors(tgv, true);
for (UI_TYPE type : UI_TYPE.values()) {
- for (String assignable : type.allAssignableTypes){
- if(allTypes.contains(assignable)){
+ for (String assignable : type.allAssignableTypes) {
+ if (allTypes.contains(assignable)) {
return type;
}
}
@@ -221,17 +241,19 @@ public class TypeGraph
}
}
- public static List<String> getAllAncestors(TypeGraphVertex tgv, boolean include) {
+ public static List<String> getAllAncestors(TypeGraphVertex tgv, boolean include)
+ {
List<String> result = new LinkedList<String>();
- if(include) {
+ if (include) {
result.add(tgv.typeName);
}
getAllAncestors(tgv, result);
return result;
}
- private static void getAllAncestors(TypeGraphVertex tgv, List<String> result) {
- for(TypeGraphVertex an : tgv.ancestors){
+ private static void getAllAncestors(TypeGraphVertex tgv, List<String> result)
+ {
+ for (TypeGraphVertex an : tgv.ancestors) {
result.add(an.typeName);
getAllAncestors(an, result);
}
@@ -245,39 +267,37 @@ public class TypeGraph
private TypeGraphVertex addNode(InputStream input, String resName) throws IOException
{
try {
-
- ClassReader reader = new ClassReader(input);
- ClassNode classN = new ClassNodeType();
- reader.accept(classN, ClassReader.SKIP_CODE);
- CompactClassNode ccn = CompactUtil.compactClassNode(classN);
- String typeName = classN.name.replace('/', '.');
- TypeGraphVertex tgv = null;
- TypeGraphVertex ptgv = null;
+
+ FastClassIndexReader fastClassIndexReader = new FastClassIndexReader(input);
+ String typeName = fastClassIndexReader.getName().replace('/', '.');
+ TypeGraphVertex tgv;
+ TypeGraphVertex ptgv;
if (typeGraph.containsKey(typeName)) {
tgv = typeGraph.get(typeName);
- tgv.setClassNode(ccn);
- tgv.setJarName(resName); // If tgv was already populated for superclass/interface, jar name needs to be updated
+ tgv.setIsRealNode(true);
+ tgv.setJarName(resName); // If tgv was already populated for superclass/interface, jar name needs to be updated
+ tgv.setIsInstantiable(fastClassIndexReader.isInstantiable());
} else {
- tgv = new TypeGraphVertex(typeName, resName, ccn);
+ tgv = new TypeGraphVertex(this, typeName, resName, true, fastClassIndexReader.isInstantiable());
typeGraph.put(typeName, tgv);
}
- String immediateP = reader.getSuperName();
+ String immediateP = fastClassIndexReader.getSuperName();
if (immediateP != null) {
immediateP = immediateP.replace('/', '.');
ptgv = typeGraph.get(immediateP);
if (ptgv == null) {
- ptgv = new TypeGraphVertex(immediateP, resName);
+ ptgv = new TypeGraphVertex(this, immediateP, resName);
typeGraph.put(immediateP, ptgv);
}
tgv.ancestors.add(ptgv);
ptgv.descendants.add(tgv);
}
- if (reader.getInterfaces() != null) {
- for (String iface : reader.getInterfaces()) {
+ if (fastClassIndexReader.getInterfaces() != null) {
+ for (String iface : fastClassIndexReader.getInterfaces()) {
iface = iface.replace('/', '.');
ptgv = typeGraph.get(iface);
if (ptgv == null) {
- ptgv = new TypeGraphVertex(iface, resName);
+ ptgv = new TypeGraphVertex(this, iface, resName);
typeGraph.put(iface, ptgv);
}
tgv.ancestors.add(ptgv);
@@ -304,75 +324,9 @@ public class TypeGraph
return addNode(jar.getInputStream(jarEntry), jar.getName());
}
- public void updatePortTypeInfoInTypeGraph(Map<String, JarFile> openJarFiles,
- Map<String, File> openClassFiles) {
- TypeGraphVertex tgv = typeGraph.get(Operator.class.getName());
- updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles, tgv);
- }
-
- public void updatePortTypeInfoInTypeGraph(Map<String, JarFile> openJarFiles,
- Map<String, File> openClassFiles, TypeGraphVertex tgv) {
- if (tgv == null)
- return;
-
- for (TypeGraphVertex operator : tgv.descendants) {
- try {
- String path = operator.getJarName();
- JarFile jar = openJarFiles.get(path);
- if (jar != null) {
- String jarEntryName = operator.getClassNode().getName()
- .replace('.', '/')
- + ".class";
- JarEntry jarEntry = jar.getJarEntry(jarEntryName);
- if (jarEntry != null) {
- updatePortInfo(operator, jar.getInputStream(jarEntry));
- }
- } else {
- File f = openClassFiles.get(path);
- if (f != null && f.exists() && f.getName().endsWith("class")) {
- updatePortInfo(operator, new FileInputStream(f));
- }
- }
- updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles, operator);
- } catch (Exception e) {
- DTThrowable.wrapIfChecked(e);
- }
- }
- }
-
- private void updatePortInfo(TypeGraphVertex tgv, InputStream input)
- throws IOException {
- try {
- ClassReader reader;
- reader = new ClassReader(input);
- ClassNodeType classN = new ClassNodeType();
- classN.setClassSignatureVisitor(tgv.getClassNode().getCsv());
- classN.setVisitFields(true);
- reader.accept(classN, ClassReader.SKIP_CODE);
- CompactClassNode ccn = tgv.getClassNode();
- CompactUtil.updateCompactClassPortInfo(classN, ccn);
- List<CompactFieldNode> prunedFields = new LinkedList<CompactFieldNode>();
- TypeGraphVertex portVertex = typeGraph.get(Operator.Port.class.getName());
- for (CompactFieldNode field : ccn.getPorts()) {
- TypeGraphVertex fieldVertex = typeGraph.get(field.getDescription());
- if(fieldVertex != null) {
- if (isAncestor(portVertex, fieldVertex)) {
- prunedFields.add(field);
- }
- }
- }
- ccn.setPorts(prunedFields);
-
- } finally {
- if (input != null) {
- input.close();
- }
- }
- }
-
private void updateInstantiableDescendants(TypeGraphVertex tgv)
{
- if(tgv.isInstantiable()){
+ if (tgv.isInstantiable()) {
tgv.allInstantiableDescendants.add(tgv);
}
for (TypeGraphVertex parent : tgv.ancestors) {
@@ -468,13 +422,12 @@ public class TypeGraph
{
String n1 = o1.typeName;
String n2 = o2.typeName;
- if(n1.startsWith("java")){
+ if (n1.startsWith("java")) {
n1 = "0" + n1;
}
- if(n2.startsWith("java")){
+ if (n2.startsWith("java")) {
n2 = "0" + n2;
}
-
return n1.compareTo(n2);
}
});
@@ -483,40 +436,59 @@ public class TypeGraph
private final transient Set<TypeGraphVertex> descendants = new HashSet<TypeGraphVertex>();
+ private transient TypeGraph owner;
+
// keep the jar file name for late fetching the detail information
private String jarName;
private boolean hasResource = false;
+ private boolean isRealNode = false;
+
+ private boolean isInstantiable = false;
+
+
@SuppressWarnings("unused")
- private TypeGraphVertex(){
+ private TypeGraphVertex()
+ {
jarName = "";
typeName = "";
}
- public TypeGraphVertex(String typeName, String jarName, CompactClassNode classNode)
+ public TypeGraphVertex(TypeGraph owner, String typeName, String jarName, boolean isRealNode, boolean isInstantiable)
{
-
- this.jarName = jarName;
this.typeName = typeName;
- this.classNode = classNode;
+ this.jarName = jarName;
+ this.isRealNode = isRealNode;
+ this.isInstantiable = isInstantiable;
+ this.owner = owner;
+ }
+
+ public TypeGraphVertex(TypeGraph owner, String typeName, String jarName)
+ {
+ this(owner, typeName, jarName, false, false);
+ }
+
+ public void setOwner(TypeGraph owner)
+ {
+ this.owner = owner;
+ }
+
+ public TypeGraph getOwner()
+ {
+ return owner;
}
public Set<TypeGraphVertex> getAncestors()
{
return ancestors;
}
+
public int numberOfInstantiableDescendants()
{
return allInstantiableDescendants.size() + (isInstantiable() ? 1 : 0);
}
- public TypeGraphVertex(String typeName, String jarName)
- {
- this.typeName = typeName;
- this.jarName = jarName;
- }
-
public boolean hasResource()
{
return hasResource;
@@ -529,33 +501,12 @@ public class TypeGraph
public boolean isInstantiable()
{
- return JACKSON_INSTANTIABLE_CLASSES.contains(this.typeName) || (isPublicConcrete() && classNode.getDefaultConstructor() != null);
+ return isInstantiable || JACKSON_INSTANTIABLE_CLASSES.contains(this.typeName);
}
- private boolean isPublicConcrete()
+ public void setIsInstantiable(boolean isInstantiable)
{
- if (classNode == null) {
- // If the class is not in the classpath
- return false;
- }
- int opCode = getOpCode();
-
- // if the class is neither abstract nor interface
- // and the class is public
- return ((opCode & (Opcodes.ACC_ABSTRACT | Opcodes.ACC_INTERFACE)) == 0) && ((opCode & Opcodes.ACC_PUBLIC) == Opcodes.ACC_PUBLIC);
- }
-
- private int getOpCode()
- {
- List<CompactClassNode> icl = classNode.getInnerClasses();
- if (typeName.contains("$")) {
- for (CompactClassNode innerClassNode : icl) {
- if (innerClassNode.getName().replace('/', '.').equals(typeName)) {
- return innerClassNode.getAccess();
- }
- }
- }
- return classNode.getAccess();
+ this.isInstantiable = isInstantiable;
}
/*
@@ -575,18 +526,23 @@ public class TypeGraph
@Override
public boolean equals(Object obj)
{
- if (this == obj)
+ if (this == obj) {
return true;
- if (obj == null)
+ }
+ if (obj == null) {
return false;
- if (getClass() != obj.getClass())
+ }
+ if (getClass() != obj.getClass()) {
return false;
- TypeGraphVertex other = (TypeGraphVertex) obj;
+ }
+ TypeGraphVertex other = (TypeGraphVertex)obj;
if (typeName == null) {
- if (other.typeName != null)
+ if (other.typeName != null) {
return false;
- } else if (!typeName.equals(other.typeName))
+ }
+ } else if (!typeName.equals(other.typeName)) {
return false;
+ }
return true;
}
@@ -600,15 +556,80 @@ public class TypeGraph
this.jarName = jarName;
}
- public CompactClassNode getClassNode()
+ /**
+ * The query on this vertex is possible to be called by multithread
+ * Thus make this method synchronized
+ * @return
+ */
+ public synchronized CompactClassNode getOrLoadClassNode()
{
+ if (classNode == null) {
+ //load the class first
+ try {
+ loadClass();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
return classNode;
}
-
- public void setClassNode(CompactClassNode classNode)
+
+ public void setIsRealNode(boolean isRealNode)
{
- this.classNode = classNode;
+ this.isRealNode = isRealNode;
}
+
+ public boolean isRealNode()
+ {
+ return isRealNode;
+ }
+
+ private void loadClass() throws IOException
+ {
+ if (classNode != null) {
+ return;
+ }
+ for (TypeGraphVertex ancestor : getAncestors()) {
+ ancestor.loadClass();
+ }
+ JarFile jarFile = null;
+ InputStream inputStream = null;
+ try {
+ if (jarName.endsWith(".jar")) {
+ JarFile jarfile = new JarFile(jarName);
+ inputStream = jarfile.getInputStream(jarfile.getEntry(typeName.replace('.', '/') + ".class"));
+ } else {
+ inputStream = new FileInputStream(jarName);
+ }
+ ClassReader reader = new ClassReader(inputStream);
+ ClassNode classN = new ClassNodeType();
+ reader.accept(classN, ClassReader.SKIP_CODE);
+ CompactClassNode ccn = CompactUtil.compactClassNode(classN);
+ this.classNode = ccn;
+
+ // update the port information if it is a Operator
+ if (owner.isAncestor(Operator.class.getName(), typeName)) {
+ // load ports if it is an Operator class
+ CompactUtil.updateCompactClassPortInfo(classN, ccn);
+ List<CompactFieldNode> prunedFields = new LinkedList<CompactFieldNode>();
+ TypeGraphVertex portVertex = owner.getTypeGraphVertex(Operator.Port.class.getName());
+ for (CompactFieldNode field : ccn.getPorts()) {
+ TypeGraphVertex fieldVertex = owner.getTypeGraphVertex(field.getDescription());
+ if (fieldVertex != null) {
+ if (isAncestor(portVertex, fieldVertex)) {
+ prunedFields.add(field);
+ }
+ }
+ }
+ ccn.setPorts(prunedFields);
+ }
+ } finally {
+ IOUtils.closeQuietly(jarFile);
+ IOUtils.closeQuietly(inputStream);
+ }
+
+ }
+
}
/**
@@ -621,7 +642,7 @@ public class TypeGraph
public List<String> getInstantiableDescendants(String clazz, String filter, String packagePrefix, String startsWith)
{
TypeGraphVertex tgv = typeGraph.get(clazz);
- if(tgv == null) {
+ if (tgv == null) {
return null;
}
@@ -636,7 +657,8 @@ public class TypeGraph
if (packagePrefix != null && !node.typeName.startsWith(packagePrefix)) {
continue;
}
- if (startsWith != null && !typeName.substring(typeName.lastIndexOf('.') + 1).toLowerCase().startsWith(startsWith.toLowerCase())){
+ if (startsWith != null && !typeName.substring(typeName.lastIndexOf('.') + 1).toLowerCase()
+ .startsWith(startsWith.toLowerCase())) {
continue;
}
result.add(node.typeName);
@@ -653,7 +675,7 @@ public class TypeGraph
if (tgv == null) {
return desc;
}
- CompactClassNode cn = tgv.classNode;
+ CompactClassNode cn = tgv.getOrLoadClassNode();
if (cn.isEnum()) {
List<String> enumNames = cn.getEnumValues();
@@ -667,7 +689,7 @@ public class TypeGraph
addClassPropertiesAndPorts(clazzName, desc);
- if(tgv.hasResource()){
+ if (tgv.hasResource()) {
desc.put("hasResource", "true");
} else {
desc.put("hasResource", "false");
@@ -677,7 +699,8 @@ public class TypeGraph
}
private Collection<JSONObject> getPortTypeInfo(String clazzName,
- Map<Type, Type> typeReplacement, List<CompactFieldNode> ports) throws JSONException {
+ Map<Type, Type> typeReplacement, List<CompactFieldNode> ports) throws JSONException
+ {
TypeGraphVertex tgv = typeGraph.get(clazzName);
if (tgv == null) {
return null;
@@ -686,43 +709,45 @@ public class TypeGraph
Collection<JSONObject> portInfo = new ArrayList<JSONObject>();
for (CompactFieldNode port : ports) {
- Type fieldType = port.getFieldSignatureNode().getFieldType();
- Type t = fieldType;
- if (fieldType instanceof ParameterizedTypeNode) {
- // TODO: Right now getPortInfo assumes a single parameterized type
- t = ((ParameterizedTypeNode) fieldType).getActualTypeArguments()[0];
- } else {
- // TODO: Check behavior for Ports not using Default Input/output ports
- TypeGraphVertex portVertex = typeGraph.get(port.getDescription());
- t = findTypeArgument(portVertex, typeReplacement);
- LOG.debug("Field is of type {}", fieldType.getClass());
- }
+ Type fieldType = port.getFieldSignatureNode().getFieldType();
+ Type t = fieldType;
+ if (fieldType instanceof ParameterizedTypeNode) {
+ // TODO: Right now getPortInfo assumes a single parameterized type
+ t = ((ParameterizedTypeNode)fieldType).getActualTypeArguments()[0];
+ } else {
+ // TODO: Check behavior for Ports not using Default Input/output ports
+ TypeGraphVertex portVertex = typeGraph.get(port.getDescription());
+ t = findTypeArgument(portVertex, typeReplacement);
+ LOG.debug("Field is of type {}", fieldType.getClass());
+ }
- JSONObject meta = new JSONObject();
- try {
- meta.put("name", port.getName());
- setTypes(meta, t, typeReplacement);
- portInfo.add(meta);
- } catch (Exception e) {
- DTThrowable.wrapIfChecked(e);
- }
+ JSONObject meta = new JSONObject();
+ try {
+ meta.put("name", port.getName());
+ setTypes(meta, t, typeReplacement);
+ portInfo.add(meta);
+ } catch (Exception e) {
+ DTThrowable.wrapIfChecked(e);
+ }
}
return portInfo;
}
- public static Type getParameterizedTypeArgument(Type type) {
+ public static Type getParameterizedTypeArgument(Type type)
+ {
if (type instanceof ParameterizedTypeNode) {
- return ((ParameterizedTypeNode) type).getActualTypeArguments()[0];
+ return ((ParameterizedTypeNode)type).getActualTypeArguments()[0];
}
return null;
}
- private Type findTypeArgument(TypeGraphVertex tgv,
- Map<Type, Type> typeReplacement) {
- if (tgv == null)
+ private Type findTypeArgument(TypeGraphVertex tgv, Map<Type, Type> typeReplacement)
+ {
+ if (tgv == null) {
return null;
- ClassSignatureVisitor csv = tgv.getClassNode().getCsv();
+ }
+ ClassSignatureVisitor csv = tgv.getOrLoadClassNode().getCsv();
Type superC = csv.getSuperClass();
addReplacement(superC, typeReplacement);
@@ -757,14 +782,16 @@ public class TypeGraph
public List<CompactFieldNode> getAllInputPorts(TypeGraphVertex tgv)
{
List<CompactFieldNode> ports = new ArrayList<CompactFieldNode>();
- if (tgv == null)
+ if (tgv == null) {
return ports;
- TypeGraphVertex portVertex = typeGraph.get(Operator.InputPort.class
- .getName());
+ }
+ TypeGraphVertex portVertex = typeGraph.get(Operator.InputPort.class.getName());
getAllPortsWithAncestor(portVertex, tgv, ports);
- Collections.sort(ports, new Comparator<CompactFieldNode>() {
+ Collections.sort(ports, new Comparator<CompactFieldNode>()
+ {
@Override
- public int compare(CompactFieldNode a, CompactFieldNode b) {
+ public int compare(CompactFieldNode a, CompactFieldNode b)
+ {
return a.getName().compareTo(b.getName());
}
});
@@ -791,7 +818,7 @@ public class TypeGraph
private void getAllPortsWithAncestor(TypeGraphVertex portVertex, TypeGraphVertex tgv, List<CompactFieldNode> ports)
{
- List<CompactFieldNode> fields = tgv.getClassNode().getPorts();
+ List<CompactFieldNode> fields = tgv.getOrLoadClassNode().getPorts();
if (fields != null) {
for (CompactFieldNode field : fields) {
TypeGraphVertex fieldVertex = typeGraph.get(field.getDescription());
@@ -889,8 +916,8 @@ public class TypeGraph
CompactClassNode exClass = null;
// check if the class needs to be excluded
for (String e : EXCLUDE_CLASSES) {
- if(e.equals(tgv.getClassNode().getName())) {
- exClass = tgv.getClassNode();
+ if (e.equals(tgv.getOrLoadClassNode().getName())) {
+ exClass = tgv.getOrLoadClassNode();
break;
}
}
@@ -914,17 +941,17 @@ public class TypeGraph
}
}
} else {
- if (tgv.getClassNode().getSetterMethods() != null) {
- setters.addAll(tgv.getClassNode().getSetterMethods());
+ if (tgv.getOrLoadClassNode().getSetterMethods() != null) {
+ setters.addAll(tgv.getOrLoadClassNode().getSetterMethods());
}
- if (tgv.getClassNode().getGetterMethods() != null) {
- getters.addAll(tgv.getClassNode().getGetterMethods());
+ if (tgv.getOrLoadClassNode().getGetterMethods() != null) {
+ getters.addAll(tgv.getOrLoadClassNode().getGetterMethods());
}
}
TypeGraphVertex portVertex = typeGraph.get(Operator.Port.class.getName());
- List<CompactFieldNode> fields = tgv.getClassNode().getPorts();
- if(fields != null) {
+ List<CompactFieldNode> fields = tgv.getOrLoadClassNode().getPorts();
+ if (fields != null) {
for (CompactFieldNode field : fields) {
TypeGraphVertex fieldVertex = typeGraph.get(field.getDescription());
if (isAncestor(portVertex, fieldVertex)) {
@@ -933,15 +960,15 @@ public class TypeGraph
}
}
- ClassSignatureVisitor csv = tgv.getClassNode().getCsv();
+ ClassSignatureVisitor csv = tgv.getOrLoadClassNode().getCsv();
Type superC = csv.getSuperClass();
addReplacement(superC, typeReplacement);
- if(csv.getInterfaces()!=null){
- for(Type it : csv.getInterfaces()){
+ if (csv.getInterfaces() != null) {
+ for (Type it : csv.getInterfaces()) {
addReplacement(it, typeReplacement);
- };
+ }
}
for (TypeGraphVertex ancestor : tgv.ancestors) {
getPublicSetterGetterAndPorts(ancestor, setters, getters, typeReplacement, ports);
@@ -950,9 +977,9 @@ public class TypeGraph
private void addReplacement(Type superT, Map<Type, Type> typeReplacement)
{
- if(superT!=null && superT instanceof ParameterizedTypeNode){
+ if (superT != null && superT instanceof ParameterizedTypeNode) {
Type[] actualTypes = ((ParameterizedTypeNode)superT).getActualTypeArguments();
- List<TypeVariableNode> tvs = typeGraph.get(((ParameterizedTypeNode)superT).getTypeObj().getClassName()).getClassNode().getCsv().getTypeV();
+ List<TypeVariableNode> tvs = typeGraph.get(((ParameterizedTypeNode)superT).getTypeObj().getClassName()).getOrLoadClassNode().getCsv().getTypeV();
int i = 0;
for (TypeVariableNode typeVariableNode : tvs) {
typeReplacement.put(typeVariableNode, actualTypes[i++]);
@@ -978,7 +1005,7 @@ public class TypeGraph
if (t instanceof WildcardTypeNode) {
propJ.put("type", "?");
} else if (t instanceof TypeNode) {
- TypeNode tn = (TypeNode) t;
+ TypeNode tn = (TypeNode)t;
String typeS = tn.getTypeObj().getClassName();
propJ.put("type", typeS);
UI_TYPE uiType = UI_TYPE.getEnumFor(typeS, typeGraph);
@@ -1001,7 +1028,7 @@ public class TypeGraph
}
if (t instanceof ParameterizedTypeNode) {
JSONArray jArray = new JSONArray();
- for (Type ttn : ((ParameterizedTypeNode) t).getActualTypeArguments()) {
+ for (Type ttn : ((ParameterizedTypeNode)t).getActualTypeArguments()) {
JSONObject objJ = new JSONObject();
if (!stopRecursive) {
setTypes(objJ, ttn, typeReplacement, visitedType);
@@ -1016,18 +1043,18 @@ public class TypeGraph
JSONArray jArray = new JSONArray();
- Type[] bounds = ((WildcardTypeNode) t).getUpperBounds();
- if(bounds!=null){
+ Type[] bounds = ((WildcardTypeNode)t).getUpperBounds();
+ if (bounds != null) {
for (Type type : bounds) {
jArray.put(type.toString());
}
}
typeBounds.put("upper", jArray);
- bounds = ((WildcardTypeNode) t).getLowerBounds();
+ bounds = ((WildcardTypeNode)t).getLowerBounds();
jArray = new JSONArray();
- if(bounds!=null){
+ if (bounds != null) {
for (Type type : bounds) {
jArray.put(type.toString());
}
@@ -1037,21 +1064,21 @@ public class TypeGraph
propJ.put("typeBounds", typeBounds);
}
- if(t instanceof ArrayTypeNode){
+ if (t instanceof ArrayTypeNode) {
propJ.put("type", t.getByteString());
propJ.put("uiType", UI_TYPE.LIST.getName());
JSONObject jObj = new JSONObject();
if (!stopRecursive) {
- setTypes(jObj, ((ArrayTypeNode) t).getActualArrayType(), typeReplacement, visitedType);
+ setTypes(jObj, ((ArrayTypeNode)t).getActualArrayType(), typeReplacement, visitedType);
}
propJ.put("itemType", jObj);
}
- if(t instanceof TypeVariableNode){
+ if (t instanceof TypeVariableNode) {
propJ.put("typeLiteral", ((TypeVariableNode)t).getTypeLiteral());
if (!stopRecursive) {
- setTypes(propJ, ((TypeVariableNode) t).getRawTypeBound(), typeReplacement, visitedType);
+ setTypes(propJ, ((TypeVariableNode)t).getRawTypeBound(), typeReplacement, visitedType);
}
}
@@ -1067,7 +1094,7 @@ public class TypeGraph
{
List<TypeGraphVertex> invalidVertexes = new LinkedList<>();
for (TypeGraphVertex tgv : typeGraph.values()) {
- if (tgv.getClassNode() == null) {
+ if (!tgv.isRealNode()) {
invalidVertexes.add(tgv);
}
}
@@ -1126,7 +1153,7 @@ public class TypeGraph
private Type resolveType(Type t, Map<Type, Type> typeReplacement)
{
- if(typeReplacement.containsKey(t)){
+ if (typeReplacement.containsKey(t)) {
return resolveType(typeReplacement.get(t), typeReplacement);
} else {
return t;
@@ -1203,6 +1230,7 @@ public class TypeGraph
TypeGraph result = new TypeGraph();
for (TypeGraphVertex typeGraphVertex : tgv) {
result.typeGraph.put(typeGraphVertex.typeName, typeGraphVertex);
+ typeGraphVertex.setOwner(result);
}
return result;
}
@@ -1217,7 +1245,7 @@ public class TypeGraph
public List<String> getParents(String className)
{
TypeGraphVertex tgv = typeGraph.get(className);
- if(tgv == null || tgv.ancestors == null){
+ if (tgv == null || tgv.ancestors == null) {
return null;
}
List<String> result = new LinkedList<String>();
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java
index 3be0184..d7cf087 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraphFactory.java
@@ -27,14 +27,15 @@ import java.util.Set;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
-import org.apache.commons.io.IOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.commons.io.IOUtils;
-import com.datatorrent.stram.webapp.TypeGraph.TypeGraphSerializer;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
+import com.datatorrent.stram.webapp.TypeGraph.TypeGraphSerializer;
+
/**
* This class keeps a precomputed index(prototype) of type graph of all classes in jdk and gateway classpath
@@ -43,7 +44,7 @@ import com.esotericsoftware.kryo.io.Output;
*/
public class TypeGraphFactory
{
- private final static byte[] preComputeGraph;
+ private static final byte[] preComputeGraph;
private static final Logger LOG = LoggerFactory.getLogger(TypeGraphFactory.class);
@@ -72,7 +73,7 @@ public class TypeGraphFactory
for (String path : pathsToScan) {
try {
File f = new File(path);
- if(!f.exists() || !f.getName().endsWith("jar")){
+ if (!f.exists() || !f.getName().endsWith("jar")) {
continue;
}
JarFile jar = new JarFile(path);
@@ -101,11 +102,12 @@ public class TypeGraphFactory
kryo.writeObject(output, tg);
output.close();
preComputeGraph = baos.toByteArray();
- LOG.warn("The size of precomputed type graph is {} KB", preComputeGraph.length/1024);
+ LOG.warn("The size of precomputed type graph is {} KB", preComputeGraph.length / 1024);
}
- public static TypeGraph createTypeGraphProtoType(){
+ public static TypeGraph createTypeGraphProtoType()
+ {
Input input = null;
try {
input = new Input(new ByteArrayInputStream(preComputeGraph));
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java
index e827042..c9b8f15 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ASMUtil.java
@@ -153,6 +153,11 @@ public class ASMUtil
return (opCode & Opcodes.ACC_PUBLIC) == Opcodes.ACC_PUBLIC;
}
+ public static boolean isAbstract(int opCode)
+ {
+ return (opCode & Opcodes.ACC_ABSTRACT) == Opcodes.ACC_ABSTRACT;
+ }
+
public static boolean isTransient(int opCode)
{
return (opCode & Opcodes.ACC_TRANSIENT) == Opcodes.ACC_TRANSIENT;
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java
index 19e00ac..80f475f 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/ClassNodeType.java
@@ -39,56 +39,40 @@ public class ClassNodeType extends ClassNode
}
ClassSignatureVisitor csv = new ClassSignatureVisitor();
- private boolean visitFields = false;
@SuppressWarnings("unchecked")
@Override
public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions)
{
- if(!isVisitFields()) {
MethodNode mn = new MethodNode(access, name, desc, signature, exceptions);
mn.typeVariableSignatureNode = csv;
methods.add(mn);
return mn;
- }
- return null;
}
@SuppressWarnings("unchecked")
@Override
public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) {
- if(isVisitFields()) {
- FieldNode fn = new FieldNode(access, name, desc, signature, value);
- fn.typeVariableSignatureNode = csv;
- fields.add(fn);
- return fn;
- }
- return null;
+ FieldNode fn = new FieldNode(access, name, desc, signature, value);
+ fn.typeVariableSignatureNode = csv;
+ fields.add(fn);
+ return fn;
}
@Override
public void visit(int version, int access, String name, String signature, String superName, String[] interfaces)
{
- if(!isVisitFields()) {
- // parse the signature first so Type variable can be captured from the signature
- if(signature!=null){
- SignatureReader sr = new SignatureReader(signature);
- sr.accept(csv);
- }
- super.visit(version, access, name, signature, superName, interfaces);
+ // parse the signature first so Type variable can be captured from the signature
+ if (signature != null) {
+ SignatureReader sr = new SignatureReader(signature);
+ sr.accept(csv);
}
+ super.visit(version, access, name, signature, superName, interfaces);
}
public void setClassSignatureVisitor(ClassSignatureVisitor csv){
this.csv = csv;
}
- public boolean isVisitFields() {
- return visitFields;
- }
-
- public void setVisitFields(boolean visitFields) {
- this.visitFields = visitFields;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java
index 0b55578..52ab342 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/CompactUtil.java
@@ -40,7 +40,7 @@ public class CompactUtil
public static CompactClassNode compactClassNode(ClassNode cn)
{
- if(cn == null){
+ if (cn == null) {
return null;
}
CompactClassNode ccn = new CompactClassNode();
@@ -69,11 +69,11 @@ public class CompactUtil
inner.setAccess(((InnerClassNode)icn).access);
}
ccn.setInnerClasses(ccns);
- if(ASMUtil.isEnum(cn)){
+ if (ASMUtil.isEnum(cn)) {
ccn.setEnumValues(ASMUtil.getEnumValues(cn));
}
- if(cn instanceof ClassNodeType){
+ if (cn instanceof ClassNodeType){
ccn.setCsv(((ClassNodeType)cn).csv);
}
@@ -97,8 +97,7 @@ public class CompactUtil
{
List<FieldNode> fields = ASMUtil.getPorts(cn);
List<CompactFieldNode> ports = new LinkedList<CompactFieldNode>();
- for(FieldNode fn : fields)
- {
+ for (FieldNode fn : fields) {
ports.add(compactFieldNode(fn));
}
ccn.setPorts(ports);
@@ -124,12 +123,10 @@ public class CompactUtil
cfn.setName(fn.name);
String className = org.apache.xbean.asm5.Type.getObjectType(fn.desc).getClassName();
- if(className.charAt(0) == 'L')
- {
+ if (className.charAt(0) == 'L') {
className = className.substring(1);
}
- if(className.endsWith(";"))
- {
+ if (className.endsWith(";")) {
className = className.substring(0, className.length() - 1);
}
cfn.setDescription(className);
@@ -138,8 +135,9 @@ public class CompactUtil
if (fn.visibleAnnotations != null) {
setAnnotationNode(fn, cfn);
}
- if(fn instanceof com.datatorrent.stram.webapp.asm.FieldNode)
+ if (fn instanceof com.datatorrent.stram.webapp.asm.FieldNode) {
cfn.setFieldSignatureNode((((com.datatorrent.stram.webapp.asm.FieldNode)fn).signatureNode));
+ }
return cfn;
}
@@ -149,7 +147,7 @@ public class CompactUtil
CompactAnnotationNode node = new CompactAnnotationNode();
Map<String, Object> annotationMap = new HashMap<String, Object>();
if (visibleAnnotation instanceof AnnotationNode) {
- AnnotationNode annotation = (AnnotationNode) visibleAnnotation;
+ AnnotationNode annotation = (AnnotationNode)visibleAnnotation;
if (annotation.desc.contains("InputPortFieldAnnotation")
|| annotation.desc.contains("OutputPortFieldAnnotation")) {
List<Object> annotationValues = annotation.values;
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java b/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java
new file mode 100644
index 0000000..fc689fc
--- /dev/null
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/asm/FastClassIndexReader.java
@@ -0,0 +1,340 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.webapp.asm;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+
+import org.apache.xbean.asm5.Opcodes;
+
+/**
+ *
+ * An improvement for ASM class reader.
+ * ASM class reader reads the whole class into structured data like ClassNode MethodNode
+ * But to build a type graph we only need to know class name, parent class name, interfaces name and
+ * whether it has a public non-arg default constructor (instantiable)
+ * This class skip most parts that are not necessary and parse only the information above
+ *
+ * And also it use shared buffer to load classes which makes it faster in our performance test
+ *
+ * Overall it is 8-9x faster than ASM class reader
+ *
+ * Keep in mind it is NOT thread safe
+ *
+ */
+public class FastClassIndexReader
+{
+
+ /**
+ * The type of CONSTANT_Class constant pool items.
+ */
+ static final int CLASS = 7;
+
+ /**
+ * The type of CONSTANT_Fieldref constant pool items.
+ */
+ static final int FIELD = 9;
+
+ /**
+ * The type of CONSTANT_Methodref constant pool items.
+ */
+ static final int METH = 10;
+
+ /**
+ * The type of CONSTANT_InterfaceMethodref constant pool items.
+ */
+ static final int IMETH = 11;
+
+ /**
+ * The type of CONSTANT_String constant pool items.
+ */
+ static final int STR = 8;
+
+ /**
+ * The type of CONSTANT_Integer constant pool items.
+ */
+ static final int INT = 3;
+
+ /**
+ * The type of CONSTANT_Float constant pool items.
+ */
+ static final int FLOAT = 4;
+
+ /**
+ * The type of CONSTANT_Long constant pool items.
+ */
+ static final int LONG = 5;
+
+ /**
+ * The type of CONSTANT_Double constant pool items.
+ */
+ static final int DOUBLE = 6;
+
+ /**
+ * The type of CONSTANT_NameAndType constant pool items.
+ */
+ static final int NAME_TYPE = 12;
+
+ /**
+ * The type of CONSTANT_Utf8 constant pool items.
+ */
+ static final int UTF8 = 1;
+
+ /**
+ * The type of CONSTANT_MethodType constant pool items.
+ */
+ static final int MTYPE = 16;
+
+ /**
+ * The type of CONSTANT_MethodHandle constant pool items.
+ */
+ static final int HANDLE = 15;
+
+ /**
+ * The type of CONSTANT_InvokeDynamic constant pool items.
+ */
+ static final int INDY = 18;
+
+ // shared buffer to hold the content of the file
+ private static byte[] b = new byte[64 * 1024];
+
+ private static int bSize = 0;
+
+ private int[] items;
+
+ private int header;
+
+ private String name;
+
+ private String superName;
+
+ private String[] interfaces;
+
+ // use this for byte array comparison which is faster than string comparison
+ public static final byte[] DEFAULT_CONSTRUCTOR_NAME = new byte[]{0, 6, '<', 'i', 'n', 'i', 't', '>'};
+
+ public static final byte[] DEFAULT_CONSTRUCTOR_DESC = new byte[]{0, 3, '(', ')', 'V'};
+
+ private boolean isInstantiable = false;
+
+ public FastClassIndexReader(final InputStream is) throws IOException
+ {
+ readIntoBuffer(is);
+
+ readConstantPool();
+
+ readIndex();
+ }
+
+ /**
+ * Read class file content into shared buffer from input stream
+ * Stream won't be closed
+ * @param is
+ * @throws IOException
+ */
+ private void readIntoBuffer(InputStream is) throws IOException
+ {
+ if (is == null) {
+ throw new IOException("Class not found");
+ }
+ bSize = 0;
+ while (true) {
+ int n = is.read(b, bSize, b.length - bSize);
+ if (n == -1) {
+ break;
+ }
+ bSize += n;
+ if (bSize >= b.length) {
+ byte[] c = new byte[b.length << 2];
+ System.arraycopy(b, 0, c, 0, b.length);
+ b = c;
+ }
+ }
+ }
+
+ /**
+ * read and index the constant pool section for getting class metadata later
+ */
+ private void readConstantPool()
+ {
+ // checks the class version
+ if (readShort(6) > Opcodes.V1_8) {
+ throw new IllegalArgumentException();
+ }
+ // parses the constant pool
+ items = new int[readUnsignedShort(8)];
+ int n = items.length;
+ int index = 10;
+ for (int i = 1; i < n; ++i) {
+ items[i] = index + 1;
+ int size;
+ switch (b[index]) {
+ case FIELD:
+ case METH:
+ case IMETH:
+ case INT:
+ case FLOAT:
+ case NAME_TYPE:
+ case INDY:
+ size = 5;
+ break;
+ case LONG:
+ case DOUBLE:
+ size = 9;
+ ++i;
+ break;
+ case UTF8:
+ size = 3 + readUnsignedShort(index + 1);
+ break;
+ case HANDLE:
+ size = 4;
+ break;
+ // case ClassWriter.CLASS:
+ // case ClassWriter.STR:
+ // case ClassWriter.MTYPE
+ default:
+ size = 3;
+ break;
+ }
+ index += size;
+ }
+ // the class header information starts just after the constant pool
+ header = index;
+ }
+
+ /**
+ * read class metadata, class name, parent name, interfaces name, is instantiable(has public non-arg constructor)
+ * or not
+ * @throws UnsupportedEncodingException
+ */
+ private void readIndex() throws UnsupportedEncodingException
+ {
+ // reads the class declaration
+ int u = header;
+
+ int access = readUnsignedShort(u);
+ isInstantiable = ASMUtil.isPublic(access) && !ASMUtil.isAbstract(access);
+
+ name = readClass(u + 2);
+ superName = readClass(u + 4);
+ interfaces = new String[readUnsignedShort(u + 6)];
+ u += 8;
+ for (int i = 0; i < interfaces.length; ++i) {
+ interfaces[i] = readClass(u);
+ u += 2;
+ }
+
+ if (!isInstantiable) {
+ return;
+ }
+
+ // reads the constructor
+
+ // skip fields
+ for (int i = readUnsignedShort(u); i > 0; --i) {
+ for (int j = readUnsignedShort(u + 8); j > 0; --j) {
+ u += 6 + readInt(u + 12);
+ }
+ u += 8;
+ }
+ u += 2;
+ for (int i = readUnsignedShort(u); i > 0; --i) {
+ if (isDefaultConstructor(u + 2)) {
+ return;
+ }
+ for (int j = readUnsignedShort(u + 8); j > 0; --j) {
+ u += 6 + readInt(u + 12);
+ }
+ u += 8;
+ }
+ isInstantiable = false;
+ }
+
+ private boolean isDefaultConstructor(int methodIndex) throws UnsupportedEncodingException
+ {
+ return arrayContains(b, items[readUnsignedShort(methodIndex + 2)], DEFAULT_CONSTRUCTOR_NAME)
+ && arrayContains(b, items[readUnsignedShort(methodIndex + 4)], DEFAULT_CONSTRUCTOR_DESC)
+ && ASMUtil.isPublic(readUnsignedShort(methodIndex));
+ }
+
+ private boolean arrayContains(byte[] bb, int i, byte[] subArray)
+ {
+ for (int l = 0; l < subArray.length; l++) {
+ if (bb[i + l] != subArray[l]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public int readUnsignedShort(final int index)
+ {
+ return ((b[index] & 0xFF) << 8) | (b[index + 1] & 0xFF);
+ }
+
+ public short readShort(final int index)
+ {
+ return (short)(((b[index] & 0xFF) << 8) | (b[index + 1] & 0xFF));
+ }
+
+ public int readInt(final int index)
+ {
+ return ((b[index] & 0xFF) << 24) | ((b[index + 1] & 0xFF) << 16)
+ | ((b[index + 2] & 0xFF) << 8) | (b[index + 3] & 0xFF);
+ }
+
+ public String readClass(final int index) throws UnsupportedEncodingException
+ {
+ // computes the start index of the CONSTANT_Class item in b
+ // and reads the CONSTANT_Utf8 item designated by
+ // the first two bytes of this CONSTANT_Class item
+ return readUTF8(items[readUnsignedShort(index)]);
+ }
+
+ public String readUTF8(int index) throws UnsupportedEncodingException
+ {
+ int item = readUnsignedShort(index);
+ if (index == 0 || item == 0) {
+ return null;
+ }
+ index = items[item];
+ return new String(b, index + 2, readUnsignedShort(index), "UTF-8");
+ }
+
+ public String getName()
+ {
+ return name;
+ }
+
+ public String getSuperName()
+ {
+ return superName;
+ }
+
+ public String[] getInterfaces()
+ {
+ return interfaces;
+ }
+
+ public boolean isInstantiable()
+ {
+ return isInstantiable;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
index 5053c1a..7c1be8f 100644
--- a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
@@ -1138,5 +1138,6 @@ public class OperatorDiscoveryTest
Assert.assertEquals("@description", OperatorDiscoverer.MethodTagType.DESCRIPTION, OperatorDiscoverer.MethodTagType.from("@description"));
Assert.assertEquals("@random", null, OperatorDiscoverer.MethodTagType.from("@random"));
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java b/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java
new file mode 100644
index 0000000..41d87bc
--- /dev/null
+++ b/engine/src/test/java/com/datatorrent/stram/webapp/asm/FastClassIndexReaderTest.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.datatorrent.stram.webapp.asm;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.xbean.asm5.ClassReader;
+
+public class FastClassIndexReaderTest
+{
+ private static final Logger LOG = LoggerFactory.getLogger(FastClassIndexReaderTest.class);
+
+ @Test
+ public void testIndexReader() throws IOException
+ {
+ String javahome = System.getProperty("java.home");
+ String jdkJar = javahome + "/lib/rt.jar";
+ JarFile jar = new JarFile(jdkJar);
+ java.util.Enumeration<JarEntry> entriesEnum = jar.entries();
+ while (entriesEnum.hasMoreElements()) {
+ JarEntry jarEntry = entriesEnum.nextElement();
+ if (jarEntry.getName().endsWith("class")) {
+ InputStream ins = jar.getInputStream(jarEntry);
+ ClassReader classReader = new ClassReader(ins);
+ ClassNodeType classN = new ClassNodeType();
+ classReader.accept(classN, ClassReader.SKIP_CODE);
+ CompactClassNode ccn = CompactUtil.compactClassNode(classN);
+ ins.close();
+
+ ins = jar.getInputStream(jarEntry);
+ FastClassIndexReader fastClassIndexReader = new FastClassIndexReader(ins);
+ Assert.assertEquals("The wrong class is " + classN.name, classN.name, fastClassIndexReader.getName());
+ Assert.assertEquals("The wrong class is " + classN.name, classN.superName, fastClassIndexReader.getSuperName());
+ Assert.assertEquals("The wrong class is " + classN.name,
+ !ASMUtil.isAbstract(classN.access) && ASMUtil.isPublic(classN.access)
+ && ccn.getDefaultConstructor() != null, fastClassIndexReader.isInstantiable());
+ Assert.assertArrayEquals("The wrong class is " + classN.name,
+ classN.interfaces.toArray(), fastClassIndexReader.getInterfaces());
+
+ }
+ }
+ }
+
+ @Test
+ public void testPerformance() throws Exception
+ {
+
+ String javahome = System.getProperty("java.home");
+ String jdkJar = javahome + "/lib/rt.jar";
+ JarFile jar = new JarFile(jdkJar);
+ java.util.Enumeration<JarEntry> entriesEnum = jar.entries();
+ long time = System.currentTimeMillis();
+ while (entriesEnum.hasMoreElements()) {
+ JarEntry jarEntry = entriesEnum.nextElement();
+ if (jarEntry.getName().endsWith("class")) {
+ InputStream ins = jar.getInputStream(jarEntry);
+ //FastClassSignatureReader fastClassSignatureReader = new FastClassSignatureReader(ins);
+ ClassReader classReader = new ClassReader(ins);
+ ClassNodeType classN = new ClassNodeType();
+ classReader.accept(classN, ClassReader.SKIP_CODE);
+ CompactClassNode ccn = CompactUtil.compactClassNode(classN);
+ ins.close();
+ }
+ }
+
+ LOG.info("The time to scan jdk using ASM ClassReader {} ", System.currentTimeMillis() - time);
+
+ jar.close();
+
+ jar = new JarFile("/usr/lib/jvm/java-7-oracle/jre/lib/rt.jar");
+ entriesEnum = jar.entries();
+ time = System.currentTimeMillis();
+ while (entriesEnum.hasMoreElements()) {
+ JarEntry jarEntry = entriesEnum.nextElement();
+ if (jarEntry.getName().endsWith("class")) {
+ InputStream ins = jar.getInputStream(jarEntry);
+ FastClassIndexReader fastClassIndexReader = new FastClassIndexReader(ins);
+ ins.close();
+ }
+ }
+
+ LOG.info("The time to scan jdk using FastClassIndexReader {} ", System.currentTimeMillis() - time);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/0631c9b9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 89abc04..500471a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,6 +98,7 @@
<email>dev@apex.incubator.apache.org</email>
</properties>
<excludes>
+ <exclude>**/*.iml</exclude>
<exclude>DISCLAIMER</exclude>
<exclude>LICENSE</exclude>
<exclude>NOTICE</exclude>
[2/2] incubator-apex-core git commit: Merge branch 'APEX-188' into
devel-3
Posted by is...@apache.org.
Merge branch 'APEX-188' into devel-3
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/71ebf172
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/71ebf172
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/71ebf172
Branch: refs/heads/devel-3
Commit: 71ebf1724323c91103c6947205031ed21277e370
Parents: f35522b 0631c9b
Author: ishark <is...@datatorrent.com>
Authored: Wed Nov 11 12:34:43 2015 -0800
Committer: ishark <is...@datatorrent.com>
Committed: Wed Nov 11 12:34:43 2015 -0800
----------------------------------------------------------------------
.../java/com/datatorrent/api/AutoMetric.java | 2 +-
.../main/java/com/datatorrent/api/Operator.java | 2 +-
checkstyle-suppressions.xml | 28 ++
engine/pom.xml | 2 +-
.../stram/engine/StreamingContainer.java | 2 +-
.../stram/webapp/LogicalOperatorInfo.java | 1 -
.../stram/webapp/OperatorDiscoverer.java | 1 -
.../com/datatorrent/stram/webapp/TypeGraph.java | 460 ++++++++++---------
.../stram/webapp/TypeGraphFactory.java | 14 +-
.../datatorrent/stram/webapp/asm/ASMUtil.java | 5 +
.../stram/webapp/asm/ClassNodeType.java | 34 +-
.../stram/webapp/asm/CompactUtil.java | 20 +-
.../stram/webapp/asm/FastClassIndexReader.java | 340 ++++++++++++++
.../stram/webapp/OperatorDiscoveryTest.java | 1 +
.../webapp/asm/FastClassIndexReaderTest.java | 109 +++++
pom.xml | 3 +
16 files changed, 760 insertions(+), 264 deletions(-)
----------------------------------------------------------------------