You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by th...@apache.org on 2015/09/01 07:59:12 UTC
[1/2] incubator-apex-core git commit: APEX-70 #resolve #comment add
hasResource label to types in typegraph
Repository: incubator-apex-core
Updated Branches:
refs/heads/devel-3 d748ed46f -> f2a40717b
APEX-70 #resolve #comment add hasResource label to types in typegraph
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/00b486f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/00b486f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/00b486f0
Branch: refs/heads/devel-3
Commit: 00b486f0040294870df3d42a88b6fd0ce143d520
Parents: f28d8d7
Author: siyuan <si...@datatorrent.com>
Authored: Mon Aug 31 17:25:22 2015 -0700
Committer: siyuan <si...@datatorrent.com>
Committed: Mon Aug 31 17:25:22 2015 -0700
----------------------------------------------------------------------
.../stram/webapp/OperatorDiscoverer.java | 44 ++++++++++++++++----
.../com/datatorrent/stram/webapp/TypeGraph.java | 39 +++++++++++++----
.../stram/webapp/OperatorDiscoveryTest.java | 21 ++++++++++
.../example/mydtapp/StdoutOperator/ExtRes.html | 1 +
4 files changed, 90 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
index b35efe6..f08a96b 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/OperatorDiscoverer.java
@@ -25,6 +25,7 @@ import com.datatorrent.stram.webapp.asm.CompactFieldNode;
import com.google.common.base.Predicate;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -277,6 +278,8 @@ public class OperatorDiscoverer
{
Map<String, JarFile> openJarFiles = new HashMap<String, JarFile>();
Map<String, File> openClassFiles = new HashMap<String, File>();
+ // use global cache to load resource in/out of the same jar as the classes
+ Set<String> resourceCacheSet = new HashSet<>();
try {
for (String path : pathsToScan) {
File f = null;
@@ -296,16 +299,44 @@ public class OperatorDiscoverer
openJarFiles.put(path, jar);
java.util.Enumeration<JarEntry> entriesEnum = jar.entries();
while (entriesEnum.hasMoreElements()) {
- java.util.jar.JarEntry jarEntry = entriesEnum.nextElement();
- if (!jarEntry.isDirectory() && jarEntry.getName().endsWith("-javadoc.xml")) {
+ final java.util.jar.JarEntry jarEntry = entriesEnum.nextElement();
+ String entryName = jarEntry.getName();
+ if (jarEntry.isDirectory()) {
+ continue;
+ }
+ if (entryName.endsWith("-javadoc.xml")) {
try {
processJavadocXml(jar.getInputStream(jarEntry));
// break;
} catch (Exception ex) {
- LOG.warn("Cannot process javadoc {} : ", jarEntry.getName(), ex);
+ LOG.warn("Cannot process javadoc {} : ", entryName, ex);
+ }
+ } else if (entryName.endsWith(".class")) {
+ TypeGraph.TypeGraphVertex newNode = typeGraph.addNode(jarEntry, jar);
+ // check if any visited resources belong to this type
+ for (Iterator<String> iter = resourceCacheSet.iterator(); iter.hasNext(); ) {
+ String entry = iter.next();
+ if (entry.startsWith(entryName.substring(0, entryName.length() - 6))) {
+ newNode.setHasResource(true);
+ iter.remove();
+ }
+ }
+ } else {
+ String className = entryName;
+ boolean foundClass = false;
+ // check if this resource belongs to any visited type
+ while (className.contains("/")) {
+ className = className.substring(0, className.lastIndexOf('/'));
+ TypeGraph.TypeGraphVertex tgv = typeGraph.getNode(className.replace('/', '.'));
+ if (tgv != null) {
+ tgv.setHasResource(true);
+ foundClass = true;
+ break;
+ }
+ }
+ if (!foundClass) {
+ resourceCacheSet.add(entryName);
}
- } else if (!jarEntry.isDirectory() && jarEntry.getName().endsWith(".class")) {
- typeGraph.addNode(jarEntry, jar);
}
}
}
@@ -317,8 +348,7 @@ public class OperatorDiscoverer
typeGraph.trim();
typeGraph.updatePortTypeInfoInTypeGraph(openJarFiles, openClassFiles);
- }
- finally {
+ } finally {
for (Entry<String, JarFile> entry : openJarFiles.entrySet()) {
try {
entry.getValue().close();
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
----------------------------------------------------------------------
diff --git a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
index fc175af..95d7529 100644
--- a/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
+++ b/engine/src/main/java/com/datatorrent/stram/webapp/TypeGraph.java
@@ -125,7 +125,12 @@ public class TypeGraph
}
return false;
}
-
+
+ public TypeGraphVertex getNode(String typeName)
+ {
+ return typeGraph.get(typeName);
+ }
+
enum UI_TYPE {
LIST("List", Collection.class.getName()),
@@ -234,7 +239,7 @@ public class TypeGraph
private final Map<String, TypeGraphVertex> typeGraph = new HashMap<String, TypeGraphVertex>();
- private void addNode(InputStream input, String resName) throws IOException
+ private TypeGraphVertex addNode(InputStream input, String resName) throws IOException
{
try {
@@ -243,7 +248,6 @@ public class TypeGraph
reader.accept(classN, ClassReader.SKIP_CODE);
CompactClassNode ccn = CompactUtil.compactClassNode(classN);
String typeName = classN.name.replace('/', '.');
-
TypeGraphVertex tgv = null;
TypeGraphVertex ptgv = null;
if (typeGraph.containsKey(typeName)) {
@@ -279,6 +283,7 @@ public class TypeGraph
}
updateInstantiableDescendants(tgv);
+ return tgv;
} finally {
if (input != null) {
input.close();
@@ -286,14 +291,14 @@ public class TypeGraph
}
}
- public void addNode(File file) throws IOException
+ public TypeGraphVertex addNode(File file) throws IOException
{
- addNode(new FileInputStream(file), file.getAbsolutePath());
+ return addNode(new FileInputStream(file), file.getAbsolutePath());
}
- public void addNode(JarEntry jarEntry, JarFile jar) throws IOException
+ public TypeGraphVertex addNode(JarEntry jarEntry, JarFile jar) throws IOException
{
- addNode(jar.getInputStream(jarEntry), jar.getName());
+ return addNode(jar.getInputStream(jarEntry), jar.getName());
}
public void updatePortTypeInfoInTypeGraph(Map<String, JarFile> openJarFiles,
@@ -477,7 +482,9 @@ public class TypeGraph
// keep the jar file name for late fetching the detail information
private String jarName;
-
+
+ private boolean hasResource = false;
+
@SuppressWarnings("unused")
private TypeGraphVertex(){
jarName = "";
@@ -507,6 +514,16 @@ public class TypeGraph
this.jarName = jarName;
}
+ public boolean hasResource()
+ {
+ return hasResource;
+ }
+
+ public void setHasResource(boolean hasResource)
+ {
+ this.hasResource = hasResource;
+ }
+
public boolean isInstantiable()
{
return JACKSON_INSTANTIABLE_CLASSES.contains(this.typeName) || (isPublicConcrete() && classNode.getDefaultConstructor() != null);
@@ -647,6 +664,12 @@ public class TypeGraph
addClassPropertiesAndPorts(clazzName, desc);
+ if(tgv.hasResource()){
+ desc.put("hasResource", "true");
+ } else {
+ desc.put("hasResource", "false");
+ }
+
return desc;
}
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
----------------------------------------------------------------------
diff --git a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
index 6ac3438..9af7d34 100644
--- a/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/webapp/OperatorDiscoveryTest.java
@@ -49,6 +49,7 @@ import com.datatorrent.api.InputOperator;
import com.datatorrent.stram.plan.logical.LogicalPlan;
import com.datatorrent.stram.plan.logical.LogicalPlan.OperatorMeta;
import com.datatorrent.stram.plan.logical.LogicalPlanConfiguration;
+import com.datatorrent.stram.support.StramTestSupport;
import com.datatorrent.stram.util.ObjectMapperFactory;
import com.datatorrent.stram.webapp.TypeDiscoverer.UI_TYPE;
@@ -494,6 +495,26 @@ public class OperatorDiscoveryTest
}
+ @Test
+ public void testExternalResource() throws Exception
+ {
+
+
+ StramTestSupport.createAppPackageFile();
+
+ String[] classFilePath = getClassFileInClasspath();
+
+ String[]cPaths = Lists.asList("src/test/resources/testAppPackage/mydtapp/target/mydtapp-1.0-SNAPSHOT.jar", classFilePath).toArray(new String[]{});
+ OperatorDiscoverer od = new OperatorDiscoverer(cPaths);
+ od.buildTypeGraph();
+
+ Assert.assertEquals("true", od.describeClass("com.example.mydtapp.StdoutOperator").getString("hasResource"));
+
+ StramTestSupport.removeAppPackageFile();
+
+ }
+
+
public static class Structured
{
private int size;
http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/00b486f0/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html
----------------------------------------------------------------------
diff --git a/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html
new file mode 100644
index 0000000..5b23309
--- /dev/null
+++ b/engine/src/test/resources/testAppPackage/mydtapp/src/main/resources/com/example/mydtapp/StdoutOperator/ExtRes.html
@@ -0,0 +1 @@
+<html>example</html>
[2/2] incubator-apex-core git commit: Merge branch 'APEX-70' of
https://github.com/siyuanh/incubator-apex-core into devel-3
Posted by th...@apache.org.
Merge branch 'APEX-70' of https://github.com/siyuanh/incubator-apex-core into devel-3
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/f2a40717
Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/f2a40717
Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/f2a40717
Branch: refs/heads/devel-3
Commit: f2a40717b940fb9c409c65de2ba240f5b6096174
Parents: d748ed4 00b486f
Author: thomas <th...@datatorrent.com>
Authored: Mon Aug 31 22:52:00 2015 -0700
Committer: thomas <th...@datatorrent.com>
Committed: Mon Aug 31 22:52:00 2015 -0700
----------------------------------------------------------------------
.../stram/webapp/OperatorDiscoverer.java | 44 ++++++++++++++++----
.../com/datatorrent/stram/webapp/TypeGraph.java | 39 +++++++++++++----
.../stram/webapp/OperatorDiscoveryTest.java | 21 ++++++++++
.../example/mydtapp/StdoutOperator/ExtRes.html | 1 +
4 files changed, 90 insertions(+), 15 deletions(-)
----------------------------------------------------------------------