You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2020/01/15 22:54:53 UTC
[twill] branch master updated (b25f403 -> 1acf1b4)
This is an automated email from the ASF dual-hosted git repository.
chtyim pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/twill.git.
from b25f403 (TWILL-260) Upgrade version of zkclient - library that kafka is using.
new c08a2c5 (TWILL-268) Upgrade to asm 7.1 to work with Java modules
new 1acf1b4 (TWILL-271) Add custom ClassLoader dependencies
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
pom.xml | 9 ++++++--
twill-core/pom.xml | 2 +-
.../apache/twill/internal/utils/Dependencies.java | 10 ++++-----
twill-yarn/pom.xml | 5 +++++
.../org/apache/twill/yarn/YarnTwillPreparer.java | 24 ++++++++++++++--------
.../apache/twill/yarn/LogLevelChangeTestRun.java | 2 +-
6 files changed, 34 insertions(+), 18 deletions(-)
[twill] 02/02: (TWILL-271) Add custom ClassLoader dependencies
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chtyim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/twill.git
commit 1acf1b4eddc2687ddf416ac26df852f6e65b931d
Author: Terence Yim <te...@google.com>
AuthorDate: Wed Jan 15 11:31:21 2020 -0800
(TWILL-271) Add custom ClassLoader dependencies
Signed-off-by: Terence Yim <te...@google.com>
---
.../org/apache/twill/yarn/YarnTwillPreparer.java | 24 ++++++++++++++--------
.../apache/twill/yarn/LogLevelChangeTestRun.java | 2 +-
2 files changed, 16 insertions(+), 10 deletions(-)
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 1ea2ab3..c69b009 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -113,6 +113,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
+import java.util.stream.Collectors;
import javax.annotation.Nullable;
/**
@@ -121,12 +122,6 @@ import javax.annotation.Nullable;
final class YarnTwillPreparer implements TwillPreparer {
private static final Logger LOG = LoggerFactory.getLogger(YarnTwillPreparer.class);
- private static final Function<Class<?>, String> CLASS_TO_NAME = new Function<Class<?>, String>() {
- @Override
- public String apply(Class<?> cls) {
- return cls.getName();
- }
- };
private final Configuration config;
private final TwillSpecification twillSpec;
@@ -575,13 +570,24 @@ final class YarnTwillPreparer implements TwillPreparer {
// Add the TwillRunnableEventHandler class
if (twillSpec.getEventHandler() != null) {
- classes.add(getClassLoader().loadClass(twillSpec.getEventHandler().getClassName()));
+ classes.add(classLoader.loadClass(twillSpec.getEventHandler().getClassName()));
+ }
+
+ // Optionally add the custom classloader class
+ if (classLoaderClassName != null) {
+ try {
+ classes.add(classLoader.loadClass(classLoaderClassName));
+ } catch (ClassNotFoundException e) {
+ // Don't throw if the classloader class is not found, as it can be available
+ // in the target cluster with appropriate classpath setting
+ LOG.debug("Cannot load custom classloader class '{}' when preparing for application launch",
+ classLoaderClassName);
+ }
}
// The location name is computed from the MD5 of all the classes names
// The localized name is always APPLICATION_JAR
- List<String> classList = Lists.newArrayList(Iterables.transform(classes, CLASS_TO_NAME));
- Collections.sort(classList);
+ List<String> classList = classes.stream().map(Class::getName).sorted().collect(Collectors.toList());
Hasher hasher = Hashing.md5().newHasher();
for (String name : classList) {
hasher.putString(name);
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelChangeTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelChangeTestRun.java
index a1d8ae6..43787f0 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelChangeTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelChangeTestRun.java
@@ -277,7 +277,7 @@ public class LogLevelChangeTestRun extends BaseYarnTest {
if (matchCount == expectedInstances) {
return;
}
- TimeUnit.MILLISECONDS.sleep(100);
+ TimeUnit.SECONDS.sleep(1);
}
Assert.fail("Timeout waiting for expected log levels");
[twill] 01/02: (TWILL-268) Upgrade to asm 7.1 to work with Java
modules
Posted by ch...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
chtyim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/twill.git
commit c08a2c59b1426788e0c5d77dcc7aba1c8a2dde6e
Author: Terence Yim <te...@google.com>
AuthorDate: Wed Jan 15 10:44:52 2020 -0800
(TWILL-268) Upgrade to asm 7.1 to work with Java modules
This closes #85 on Github.
Signed-off-by: Terence Yim <te...@google.com>
---
pom.xml | 9 +++++++--
twill-core/pom.xml | 2 +-
.../java/org/apache/twill/internal/utils/Dependencies.java | 10 +++++-----
twill-yarn/pom.xml | 5 +++++
4 files changed, 18 insertions(+), 8 deletions(-)
diff --git a/pom.xml b/pom.xml
index 09a18fe..b76b1b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -173,7 +173,7 @@
<netty.version>4.1.16.Final</netty.version>
<snappy-java.version>1.0.5</snappy-java.version>
<jcl-over-slf4j.version>1.7.2</jcl-over-slf4j.version>
- <asm.version>5.0.2</asm.version>
+ <asm.version>7.1</asm.version>
<kafka.version>0.8.0</kafka.version>
<zkclient.version>0.10</zkclient.version>
<zookeeper.version>3.4.5</zookeeper.version>
@@ -729,7 +729,12 @@
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
- <artifactId>asm-all</artifactId>
+ <artifactId>asm</artifactId>
+ <version>${asm.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-commons</artifactId>
<version>${asm.version}</version>
</dependency>
<dependency>
diff --git a/twill-core/pom.xml b/twill-core/pom.xml
index e0c482c..249c061 100644
--- a/twill-core/pom.xml
+++ b/twill-core/pom.xml
@@ -71,7 +71,7 @@
</dependency>
<dependency>
<groupId>org.ow2.asm</groupId>
- <artifactId>asm-all</artifactId>
+ <artifactId>asm</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git a/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java b/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
index eb55557..d06062d 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
@@ -144,7 +144,7 @@ public final class Dependencies {
private final DependencyAcceptor acceptor;
public DependencyClassVisitor(DependencyAcceptor acceptor) {
- super(Opcodes.ASM5);
+ super(Opcodes.ASM7);
this.acceptor = acceptor;
this.signatureVisitor = createSignatureVisitor();
this.annotationVisitor = createAnnotationVisitor();
@@ -189,7 +189,7 @@ public final class Dependencies {
addType(Type.getType(desc));
}
- return new FieldVisitor(Opcodes.ASM5) {
+ return new FieldVisitor(Opcodes.ASM7) {
@Override
public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
if (!visible) {
@@ -210,7 +210,7 @@ public final class Dependencies {
}
addClasses(exceptions);
- return new MethodVisitor(Opcodes.ASM5) {
+ return new MethodVisitor(Opcodes.ASM7) {
@Override
public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
@@ -315,7 +315,7 @@ public final class Dependencies {
* Creates a {@link SignatureVisitor} for gathering dependency information from class signature.
*/
private SignatureVisitor createSignatureVisitor() {
- return new SignatureVisitor(Opcodes.ASM5) {
+ return new SignatureVisitor(Opcodes.ASM7) {
private String currentClass;
@Override
@@ -335,7 +335,7 @@ public final class Dependencies {
* Creates an {@link AnnotationVisitor} for gathering dependency information from annotations.
*/
private AnnotationVisitor createAnnotationVisitor() {
- return new AnnotationVisitor(Opcodes.ASM5) {
+ return new AnnotationVisitor(Opcodes.ASM7) {
@Override
public void visit(String name, Object value) {
if (value instanceof Type) {
diff --git a/twill-yarn/pom.xml b/twill-yarn/pom.xml
index c3816ab..508f0be 100644
--- a/twill-yarn/pom.xml
+++ b/twill-yarn/pom.xml
@@ -84,6 +84,11 @@
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-commons</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>