You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by yo...@apache.org on 2016/08/18 20:50:39 UTC
incubator-eagle git commit: EAGLE-474 some improvements of app
framework: appId naming, jarFile field,
InstallOperation etc. improvements of app framework: appId naming,
jarFile field, InstallOperation
Repository: incubator-eagle
Updated Branches:
refs/heads/develop 92ca7c542 -> 6fc103338
EAGLE-474 some improvements of app framework: appId naming, jarFile field, InstallOperation etc.
improvements of app framework: appId naming, jarFile field, InstallOperation
https://issues.apache.org/jira/browse/EAGLE-474
Author: @yonzhang <yo...@apache.org>
Closes: #362
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/6fc10333
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/6fc10333
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/6fc10333
Branch: refs/heads/develop
Commit: 6fc103338fa3e3275a1357144dfab2feb1a103ec
Parents: 92ca7c5
Author: yonzhang <yo...@gmail.com>
Authored: Thu Aug 18 13:54:37 2016 -0700
Committer: yonzhang <yo...@gmail.com>
Committed: Thu Aug 18 13:54:37 2016 -0700
----------------------------------------------------------------------
.../environment/impl/StormExecutionRuntime.java | 7 ++++--
.../eagle/app/service/ApplicationContext.java | 1 +
.../app/service/ApplicationOperations.java | 11 +++++++++-
.../impl/ApplicationManagementServiceImpl.java | 4 ++++
.../app/spi/AbstractApplicationProvider.java | 3 +--
.../eagle/metadata/model/ApplicationDesc.java | 23 +++-----------------
.../eagle/metadata/model/ApplicationEntity.java | 17 ++++++++++-----
.../src/main/resources/scripts | 8 +++----
.../src/main/resources/application.conf | 2 +-
.../src/assembly/eagle-topology-assembly.xml | 1 +
10 files changed, 41 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
index 5b236dd..83d3592 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
@@ -85,6 +85,7 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
}
conf.put(backtype.storm.Config.NIMBUS_HOST, nimbusHost);
conf.put(backtype.storm.Config.NIMBUS_THRIFT_PORT, nimbusThriftPort);
+ conf.put(Config.STORM_THRIFT_TRANSPORT_PLUGIN, "backtype.storm.security.auth.SimpleTransportPlugin");
return conf;
}
@@ -96,8 +97,7 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
LOG.info("Starting {} ({})",topologyName,executor.getClass().getCanonicalName());
Config conf = getStormConfig();
if(config.getString("mode") == ApplicationEntity.Mode.CLUSTER.name()){
-// if(config.getString("jarPath") == null) config.setJarPath(DynamicJarPathFinder.findPath(executor.getClass()));
- String jarFile = config.getString("jarPath");
+ String jarFile = config.hasPath("jarPath") ? config.getString("jarPath") : null;
if(jarFile == null){
jarFile = DynamicJarPathFinder.findPath(executor.getClass());
}
@@ -118,11 +118,13 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
getLocalCluster().submitTopology(topologyName, conf, topology);
LOG.info("Submitted");
}
+ LOG.info("Started {} ({})",topologyName,executor.getClass().getCanonicalName());
}
@Override
public void stop(Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
String appId = config.getString("appId");
+ LOG.info("Stopping topology {} ..." + appId);
if(config.getString("mode") == ApplicationEntity.Mode.CLUSTER.name()){
Nimbus.Client stormClient = NimbusClient.getConfiguredClient(getStormConfig()).getClient();
try {
@@ -135,6 +137,7 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
killOptions.set_wait_secs(0);
getLocalCluster().killTopologyWithOpts(appId,killOptions);
}
+ LOG.info("Stopped topology {} ..." + appId);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
index b0d6118..76ee289 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationContext.java
@@ -67,6 +67,7 @@ public class ApplicationContext implements Serializable, ApplicationLifecycle {
executionConfig.put("siteId", metadata.getSite().getSiteId());
executionConfig.put("mode", metadata.getMode().name());
executionConfig.put("appId", metadata.getAppId());
+ executionConfig.put("jarPath", metadata.getJarPath());
this.config = ConfigFactory.parseMap(executionConfig).withFallback(envConfig);
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperations.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperations.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperations.java
index 792b7d4..4815a90 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperations.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/ApplicationOperations.java
@@ -35,6 +35,7 @@ public final class ApplicationOperations {
private String siteId;
private String appType;
private ApplicationEntity.Mode mode = ApplicationEntity.Mode.LOCAL;
+ private String jarPath;
private Map<String,Object> configuration;
public InstallOperation(){}
@@ -83,6 +84,14 @@ public final class ApplicationOperations {
this.mode = mode;
}
+ public String getJarPath() {
+ return jarPath;
+ }
+
+ public void setJarPath(String jarPath) {
+ this.jarPath = jarPath;
+ }
+
@Override
public String getType() {
return INSTALL;
@@ -186,4 +195,4 @@ public final class ApplicationOperations {
return STOP;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
index a0b0dc4..314b0fb 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationManagementServiceImpl.java
@@ -64,6 +64,9 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe
public ApplicationEntity install(ApplicationOperations.InstallOperation operation) throws EntityNotFoundException {
Preconditions.checkNotNull(operation.getSiteId(),"siteId is null");
Preconditions.checkNotNull(operation.getAppType(),"appType is null");
+ if(operation.getMode().equals(ApplicationEntity.Mode.CLUSTER)){
+ Preconditions.checkNotNull(operation.getJarPath(),"jarPath is null when mode is CLUSTER");
+ }
SiteEntity siteEntity = siteEntityService.getBySiteId(operation.getSiteId());
Preconditions.checkNotNull(siteEntity,"Site with ID: "+operation.getSiteId()+" is not found");
ApplicationDesc appDesc = applicationProviderService.getApplicationDescByType(operation.getAppType());
@@ -72,6 +75,7 @@ public class ApplicationManagementServiceImpl implements ApplicationManagementSe
applicationEntity.setDescriptor(appDesc);
applicationEntity.setSite(siteEntity);
applicationEntity.setMode(operation.getMode());
+ applicationEntity.setJarPath(operation.getJarPath());
applicationEntity.ensureDefault();
/**
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/AbstractApplicationProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/AbstractApplicationProvider.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/AbstractApplicationProvider.java
index bc6cdba..6975b6b 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/AbstractApplicationProvider.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/AbstractApplicationProvider.java
@@ -81,7 +81,6 @@ public abstract class AbstractApplicationProvider<T extends Application> impleme
@Override
public void prepare(ApplicationProviderConfig providerConfig, Config envConfig) {
- this.applicationDesc.setJarPath(providerConfig.getJarPath());
}
protected void setVersion(String version) {
@@ -143,4 +142,4 @@ public abstract class AbstractApplicationProvider<T extends Application> impleme
public void register(ModuleRegistry registry) {
LOG.debug("Registering modules {}",this.getClass().getName());
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
index c4a609b..f8d9c41 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationDesc.java
@@ -30,12 +30,10 @@ public class ApplicationDesc implements Serializable {
private String version;
private String description;
private Class<?> appClass;
- private String jarPath;
private String viewPath;
private Class<?> providerClass;
private Configuration configuration;
private List<StreamDefinition> streams;
-// private Class<?> sinkClass;
private ApplicationDocs docs;
public String getDescription() {
@@ -62,10 +60,6 @@ public class ApplicationDesc implements Serializable {
return viewPath;
}
- public String getJarPath() {
- return jarPath;
- }
-
public void setType(String type) {
this.type = type;
}
@@ -81,9 +75,6 @@ public class ApplicationDesc implements Serializable {
public void setDescription(String description) {
this.description = description;
}
- public void setJarPath(String jarPath) {
- this.jarPath = jarPath;
- }
public void setViewPath(String viewPath) {
this.viewPath = viewPath;
@@ -107,22 +98,14 @@ public class ApplicationDesc implements Serializable {
@Override
public String toString() {
- return String.format("ApplicationDesc [type=%s, name=%s, version=%s, appClass=%s, viewPath=%s, jarPath=%s, providerClass=%s, configuration= %s properties, description=%s",
- getType(),getName(),getVersion(),getAppClass(), getViewPath(),getJarPath(),getProviderClass(), getConfiguration() == null ? 0: getConfiguration().size(),getDescription());
+ return String.format("ApplicationDesc [type=%s, name=%s, version=%s, appClass=%s, viewPath=%s, providerClass=%s, configuration= %s properties, description=%s",
+ getType(),getName(),getVersion(),getAppClass(), getViewPath(),getProviderClass(), getConfiguration() == null ? 0: getConfiguration().size(),getDescription());
}
public void setConfiguration(Configuration configuration) {
this.configuration = configuration;
}
-// public Class<?> getSinkType() {
-// return sinkClass;
-// }
-//
-// public void setSinkClass(Class<?> sinkClass) {
-// this.sinkClass = sinkClass;
-// }
-
public List<StreamDefinition> getStreams() {
return streams;
}
@@ -138,4 +121,4 @@ public class ApplicationDesc implements Serializable {
public void setDocs(ApplicationDocs docs) {
this.docs = docs;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
index 80b3669..5f3adb1 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/ApplicationEntity.java
@@ -19,7 +19,6 @@ package org.apache.eagle.metadata.model;
import org.apache.eagle.metadata.persistence.PersistenceEntity;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -33,15 +32,13 @@ public class ApplicationEntity extends PersistenceEntity {
private String appId;
private SiteEntity site;
- /**
- * TODO: Think about keeping ApplicationDesc as a reference or deep clone into current instance
- */
private ApplicationDesc descriptor;
private Map<String,Object> configuration = new HashMap<>();
private Map<String,String> context = new HashMap<>();
private List<StreamDesc> streams;
private Mode mode = Mode.CLUSTER;
+ private String jarPath;
private Status status = Status.INITIALIZED;
public SiteEntity getSite() {
@@ -80,7 +77,7 @@ public class ApplicationEntity extends PersistenceEntity {
public void ensureDefault() {
super.ensureDefault();
if(this.appId == null){
- this.appId = String.format("EAGLE_APP[TYPE=%s,SITE=%s]",this.getDescriptor().getType(),this.getSite().getSiteId());
+ this.appId = String.format("%s-%s", this.getDescriptor().getType(), this.getSite().getSiteId());
}
if(this.status == null){
this.status = Status.INITIALIZED;
@@ -103,6 +100,14 @@ public class ApplicationEntity extends PersistenceEntity {
this.mode = mode;
}
+ public String getJarPath() {
+ return jarPath;
+ }
+
+ public void setJarPath(String jarPath) {
+ this.jarPath = jarPath;
+ }
+
public Status getStatus() {
return status;
}
@@ -151,4 +156,4 @@ public class ApplicationEntity extends PersistenceEntity {
return this.name;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-security/eagle-security-hbase-auditlog/src/main/resources/scripts
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/scripts b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/scripts
index 7d04fdc..02779e3 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/scripts
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/scripts
@@ -13,10 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-./kafka-topics.sh --topic sandbox_hbase_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2
+./kafka-topics.sh --topic hbase_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2
-./kafka-topics.sh --topic sandbox_hbase_audit_log_parsed --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2
+./kafka-topics.sh --topic hbase_audit_log --zookeeper localhost:2181 --create --replication-factor 1 --partitions 2
-./kafka-console-producer.sh --topic sandbox_hbase_audit_log --broker-list sandbox.hortonworks.com:6667
+./kafka-console-producer.sh --topic hbase_audit_log --broker-list sandbox.hortonworks.com:6667
-./kafka-console-consumer.sh --topic sandbox_hbase_audit_log_parsed --zookeeper sandbox.hortonworks.com:2181 --from-beginning
+./kafka-console-consumer.sh --topic hbase_audit_log_parsed --zookeeper sandbox.hortonworks.com:2181 --from-beginning
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-server/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/resources/application.conf b/eagle-server/src/main/resources/application.conf
index 9363445..6db7707 100644
--- a/eagle-server/src/main/resources/application.conf
+++ b/eagle-server/src/main/resources/application.conf
@@ -53,7 +53,7 @@
"type": "org.apache.eagle.app.sink.KafkaStreamSink"
},
"storm": {
- "nimbusHost": "localhost"
+ "nimbusHost": "server.eagle.apache.org"
"nimbusThriftPort": 6627
},
"provider" : {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/6fc10333/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
index 76cd5b1..660a740 100644
--- a/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
+++ b/eagle-topology-assembly/src/assembly/eagle-topology-assembly.xml
@@ -41,6 +41,7 @@
</unpackOptions>
<excludes>
<exclude>org.slf4j:slf4j-api</exclude>
+ <exclude>org.slf4j:slf4j-log4j12</exclude>
<exclude>io.dropwizard:**</exclude>
<exclude>com.sun.jersey:**</exclude>
<exclude>com.sun.jersey.contribs:**</exclude>