You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ja...@apache.org on 2017/05/07 18:12:50 UTC
[01/17] eagle git commit: adding myself in developer list as a first
commit
Repository: eagle
Updated Branches:
refs/heads/branch-0.5 31f2582da -> 31aa29f59
adding myself in developer list as a first commit
adding me in developer list and trying to merge this as my first commit to repo.
Author: Jay <jh...@gmail.com>
Closes #915 from jhsenjaliya/add-me-in-developer-list.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/d20f2e11
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/d20f2e11
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/d20f2e11
Branch: refs/heads/branch-0.5
Commit: d20f2e1162f986c66fd5e5e23c85a22aabd5d417
Parents: c243f41
Author: Jay <jh...@gmail.com>
Authored: Sun Apr 16 22:05:15 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Sun Apr 16 22:05:15 2017 -0700
----------------------------------------------------------------------
pom.xml | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/d20f2e11/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bf21b12..7862e6f 100755
--- a/pom.xml
+++ b/pom.xml
@@ -123,6 +123,13 @@
<organization>Apache Software Foundation</organization>
<organizationUrl>https://eagle.apache.org</organizationUrl>
</developer>
+ <developer>
+ <id>jaysen</id>
+ <name>Jayesh Senjaliya</name>
+ <email>jaysen@apache.org</email>
+ <organization>Apache Software Foundation</organization>
+ <organizationUrl>https://eagle.apache.org</organizationUrl>
+ </developer>
</developers>
<modules>
<module>eagle-core</module>
[09/17] eagle git commit: [EAGLE-1008] java.lang.NullPointerException
in JHFEventReaderBase.close
Posted by ja...@apache.org.
[EAGLE-1008] java.lang.NullPointerException in JHFEventReaderBase.close
https://issues.apache.org/jira/browse/EAGLE-1008
Author: Zhao, Qingwen <qi...@apache.org>
Closes #919 from qingwen220/minor.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/3b66c0d3
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/3b66c0d3
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/3b66c0d3
Branch: refs/heads/branch-0.5
Commit: 3b66c0d3c710b7c64be7839c0246218edf806e8b
Parents: cc5d23b
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Tue Apr 18 16:44:11 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Tue Apr 18 16:44:11 2017 +0800
----------------------------------------------------------------------
.../eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java | 12 ++++++++----
.../analyzer/mr/suggestion/JobSuggestionEvaluator.java | 12 ++++++------
.../apache/eagle/jpm/util/jobcounter/JobCounters.java | 3 ++-
3 files changed, 16 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/3b66c0d3/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
index 34365dc..cca6b18 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/MRJobPerformanceAnalyzer.java
@@ -61,14 +61,18 @@ public class MRJobPerformanceAnalyzer<T extends AnalyzerEntity> implements JobAn
Result result = new Result();
for (Evaluator evaluator : evaluators) {
- Result.EvaluatorResult evaluatorResult = evaluator.evaluate(analyzerJobEntity);
- if (evaluatorResult != null) {
- result.addEvaluatorResult(evaluator.getClass(), evaluatorResult);
+ try {
+ Result.EvaluatorResult evaluatorResult = evaluator.evaluate(analyzerJobEntity);
+ if (evaluatorResult != null) {
+ result.addEvaluatorResult(evaluator.getClass(), evaluatorResult);
+ }
+ } catch (Throwable e) {
+ LOG.error("evaluator {} fails to analyse job {}", evaluator, analyzerJobEntity.getJobId(), e);
}
}
if (alertDeduplicator.dedup(analyzerJobEntity, result)) {
- LOG.info("skip publish job {} alert because it is duplicated", analyzerJobEntity.getJobDefId());
+ LOG.info("skip publish job {} alert because it is duplicated", analyzerJobEntity.getJobId());
return;
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/3b66c0d3/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java
index e1a357a..4e2726b 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/mr/suggestion/JobSuggestionEvaluator.java
@@ -63,7 +63,6 @@ public class JobSuggestionEvaluator implements Evaluator<MapReduceAnalyzerEntity
return null;
}
-
if (analyzerEntity.getTotalCounters() == null) {
LOG.warn("Total counters of Job {} is null", analyzerEntity.getJobId());
return null;
@@ -73,13 +72,14 @@ public class JobSuggestionEvaluator implements Evaluator<MapReduceAnalyzerEntity
return null;
}
- MapReduceJobSuggestionContext jobContext = new MapReduceJobSuggestionContext(analyzerEntity);
- if (jobContext.getNumMaps() == 0) {
- return null;
- }
-
try {
Result.EvaluatorResult result = new Result.EvaluatorResult();
+
+ MapReduceJobSuggestionContext jobContext = new MapReduceJobSuggestionContext(analyzerEntity);
+ if (jobContext.getNumMaps() == 0) {
+ return null;
+ }
+
for (Processor processor : loadProcessors(jobContext)) {
Result.ProcessorResult processorResult = processor.process(analyzerEntity);
if (processorResult != null) {
http://git-wip-us.apache.org/repos/asf/eagle/blob/3b66c0d3/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java
index bbb80cd..e337c13 100644
--- a/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java
+++ b/eagle-jpm/eagle-jpm-util/src/main/java/org/apache/eagle/jpm/util/jobcounter/JobCounters.java
@@ -48,7 +48,8 @@ public final class JobCounters implements Serializable {
}
public Long getCounterValue(CounterName counterName) {
- if (counters.get(counterName.group.name).containsKey(counterName.name)) {
+ if (counters.containsKey(counterName.group.name)
+ && counters.get(counterName.group.name).containsKey(counterName.name)) {
return counters.get(counterName.group.name).get(counterName.name);
} else {
return 0L;
[17/17] eagle git commit: Merge remote-tracking branch
'origin/master' into branch-0.5
Posted by ja...@apache.org.
Merge remote-tracking branch 'origin/master' into branch-0.5
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/31aa29f5
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/31aa29f5
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/31aa29f5
Branch: refs/heads/branch-0.5
Commit: 31aa29f593c7adeda50c755569a59493152d3e60
Parents: 31f2582 8da0663
Author: Jay <jh...@gmail.com>
Authored: Sun May 7 11:11:00 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Sun May 7 11:11:00 2017 -0700
----------------------------------------------------------------------
.travis.yml | 18 +-
eagle-assembly/src/main/doc/metadata-ddl.sql | 7 -
.../src/assembly/alert-assembly.xml | 3 +-
.../engine/coordinator/AlertDeduplication.java | 71 ++
.../engine/coordinator/PolicyDefinition.java | 13 +-
.../engine/coordinator/PublishmentType.java | 21 +-
.../publisher/email/AlertEmailGenerator.java | 4 +-
.../publisher/impl/AbstractPublishPlugin.java | 17 +-
.../publisher/impl/AlertEmailPublisher.java | 2 -
.../publisher/impl/AlertPublisherImpl.java | 43 +-
.../publisher/impl/DefaultDeduplicator.java | 10 +-
.../template/VelocityAlertTemplateEngine.java | 8 +-
.../alert/engine/runner/AlertPublisherBolt.java | 18 +-
.../alert/engine/spout/CorrelationSpout.java | 14 +-
.../main/resources/ALERT_INLINED_TEMPLATE.vm | 2 +-
.../VelocityAlertTemplateEngineTest.java | 2 +-
.../metadata/impl/JdbcMetadataDaoImpl.java | 2 +-
.../metadata/impl/JdbcMetadataHandler.java | 30 +
.../app/environment/AbstractEnvironment.java | 10 +
.../environment/impl/StormExecutionRuntime.java | 31 +-
.../ExecutionRuntimeManagerTest.java | 45 +
.../apache/eagle/common/config/EagleConfig.java | 2 +-
.../eagle/common/config/EagleConfigFactory.java | 2 +-
.../eagle/service/hbase/EmbeddedHBase.java | 141 +++
.../eagle/service/hbase/EmbeddedHbase.java | 141 ---
.../eagle/service/hbase/EmbeddedHBaseTest.java | 28 +
.../eagle/service/hbase/EmbeddedHbaseTest.java | 28 -
.../eagle/audit/listener/TestAuditSupport.java | 2 +-
.../eagle/service/client/ClientTestBase.java | 6 +-
.../log/entity/TestHBaseWritePerformance.java | 100 +++
.../log/entity/TestHbaseWritePerformance.java | 100 ---
.../apache/eagle/query/TestHBaseLogReader2.java | 4 +-
.../EagleServiceSelfCheckAPIEntity.java | 8 +-
.../EagleServiceSelfCheckResource.java | 4 +-
.../storage/hbase/HBaseEntitySchemaManager.java | 2 +-
eagle-external/eagle-cdh/README.md | 66 ++
eagle-external/eagle-cdh/assembly.xml | 29 +
eagle-external/eagle-cdh/pom.xml | 51 ++
.../eagle-cdh/src/descriptor/service.sdl | 868 +++++++++++++++++++
eagle-external/eagle-cdh/src/images/icon.png | Bin 0 -> 3346 bytes
eagle-external/eagle-cdh/src/scripts/control.sh | 35 +
eagle-external/pom.xml | 1 +
.../analyzer/mr/MRJobPerformanceAnalyzer.java | 12 +-
.../mr/suggestion/JobSuggestionEvaluator.java | 12 +-
.../apache/eagle/jpm/analyzer/util/Utils.java | 53 +-
.../jpm/mr/history/storm/JobHistorySpout.java | 4 +-
.../jpm/mr/history/JHFEventReaderBaseTest.java | 74 ++
.../src/test/resources/application.conf | 74 ++
.../resources/job_1479206441898_508949_conf.xml | 191 +++-
.../eagle/jpm/util/jobcounter/JobCounters.java | 3 +-
.../apps/hbase/ctrls/backupMasterListCtrl.js | 4 +-
.../app/apps/hbase/ctrls/regionListCtrl.js | 2 +-
.../src/main/webapp/app/apps/hbase/index.js | 4 +-
.../app/apps/hdfs/ctrl/datanodeListCtrl.js | 2 +-
.../app/apps/hdfs/ctrl/namenodeListCtrl.js | 2 +-
.../security/entity/HBaseResourceEntity.java | 105 +++
.../security/entity/HbaseResourceEntity.java | 105 ---
.../security/hdfs/MAPRFSAuditLogParser.java | 20 +-
.../service/HBaseSensitivityEntity.java | 4 +-
.../security/service/InMemMetadataDaoImpl.java | 3 +-
.../service/JDBCSecurityMetadataDAO.java | 4 +-
.../audit/JDBCSecurityMetadataDAOTest.java | 2 +-
.../audit/TestMetaDataAccessConfigRepo.java | 2 +-
.../eagle-security-hbase-auditlog/README.md | 2 +-
.../hbase/HBaseAuditLogApplication.java | 2 +-
.../security/hbase/HBaseAuditLogObject.java | 28 +
.../security/hbase/HBaseAuditLogParser.java | 148 ++++
.../security/hbase/HBaseAuditLogParserBolt.java | 4 +-
.../HBaseResourceSensitivityDataJoinBolt.java | 80 ++
.../hbase/HBaseSensitivityDataEnrichLCM.java | 2 +-
.../security/hbase/HbaseAuditLogObject.java | 28 -
.../security/hbase/HbaseAuditLogParser.java | 148 ----
.../HbaseResourceSensitivityDataJoinBolt.java | 80 --
....security.hbase.HBaseAuditLogAppProvider.xml | 4 +-
.../security/hbase/TestHBaseAuditLogParser.java | 44 +
.../security/hbase/TestHbaseAuditLogParser.java | 44 -
.../hbase/HBaseMetadataBrowseWebResource.java | 189 ++++
.../hbase/HBaseMetadataBrowseWebResponse.java | 45 +
.../hbase/HbaseMetadataBrowseWebResource.java | 190 ----
.../hbase/HbaseMetadataBrowseWebResponse.java | 45 -
.../hbase/dao/HBaseMetadataDAOImpl.java | 95 ++
.../hbase/dao/HbaseMetadataDAOImpl.java | 95 --
.../hbase/resolver/HBaseActionResolver.java | 41 +
.../hbase/resolver/HBaseMetadataResolver.java | 130 +++
.../hbase/resolver/HBaseRequestResolver.java | 67 ++
.../resolver/HBaseSensitivityTypeResolver.java | 58 ++
.../hbase/resolver/HbaseActionResolver.java | 41 -
.../hbase/resolver/HbaseMetadataResolver.java | 130 ---
.../hbase/resolver/HbaseRequestResolver.java | 67 --
.../resolver/HbaseSensitivityTypeResolver.java | 58 --
.../hbase/TestHBaseMetadataResolver.java | 41 +
.../hbase/TestHbaseMetadataResolver.java | 41 -
.../extractor/TopologyExtractorFactory.java | 5 +-
.../extractor/hbase/HBaseTopologyCrawler.java | 58 ++
.../hbase/HBaseTopologyEntityParser.java | 176 ++++
.../extractor/hbase/HbaseTopologyCrawler.java | 61 --
.../hbase/HbaseTopologyEntityParser.java | 176 ----
....eagle.topology.TopologyCheckAppProvider.xml | 10 +-
.../topology/TestHBaseTopologyCrawler.java | 40 +
.../topology/TestHbaseTopologyCrawler.java | 40 -
.../eagle/topology/TopologyConstants.java | 2 +-
pom.xml | 2 +-
102 files changed, 3383 insertions(+), 1805 deletions(-)
----------------------------------------------------------------------
[15/17] eagle git commit: [EAGLE-1015] add an interface to add storm
configuration in an application
Posted by ja...@apache.org.
[EAGLE-1015] add an interface to add storm configuration in an application
https://issues.apache.org/jira/browse/EAGLE-1015
Support to add storm config value of type number or string in an application. However, to make storm overrides these custom values, one rule is the configuration must start with 'application.storm.'. For example:
`application.storm.workers` to override `workers`
`application.storm.nimbus.host` to override `nimbus.host`
Author: Zhao, Qingwen <qi...@apache.org>
Closes #928 from qingwen220/EAGLE-1015.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/47f00f15
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/47f00f15
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/47f00f15
Branch: refs/heads/branch-0.5
Commit: 47f00f159231958fb39748b4a6a01c4520371dec
Parents: eaad6cf
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Fri May 5 12:54:21 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Fri May 5 12:54:21 2017 +0800
----------------------------------------------------------------------
.../environment/impl/StormExecutionRuntime.java | 31 +++++++++++++++-----
1 file changed, 24 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/47f00f15/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
index 2b4180d..8045e43 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/impl/StormExecutionRuntime.java
@@ -22,6 +22,8 @@ import backtype.storm.generated.*;
import backtype.storm.utils.NimbusClient;
import com.google.common.base.Preconditions;
import com.typesafe.config.ConfigRenderOptions;
+import com.typesafe.config.ConfigValue;
+import org.apache.commons.lang.math.NumberUtils;
import org.apache.eagle.alert.engine.runner.StormMetricTaggedConsumer;
import org.apache.eagle.alert.metric.MetricConfigs;
import org.apache.eagle.app.Application;
@@ -36,6 +38,7 @@ import scala.Int;
import storm.trident.spout.RichSpoutBatchExecutor;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,StormTopology> {
@@ -67,14 +70,16 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
return this.environment;
}
- public static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs";
+ private static final String WORKERS = "workers";
+ private static final String TOPOLOGY_MESSAGE_TIMEOUT_SECS = "topology.message.timeout.secs";
private static final String STORM_NIMBUS_HOST_CONF_PATH = "application.storm.nimbusHost";
- private static final String STORM_NIMBUS_HOST_DEFAULT = "localhost";
- private static final Integer STORM_NIMBUS_THRIFT_DEFAULT = 6627;
private static final String STORM_NIMBUS_THRIFT_CONF_PATH = "application.storm.nimbusThriftPort";
- private static final String WORKERS = "workers";
+ private static final String APP_STORM_CONF_PATH_DEFAULT = "application.storm";
+
+ private static final String STORM_NIMBUS_HOST_DEFAULT = "localhost";
+ private static final Integer STORM_NIMBUS_THRIFT_DEFAULT = 6627;
private backtype.storm.Config getStormConfig(com.typesafe.config.Config config) {
backtype.storm.Config conf = new backtype.storm.Config();
@@ -85,16 +90,17 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Int.box(20480000));
String nimbusHost = STORM_NIMBUS_HOST_DEFAULT;
+
if (environment.config().hasPath(STORM_NIMBUS_HOST_CONF_PATH)) {
nimbusHost = environment.config().getString(STORM_NIMBUS_HOST_CONF_PATH);
- LOG.info("Overriding {} = {}",STORM_NIMBUS_HOST_CONF_PATH,nimbusHost);
+ LOG.info("Overriding {} = {}",STORM_NIMBUS_HOST_CONF_PATH, nimbusHost);
} else {
- LOG.info("Using default {} = {}",STORM_NIMBUS_HOST_CONF_PATH,STORM_NIMBUS_HOST_DEFAULT);
+ LOG.info("Using default {} = {}",STORM_NIMBUS_HOST_CONF_PATH, STORM_NIMBUS_HOST_DEFAULT);
}
Integer nimbusThriftPort = STORM_NIMBUS_THRIFT_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_THRIFT_CONF_PATH)) {
nimbusThriftPort = environment.config().getInt(STORM_NIMBUS_THRIFT_CONF_PATH);
- LOG.info("Overriding {} = {}",STORM_NIMBUS_THRIFT_CONF_PATH,nimbusThriftPort);
+ LOG.info("Overriding {} = {}",STORM_NIMBUS_THRIFT_CONF_PATH, nimbusThriftPort);
} else {
LOG.info("Using default {} = {}",STORM_NIMBUS_THRIFT_CONF_PATH,STORM_NIMBUS_THRIFT_DEFAULT);
}
@@ -112,6 +118,17 @@ public class StormExecutionRuntime implements ExecutionRuntime<StormEnvironment,
if (config.hasPath(MetricConfigs.METRIC_SINK_CONF)) {
conf.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1);
}
+
+ if (config.hasPath(APP_STORM_CONF_PATH_DEFAULT)) {
+ com.typesafe.config.Config appStormConf = config.getConfig(APP_STORM_CONF_PATH_DEFAULT);
+ for (Map.Entry<String, ConfigValue> entry: appStormConf.entrySet()) {
+ if (NumberUtils.isNumber(entry.getValue().unwrapped().toString())) {
+ conf.put(entry.getKey(), appStormConf.getNumber(entry.getKey()));
+ } else {
+ conf.put(entry.getKey(), entry.getValue().unwrapped());
+ }
+ }
+ }
return conf;
}
[14/17] eagle git commit: [EAGLE-1014] add exception handling in
CorrelationSpout.java
Posted by ja...@apache.org.
[EAGLE-1014] add exception handling in CorrelationSpout.java
https://issues.apache.org/jira/browse/EAGLE-1014
Author: Zhao, Qingwen <qi...@apache.org>
Closes #927 from qingwen220/EAGLE-1014.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/eaad6cf7
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/eaad6cf7
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/eaad6cf7
Branch: refs/heads/branch-0.5
Commit: eaad6cf74c896a97a061a3f600a0ec64a95c0963
Parents: 84d40ae
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Tue May 2 11:29:46 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Tue May 2 11:29:46 2017 -0700
----------------------------------------------------------------------
.../eagle/alert/engine/spout/CorrelationSpout.java | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/eaad6cf7/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
index e9ee892..4338964 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/spout/CorrelationSpout.java
@@ -172,6 +172,7 @@ public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener
wrapper.nextTuple();
} catch (Exception e) {
LOG.error("unexpected exception is caught: {}", e.getMessage(), e);
+ collector.reportError(e);
}
}
@@ -256,9 +257,14 @@ public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener
LOG.warn(MessageFormat.format("try to create new topic {0}, but found in the active spout list, this may indicate some inconsistency", topic));
continue;
}
- KafkaSpoutWrapper newWrapper = createKafkaSpout(ConfigFactory.parseMap(dataSourceProperties.get(topic)).withFallback(this.config),
- conf, context, collector, topic, newSchemaName.get(topic), newMeta, sds);
- newKafkaSpoutList.put(topic, newWrapper);
+ try {
+ KafkaSpoutWrapper newWrapper = createKafkaSpout(ConfigFactory.parseMap(dataSourceProperties.get(topic)).withFallback(this.config),
+ conf, context, collector, topic, newSchemaName.get(topic), newMeta, sds);
+ newKafkaSpoutList.put(topic, newWrapper);
+ } catch (Exception e) {
+ LOG.error("fail to create KafkaSpoutWrapper for topic {} due to {}", topic, e.getMessage(), e);
+ collector.reportError(e);
+ }
}
// iterate remove topics and then close KafkaSpout
for (String topic : removeTopics) {
@@ -285,6 +291,8 @@ public class CorrelationSpout extends BaseRichSpout implements SpoutSpecListener
this.cachedSpoutSpec = newMeta;
this.kafkaSpoutList = newKafkaSpoutList;
this.sds = sds;
+
+ LOG.info("after CorrelationSpout reloads, {} kafkaSpouts are generated for {} topics", kafkaSpoutList.size(), topics.size());
}
/**
[05/17] eagle git commit: [EAGLE-998] Add eagle csd
Posted by ja...@apache.org.
[EAGLE-998] Add eagle csd
Add eagle csd
https://issues.apache.org/jira/browse/EAGLE-998
Author: chitin <ch...@gmail.com>
Closes #908 from chitin/eaglecdh.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/391c6723
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/391c6723
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/391c6723
Branch: refs/heads/branch-0.5
Commit: 391c6723b3281ac0a4f41b1f87760216dfa08102
Parents: d182819
Author: chitin <ch...@gmail.com>
Authored: Tue Apr 18 01:00:23 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Tue Apr 18 01:00:23 2017 -0700
----------------------------------------------------------------------
eagle-external/eagle-cdh/README.md | 47 +
eagle-external/eagle-cdh/assembly.xml | 33 +
eagle-external/eagle-cdh/pom.xml | 51 ++
.../eagle-cdh/src/descriptor/service.sdl | 853 +++++++++++++++++++
eagle-external/eagle-cdh/src/images/icon.png | Bin 0 -> 3346 bytes
eagle-external/eagle-cdh/src/scripts/control.sh | 36 +
eagle-external/pom.xml | 1 +
7 files changed, 1021 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/eagle-cdh/README.md
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/README.md b/eagle-external/eagle-cdh/README.md
new file mode 100644
index 0000000..d626aa4
--- /dev/null
+++ b/eagle-external/eagle-cdh/README.md
@@ -0,0 +1,47 @@
+EAGLE Parcel and CSD for Cloudera's CDH 5
+=============
+This Eagle Parcel and CSD greatly simplifies setting up a Eagle cluster on your Cloudera cluster.
+
+## Table of Contents
+
+- [Eagle Parcel and CSD for Cloudera's CDH 5](#eagle-parcel-and-csd-for-clouderas-cdh-5)
+ - [Preparation](#preparation)
+ - [1. Create Eagle Parcel](#1-create-eagle-parcel)
+ - [Installation](#installation)
+ - [1. Install the CSD](#1-install-the-csd)
+ - [2. Downloading and Distributing Eagle Parcel](#2-downloading-and-distributing-eagle-parcel)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+### Preparation
+
+#### 1. Create Eagle Parcel
+
+1. Download the eagle realse and decompression it.
+2. Modify "eagle.conf" to "eagle.properties" in eagle-server.sh.Modify log path to specified path in server.yml.
+3. Build the eagle parcel, please refer to [Building-a-parcel](https://github.com/cloudera/cm_ext/wiki/Building-a-parcel).
+
+### Installation
+
+#### 1. Install the CSD
+
+1. In /apache-eagle/eagle-external/eagle-cdh,you can build it using the maven command "mvn assembly:assembly" and use the jar compiled in target folder
+2. Upload the CSD jar to your Cloudera Manager node, place it in /opt/cloudera/csd/
+3. Restart cloudera-scm-manager on Clouder Manager node:
+
+```bash
+service cloudera-scm-server restart
+```
+#### 2. Downloading and Distributing Eagle Parcel
+
+1. Download your parcel file,generate .sha, and mainfest.json.
+2. Copy your parcel files(.parcel, .sha, mainfest.json) to host /var/www/html/eagle
+3. Go to your Cloudera Manager: http://cloudera_host:7180 and login with your admin credentials
+4. Click on the parcels icon on the top right corner (next to the search bar)
+5. Click on "Edit Settings"
+6. Add a new value in "Remote Parcel Repository URLs", enter the IP address for the machine on which you just setup your HTTP server, for example: http://hostname/eagle
+7. Save your changes and go back to Parcels page by clicking on the Parcels icon on the top right corner (next to the search bar)
+8. Click on "Check for New Parcels"
+9. You should see "Eagle" parcel in the list. Download it, then distribute it and activate it
+10. You will be prompted to restart your cluster, click "Restart". Once that's done, you will need to restart "Cloudera Management Service" manually as well
+11. Now you should be able to simply add your new Eagle service through CM. (click on the little arrow next your cluster name and click "Add Service", follow the wizard instructions)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/eagle-cdh/assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/assembly.xml b/eagle-external/eagle-cdh/assembly.xml
new file mode 100644
index 0000000..5eb9daf
--- /dev/null
+++ b/eagle-external/eagle-cdh/assembly.xml
@@ -0,0 +1,33 @@
+<!--
+# Copyright (c) 2015 Rakuten Marketing, LLC
+# Licensed to Rakuten Marketing, LLC under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. Rakuten Marketing licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Author: Aiman Najjar <ai...@rakuten.com>
+-->
+<assembly>
+ <id>assemble</id>
+ <formats>
+ <format>jar</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src</directory>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/eagle-cdh/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/pom.xml b/eagle-external/eagle-cdh/pom.xml
new file mode 100644
index 0000000..52b0ccd
--- /dev/null
+++ b/eagle-external/eagle-cdh/pom.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>eagle-external-parent</artifactId>
+ <groupId>org.apache.eagle</groupId>
+ <version>0.5.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>eagle-cdh</artifactId>
+ <name>Eagle::External::EagleCDH</name>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>assembly.xml</descriptor>
+ </descriptors>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/eagle-cdh/src/descriptor/service.sdl
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/src/descriptor/service.sdl b/eagle-external/eagle-cdh/src/descriptor/service.sdl
new file mode 100644
index 0000000..786d864
--- /dev/null
+++ b/eagle-external/eagle-cdh/src/descriptor/service.sdl
@@ -0,0 +1,853 @@
+{
+ "name": "EAGLE",
+ "label": "Eagle",
+ "description": "Analyze Big Data Platforms For Security and Performance",
+ "version": "0.5",
+ "icon": "images/icon.png",
+ "inExpressWizard": true,
+ "runAs": {
+ "user": "eagle",
+ "group": "eagle"
+ },
+ "rolesWithExternalLinks": [
+ "EAGLE_SERVER"
+ ],
+ "parameters": [
+ {
+ "name": "service_env",
+ "label": "service_env",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.env",
+ "required": "true",
+ "type": "string",
+ "default": "testing"
+ },
+ {
+ "name": "service_host",
+ "label": "service_host",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.host",
+ "required": "true",
+ "type": "string",
+ "default": "localhost"
+ },
+ {
+ "name": "service_port",
+ "label": "service_port",
+ "description": "See Eagle documentation for description of this parameter",
+ "required": "true",
+ "type": "port",
+ "default": "9090"
+ },
+ {
+ "name": "service_username",
+ "label": "service_username",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.username",
+ "required": "true",
+ "type": "string",
+ "default": "admin"
+ },
+ {
+ "name": "service_password",
+ "label": "service_password",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.password",
+ "required": "true",
+ "type": "string",
+ "default": "secret"
+ },
+ {
+ "name": "service_readTimeOutSeconds",
+ "label": "service_readTimeOutSeconds",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.readTimeOutSeconds",
+ "required": "true",
+ "type": "long",
+ "default": "60"
+ },
+ {
+ "name": "service_context",
+ "label": "service_context",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.context",
+ "required": "true",
+ "type": "string",
+ "default": "/rest"
+ },
+ {
+ "name": "service_timezone",
+ "label": "service_timezone",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "service.timezone",
+ "required": "true",
+ "type": "string",
+ "default": "UTC"
+ },
+ {
+ "name": "zookeeper_zkQuorum",
+ "label": "zookeeper_zkQuorum",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "zookeeper.zkQuorum",
+ "required": "true",
+ "type": "string",
+ "default": "localhost:2181"
+ },
+ {
+ "name": "zookeeper_zkSessionTimeoutMs",
+ "label": "zookeeper_zkSessionTimeoutMs",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "zookeeper.zkSessionTimeoutMs",
+ "required": "true",
+ "type": "long",
+ "default": "15000"
+ },
+ {
+ "name": "zookeeper_zkRetryTimes",
+ "label": "zookeeper_zkRetryTimes",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "zookeeper.zkRetryTimes",
+ "required": "true",
+ "type": "long",
+ "default": "3"
+ },
+ {
+ "name": "zookeeper_zkRetryInterval",
+ "label": "zookeeper_zkRetryInterval",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "zookeeper.zkRetryInterval",
+ "required": "true",
+ "type": "long",
+ "default": "20000"
+ },
+ {
+ "name": "storage_type",
+ "label": "storage_type",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.type",
+ "required": "true",
+ "type": "string",
+ "default": "hbase"
+ },
+ {
+ "name": "storage_hbase_autoCreateTable",
+ "label": "storage_hbase_autoCreateTable",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName" : "storage.hbase.autoCreateTable",
+ "required": "true",
+ "type": "boolean",
+ "default": "true"
+ },
+ {
+ "name": "storage_hbase_zookeeperQuorum",
+ "label": "storage_hbase_zookeeperQuorum",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName" : "storage.hbase.zookeeperQuorum",
+ "required": "true",
+ "type": "string",
+ "default": "localhost"
+ },
+ {
+ "name": "storage_hbase_zookeeperPropertyClientPort",
+ "label": "storage_hbase_zookeeperPropertyClientPort",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName" : "storage.hbase.zookeeperPropertyClientPort",
+ "required": "true",
+ "type": "port",
+ "default": "2181"
+ },
+ {
+ "name": "storage_hbase_zookeeperZnodeParent",
+ "label": "storage_hbase_zookeeperZnodeParent",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName" : "storage.hbase.zookeeperZnodeParent",
+ "required": "true",
+ "type": "string",
+ "default": "/hbase-unsecure"
+ },
+ {
+ "name": "storage_hbase_tableNamePrefixedWithEnvironment",
+ "label": "storage_hbase_tableNamePrefixedWithEnvironment",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName" : "storage.hbase.tableNamePrefixedWithEnvironment",
+ "required": "true",
+ "type": "boolean",
+ "default": "false"
+ },
+ {
+ "name": "storage_hbase_coprocessorEnabled",
+ "label": "storage_hbase_coprocessorEnabled",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName" : "storage.hbase.coprocessorEnabled",
+ "required": "true",
+ "type": "boolean",
+ "default": "false"
+ },
+ {
+ "name": "storage_jdbc_adapter",
+ "label": "storage_jdbc_adapter",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.adapter",
+ "required": "true",
+ "type": "string",
+ "default": "mysql"
+ },
+ {
+ "name": "storage_jdbc_username",
+ "label": "storage_jdbc_username",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.username",
+ "required": "true",
+ "type": "string",
+ "default": "root"
+ },
+ {
+ "name": "storage_jdbc_password",
+ "label": "storage_jdbc_password",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.password",
+ "required": "true",
+ "type": "password",
+ "default": "root"
+ },
+ {
+ "name": "storage_jdbc_database",
+ "label": "storage_jdbc_database",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.database",
+ "required": "true",
+ "type": "string",
+ "default": "eagle"
+ },
+ {
+ "name": "storage_jdbc_connectionUrl",
+ "label": "storage_jdbc_connectionUrl",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.connectionUrl",
+ "required": "true",
+ "type": "string",
+ "default": "jdbc:mysql://localhost:3306/eagle"
+ },
+ {
+ "name": "storage_jdbc_connectionProps",
+ "label": "storage_jdbc_connectionProps",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.connectionProps",
+ "required": "true",
+ "type": "string",
+ "default": "encoding=UTF-8"
+ },
+ {
+ "name": "storage_jdbc_driverClass",
+ "label": "storage_jdbc_driverClass",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.driverClass",
+ "required": "true",
+ "type": "string",
+ "default": "com.mysql.jdbc.Driver"
+ },
+ {
+ "name": "storage_jdbc_connectionMax",
+ "label": "storage_jdbc_connectionMax",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "storage.jdbc.connectionMax",
+ "required": "true",
+ "type": "long",
+ "default": "8"
+ },
+ {
+ "name": "metadata_store",
+ "label": "metadata_store",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.store",
+ "required": "true",
+ "type": "string",
+ "default": "org.apache.eagle.metadata.service.memory.MemoryMetadataStore"
+ },
+ {
+ "name": "metadata_jdbc_connectionProperties",
+ "label": "metadata_jdbc_connectionProperties",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.jdbc.connectionProperties",
+ "required": "true",
+ "type": "string",
+ "default": "encoding=UTF8"
+ },
+ {
+ "name": "metadata_jdbc_username",
+ "label": "metadata_jdbc_username",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.jdbc.username",
+ "required": "true",
+ "type": "string",
+ "default": "root"
+ },
+ {
+ "name": "metadata_jdbc_password",
+ "label": "metadata_jdbc_password",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.jdbc.password",
+ "required": "true",
+ "type": "password",
+ "default": "root"
+ },
+ {
+ "name": "metadata_jdbc_connection",
+ "label": "metadata_jdbc_connection",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.jdbc.connection",
+ "required": "true",
+ "type": "string",
+ "default": "jdbc:mysql://localhost:3306/eagle"
+ },
+ {
+ "name": "metadata_jdbc_database",
+ "label": "metadata_jdbc_database",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.jdbc.database",
+ "required": "true",
+ "type": "string",
+ "default": "eagle"
+ },
+ {
+ "name": "metadata_jdbc_driverClassName",
+ "label": "metadata_jdbc_driverClassName",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "metadata.jdbc.driverClassName",
+ "required": "true",
+ "type": "string",
+ "default": "com.mysql.jdbc.Driver"
+ },
+ {
+ "name": "application_stream_provider",
+ "label": "application_stream_provider",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.stream.provider",
+ "required": "true",
+ "type": "string",
+ "default": "org.apache.eagle.app.messaging.KafkaStreamProvider"
+ },
+ {
+ "name": "application_storm_nimbusHost",
+ "label": "application_storm_nimbusHost",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.storm.nimbusHost",
+ "required": "true",
+ "type": "string",
+ "default": "server.eagle.apache.org"
+ },
+ {
+ "name": "application_storm_nimbusThriftPort",
+ "label": "application_storm_nimbusThriftPort",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.storm.nimbusThriftPort",
+ "required": "true",
+ "type": "long",
+ "default": "6627"
+ },
+ {
+ "name": "application_updateStatus_initialDelay",
+ "label": "application_updateStatus_initialDelay",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.updateStatus.initialDelay",
+ "required": "true",
+ "type": "long",
+ "default": "10"
+ },
+ {
+ "name": "application_updateStatus_period",
+ "label": "application_updateStatus_period",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.updateStatus.period",
+ "required": "true",
+ "type": "long",
+ "default": "10"
+ },
+ {
+ "name": "application_healthCheck_initialDelay",
+ "label": "application_healthCheck_initialDelay",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.initialDelay",
+ "required": "true",
+ "type": "long",
+ "default": "30"
+ },
+ {
+ "name": "application_healthCheck_publisher_publisherImpl",
+ "label": "application_healthCheck_publisher_publisherImpl",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.publisherImpl",
+ "required": "true",
+ "type": "string",
+ "default": "org.apache.eagle.app.service.impl.ApplicationHealthCheckEmailPublisher"
+ },
+ {
+ "name": "application_healthCheck_publisher_dailySendHour",
+ "label": "application_healthCheck_publisher_dailySendHour",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.dailySendHour",
+ "required": "true",
+ "type": "long",
+ "default": "11"
+ },
+ {
+ "name": "application_healthCheck_publisher_mail_smtp_host",
+ "label": "application_healthCheck_publisher_mail_smtp_host",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.mail.smtp.host",
+ "required": "true",
+ "type": "string",
+ "default": "mail.host.com"
+ },
+ {
+ "name": "application_healthCheck_publisher_mail_smtp_port",
+ "label": "application_healthCheck_publisher_mail_smtp_port",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.mail.smtp.port",
+ "required": "true",
+ "type": "long",
+ "default": "25"
+ },
+ {
+ "name": "application_healthCheck_publisher_mail_smtp_recipients",
+ "label": "application_healthCheck_publisher_mail_smtp_recipients",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.mail.smtp.recipients",
+ "required": "true",
+ "type": "string",
+ "default": "someone@email.com"
+ },
+ {
+ "name": "application_healthCheck_publisher_mail_smtp_subject",
+ "label": "application_healthCheck_publisher_mail_smtp_subject",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.mail.smtp.subject",
+ "required": "true",
+ "type": "string",
+ "default": "Eagle Application Health Check"
+ },
+ {
+ "name": "application_healthCheck_publisher_mail_smtp_template",
+ "label": "application_healthCheck_publisher_mail_smtp_template",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.healthCheck.publisher.mail.smtp.template",
+ "required": "true",
+ "type": "string",
+ "default": "HealthCheckTemplate.vm"
+ },
+ {
+ "name": "application_mailService_mailSmtpServer",
+ "label": "application_mailService_mailSmtpServer",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.mailService.mailSmtpServer",
+ "required": "true",
+ "type": "string",
+ "default": ""
+ },
+ {
+ "name": "application_mailService_mailSmtpPort",
+ "label": "application_mailService_mailSmtpPort",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.mailService.mailSmtpPort",
+ "required": "true",
+ "type": "long",
+ "default": "25"
+ },
+ {
+ "name": "application_mailService_mailSmtpAuth",
+ "label": "application_mailService_mailSmtpAuth",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.mailService.mailSmtpAuth",
+ "required": "true",
+ "type": "boolean",
+ "default": "false"
+ },
+ {
+ "name": "application_dailyJobReport_reportHourTime",
+ "label": "application_dailyJobReport_reportHourTime",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.reportHourTime",
+ "required": "true",
+ "type": "long",
+ "default": "1"
+ },
+ {
+ "name": "application_dailyJobReport_reportPeriodInHour",
+ "label": "application_dailyJobReport_reportPeriodInHour",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.reportPeriodInHour",
+ "required": "true",
+ "type": "long",
+ "default": "12"
+ },
+ {
+ "name": "application_dailyJobReport_numTopUsers",
+ "label": "application_dailyJobReport_numTopUsers",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.numTopUsers",
+ "required": "true",
+ "type": "long",
+ "default": "10"
+ },
+ {
+ "name": "application_dailyJobReport_jobOvertimeLimitInHour",
+ "label": "application_dailyJobReport_jobOvertimeLimitInHour",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.jobOvertimeLimitInHour",
+ "required": "true",
+ "type": "long",
+ "default": "6"
+ },
+ {
+ "name": "application_dailyJobReport_subject",
+ "label": "application_dailyJobReport_subject",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.subject",
+ "required": "true",
+ "type": "string",
+ "default": "Job Report For 12 hours"
+ },
+ {
+ "name": "application_dailyJobReport_recipients",
+ "label": "application_dailyJobReport_recipients",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.recipients",
+ "required": "true",
+ "type": "string",
+ "default": "someone@email.com"
+ },
+ {
+ "name": "application_dailyJobReport_template",
+ "label": "application_dailyJobReport_template",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.dailyJobReport.template",
+ "required": "true",
+ "type": "string",
+ "default": "JobReportTemplate.vm"
+ },
+ {
+ "name": "application_analyzerReport_sender",
+ "label": "application_analyzerReport_sender",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.analyzerReport.sender",
+ "required": "true",
+ "type": "string",
+ "default": "nobody@abc.com"
+ },
+ {
+ "name": "application_analyzerReport_recipients",
+ "label": "application_analyzerReport_recipients",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.analyzerReport.recipients",
+ "required": "true",
+ "type": "string",
+ "default": "nobody@abc.com"
+ },
+ {
+ "name": "application_analyzerReport_template",
+ "label": "application_analyzerReport_template",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.analyzerReport.template",
+ "required": "true",
+ "type": "string",
+ "default": "AnalyzerReportTemplate.vm"
+ },
+ {
+ "name": "application_analyzerReport_cc",
+ "label": "application_analyzerReport_cc",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "application.analyzerReport.cc",
+ "required": "true",
+ "type": "string",
+ "default": "nobody@abc.com"
+ },
+ {
+ "name": "coordinator_policiesPerBolt",
+ "label": "coordinator_policiesPerBolt",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.policiesPerBolt",
+ "required": "true",
+ "type": "long",
+ "default": "5"
+ },
+ {
+ "name": "coordinator_boltParallelism",
+ "label": "coordinator_boltParallelism",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.boltParallelism",
+ "required": "true",
+ "type": "long",
+ "default": "5"
+ },
+ {
+ "name": "coordinator_policyDefaultParallelism",
+ "label": "coordinator_policyDefaultParallelism",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.policyDefaultParallelism",
+ "required": "true",
+ "type": "long",
+ "default": "5"
+ },
+ {
+ "name": "coordinator_boltLoadUpbound",
+ "label": "coordinator_boltLoadUpbound",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.boltLoadUpbound",
+ "required": "true",
+ "type": "double",
+ "default": "0.8"
+ },
+ {
+ "name": "coordinator_topologyLoadUpbound",
+ "label": "coordinator_topologyLoadUpbound",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.topologyLoadUpbound",
+ "required": "true",
+ "type": "double",
+ "default": "0.8"
+ },
+ {
+ "name": "coordinator_numOfAlertBoltsPerTopology",
+ "label": "coordinator_numOfAlertBoltsPerTopology",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.numOfAlertBoltsPerTopology",
+ "required": "true",
+ "type": "long",
+ "default": "5"
+ },
+ {
+ "name": "coordinator_zkConfig_zkQuorum",
+ "label": "coordinator_zkConfig_zkQuorum",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.zkConfig.zkQuorum",
+ "required": "true",
+ "type": "string",
+ "default": "server.eagle.apache.org:2181"
+ },
+ {
+ "name": "coordinator_zkConfig_zkRoot",
+ "label": "coordinator_zkConfig_zkRoot",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.zkConfig.zkRoot",
+ "required": "true",
+ "type": "string",
+ "default": "/alert"
+ },
+ {
+ "name": "coordinator_zkConfig_zkSessionTimeoutMs",
+ "label": "coordinator_zkConfig_zkSessionTimeoutMs",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.zkConfig.zkSessionTimeoutMs",
+ "required": "true",
+ "type": "long",
+ "default": "10000"
+ },
+ {
+ "name": "coordinator_zkConfig_connectionTimeoutMs",
+ "label": "coordinator_zkConfig_connectionTimeoutMs",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.zkConfig.connectionTimeoutMs",
+ "required": "true",
+ "type": "long",
+ "default": "10000"
+ },
+ {
+ "name": "coordinator_zkConfig_zkRetryTimes",
+ "label": "coordinator_zkConfig_zkRetryTimes",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.zkConfig.zkRetryTimes",
+ "required": "true",
+ "type": "long",
+ "default": "3"
+ },
+ {
+ "name": "coordinator_zkConfig_zkRetryInterval",
+ "label": "coordinator_zkConfig_zkRetryInterval",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.zkConfig.zkRetryInterval",
+ "required": "true",
+ "type": "long",
+ "default": "3000"
+ },
+ {
+ "name": "coordinator_metadataService_host",
+ "label": "coordinator_metadataService_host",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataService.host",
+ "required": "true",
+ "type": "string",
+ "default": "localhost"
+ },
+ {
+ "name": "coordinator_metadataService_port",
+ "label": "coordinator_metadataService_port",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataService.port",
+ "required": "true",
+ "type": "port",
+ "default": "9090"
+ },
+ {
+ "name": "coordinator_metadataService_context",
+ "label": "coordinator_metadataService_context",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataService.context",
+ "required": "true",
+ "type": "string",
+ "default": "/rest"
+ },
+ {
+ "name": "coordinator_metadataDynamicCheck_initDelayMillis",
+ "label": "coordinator_metadataDynamicCheck_initDelayMillis",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataDynamicCheck.initDelayMillis",
+ "required": "true",
+ "type": "long",
+ "default": "1000"
+ },
+ {
+ "name": "coordinator_metadataDynamicCheck_delayMillis",
+ "label": "coordinator_metadataDynamicCheck_delayMillis",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataDynamicCheck.delayMillis",
+ "required": "true",
+ "type": "long",
+ "default": "30000"
+ },
+ {
+ "name": "coordinator_metadataDynamicCheck_stateClearPeriodMin",
+ "label": "coordinator_metadataDynamicCheck_stateClearPeriodMin",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataDynamicCheck.stateClearPeriodMin",
+ "required": "true",
+ "type": "long",
+ "default": "1440"
+ },
+ {
+ "name": "coordinator_metadataDynamicCheck_stateReservedCapacity",
+ "label": "coordinator_metadataDynamicCheck_stateReservedCapacity",
+ "description": "See Eagle documentation for description of this parameter",
+ "configName": "coordinator.metadataDynamicCheck.stateReservedCapacity",
+ "required": "true",
+ "type": "long",
+ "default": "100"
+ }
+ ],
+ "roles": [
+ {
+ "name": "EAGLE_SERVER",
+ "label": "Eagle Server",
+ "pluralLabel": "Web Servers",
+ "externalLink": {
+ "name": "eagle_web",
+ "label": "Eagle Web",
+ "url": "http://${host}:${service_port}"
+ },
+ "startRunner": {
+ "program": "scripts/control.sh",
+ "args": [
+ "start"
+ ],
+ "environmentVariables": {
+ "WEBSERVER_PORT": "${service_port}"
+ }
+ },
+ "logging": {
+ "dir": "/var/log/eagle",
+ "filename": "eagle-server.log",
+ "isModifiable": true,
+ "configName": "log.dir"
+ },
+ "configWriter": {
+ "generators": [
+ {
+ "filename": "eagle.properties",
+ "configFormat": "properties",
+ "includedParams": [
+ "service_env",
+ "service_host",
+ "service_port",
+ "service_username",
+ "service_password",
+ "service_readTimeOutSeconds",
+ "service_context",
+ "service_timezone",
+ "zookeeper_zkQuorum",
+ "zookeeper_zkSessionTimeoutMs",
+ "zookeeper_zkRetryTimes",
+ "zookeeper_zkRetryInterval",
+ "storage_type",
+ "storage_hbase_autoCreateTable",
+ "storage_hbase_zookeeperQuorum",
+ "storage_hbase_zookeeperPropertyClientPort",
+ "storage_hbase_zookeeperZnodeParent",
+ "storage_hbase_tableNamePrefixedWithEnvironment",
+ "storage_hbase_coprocessorEnabled",
+ "storage_jdbc_adapter",
+ "storage_jdbc_username",
+ "storage_jdbc_password",
+ "storage_jdbc_database",
+ "storage_jdbc_connectionUrl",
+ "storage_jdbc_connectionProps",
+ "storage_jdbc_driverClass",
+ "storage_jdbc_connectionMax",
+ "metadata_store",
+ "metadata_jdbc_connectionProperties",
+ "metadata_jdbc_username",
+ "metadata_jdbc_password",
+ "metadata_jdbc_connection",
+ "metadata_jdbc_database",
+ "metadata_jdbc_driverClassName",
+ "application_stream_provider",
+ "application_storm_nimbusHost",
+ "application_storm_nimbusThriftPort",
+ "application_updateStatus_initialDelay",
+ "application_updateStatus_period",
+ "application_healthCheck_initialDelay",
+ "application_healthCheck_publisher_publisherImpl",
+ "application_healthCheck_publisher_dailySendHour",
+ "application_healthCheck_publisher_mail_smtp_host",
+ "application_healthCheck_publisher_mail_smtp_port",
+ "application_healthCheck_publisher_mail_smtp_recipients",
+ "application_healthCheck_publisher_mail_smtp_subject",
+ "application_healthCheck_publisher_mail_smtp_template",
+ "application_mailService_mailSmtpServer",
+ "application_mailService_mailSmtpPort",
+ "application_mailService_mailSmtpAuth",
+ "application_dailyJobReport_reportHourTime",
+ "application_dailyJobReport_reportPeriodInHour",
+ "application_dailyJobReport_numTopUsers",
+ "application_dailyJobReport_jobOvertimeLimitInHour",
+ "application_dailyJobReport_subject",
+ "application_dailyJobReport_recipients",
+ "application_dailyJobReport_template",
+ "application_analyzerReport_sender",
+ "application_analyzerReport_recipients",
+ "application_analyzerReport_template",
+ "application_analyzerReport_cc",
+ "coordinator_policiesPerBolt",
+ "coordinator_boltParallelism",
+ "coordinator_policyDefaultParallelism",
+ "coordinator_boltLoadUpbound",
+ "coordinator_topologyLoadUpbound",
+ "coordinator_numOfAlertBoltsPerTopology",
+ "coordinator_zkConfig_zkQuorum",
+ "coordinator_zkConfig_zkRoot",
+ "coordinator_zkConfig_zkSessionTimeoutMs",
+ "coordinator_zkConfig_connectionTimeoutMs",
+ "coordinator_zkConfig_zkRetryTimes",
+ "coordinator_zkConfig_zkRetryInterval",
+ "coordinator_metadataService_host",
+ "coordinator_metadataService_port",
+ "coordinator_metadataService_context",
+ "coordinator_metadataDynamicCheck_initDelayMillis",
+ "coordinator_metadataDynamicCheck_delayMillis",
+ "coordinator_metadataDynamicCheck_stateClearPeriodMin",
+ "coordinator_metadataDynamicCheck_stateReservedCapacity"
+ ]
+ }
+ ]
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/eagle-cdh/src/images/icon.png
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/src/images/icon.png b/eagle-external/eagle-cdh/src/images/icon.png
new file mode 100644
index 0000000..57fce5d
Binary files /dev/null and b/eagle-external/eagle-cdh/src/images/icon.png differ
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/eagle-cdh/src/scripts/control.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/src/scripts/control.sh b/eagle-external/eagle-cdh/src/scripts/control.sh
new file mode 100644
index 0000000..ad4230c
--- /dev/null
+++ b/eagle-external/eagle-cdh/src/scripts/control.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+CMD=$1
+
+cp -r /opt/cloudera/parcels/EAGLE/* $CONF_DIR
+mv -f $CONF_DIR/eagle.properties $CONF_DIR/conf
+
+case $CMD in
+ (start)
+ echo "Starting the web server on port [$WEBSERVER_PORT]"
+ cmd="bin/eagle-server.sh start"
+ exec ${cmd}
+ ;;
+ (stop)
+ pkill -u eagle -f ServerMain
+ ;;
+ (*)
+ echo "Don't understand [$CMD]"
+ exit 1
+ ;;
+esac
http://git-wip-us.apache.org/repos/asf/eagle/blob/391c6723/eagle-external/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-external/pom.xml b/eagle-external/pom.xml
index 9b1a850..6ba1278 100644
--- a/eagle-external/pom.xml
+++ b/eagle-external/pom.xml
@@ -32,5 +32,6 @@
<modules>
<module>eagle-log4jkafka</module>
<module>eagle-kafka</module>
+ <module>eagle-cdh</module>
</modules>
</project>
\ No newline at end of file
[07/17] eagle git commit: HBase Naming that unify `Hbase` and `HBase`
into `HBase`
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
deleted file mode 100644
index 467cc2f..0000000
--- a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHbaseAuditLogParser.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-
-public class TestHbaseAuditLogParser {
- HbaseAuditLogParser parser = new HbaseAuditLogParser();
-
- @Test
- public void test() throws Exception {
- String log = "2015-08-11 13:31:03,729 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Table permission granted; remote address: /127.0.0.1; request: get; context: (user=eagle,scope=hbase:namespace,family=info, action=READ)";
- HbaseAuditLogObject obj = parser.parse(log);
- Assert.assertEquals(obj.action, "READ");
- Assert.assertEquals(obj.host, "127.0.0.1");
- Assert.assertEquals(obj.scope, "hbase:namespace:info");
- }
-
- @Test
- public void test2() throws Exception {
- String log = "2015-08-04 12:29:03,073 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Global check allowed; remote address: ; request: preOpen; context: (user=eagle, scope=GLOBAL, family=, action=ADMIN)";
- HbaseAuditLogObject obj = parser.parse(log);
- Assert.assertEquals(obj.action, "ADMIN");
- Assert.assertEquals(obj.host, "");
- Assert.assertEquals(obj.scope, "GLOBAL");
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java
new file mode 100644
index 0000000..91dd5a3
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResource.java
@@ -0,0 +1,189 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. See the NOTICE file distributed with
+ * * this work for additional information regarding copyright ownership.
+ * * The ASF licenses this file to You under the Apache License, Version 2.0
+ * * (the "License"); you may not use this file except in compliance with
+ * * the License. You may obtain a copy of the License at
+ * * <p/>
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * * <p/>
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License.
+ *
+ */
+package org.apache.eagle.service.security.hbase;
+
+import com.google.inject.Inject;
+import com.typesafe.config.Config;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+import org.apache.eagle.security.entity.HBaseResourceEntity;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.service.ISecurityMetadataDAO;
+import org.apache.eagle.security.service.MetadataDaoFactory;
+import org.apache.eagle.service.common.EagleExceptionWrapper;
+import org.apache.eagle.service.security.hbase.dao.HBaseMetadataDAOImpl;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.util.*;
+import java.util.regex.Pattern;
+
+@Path("/hbaseResource")
+public class HBaseMetadataBrowseWebResource {
+ private static Logger LOG = LoggerFactory.getLogger(HBaseMetadataBrowseWebResource.class);
+ final public static String HBASE_APPLICATION = "HBaseAuditLogApplication";
+
+ private ApplicationEntityService entityService;
+ private ISecurityMetadataDAO dao;
+
+ @Inject
+ public HBaseMetadataBrowseWebResource(ApplicationEntityService entityService, Config eagleServerConfig){
+ this.entityService = entityService;
+ this.dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig);
+ }
+
+ private Map<String, Map<String, String>> getAllSensitivities(){
+ Map<String, Map<String, String>> all = new HashMap<>();
+ Collection<HBaseSensitivityEntity> entities = dao.listHBaseSensitivities();
+ for(HBaseSensitivityEntity entity : entities){
+ if(!all.containsKey(entity.getSite())){
+ all.put(entity.getSite(), new HashMap<>());
+ }
+ all.get(entity.getSite()).put(entity.getHBaseResource(), entity.getSensitivityType());
+ }
+ return all;
+ }
+
+ private Map<String, Object> getAppConfig(String site, String appType){
+ ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType);
+ return entity.getConfiguration();
+ }
+
+
+ private Configuration convert(Map<String, Object> originalConfig) throws Exception {
+ Configuration config = new Configuration();
+ for (Map.Entry<String, Object> entry : originalConfig.entrySet()) {
+ config.set(entry.getKey(), entry.getValue().toString());
+ }
+ return config;
+ }
+
+ @Path("/namespaces")
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public HBaseMetadataBrowseWebResponse getNamespace(@QueryParam("site") String site) {
+ List<String> namespaces = null;
+ List<HBaseResourceEntity> values = new ArrayList<>();
+ HBaseMetadataBrowseWebResponse response = new HBaseMetadataBrowseWebResponse();
+ try {
+ Map<String, Object> config = getAppConfig(site, HBASE_APPLICATION);
+ Configuration conf = convert(config);
+ HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf);
+ namespaces = dao.getNamespaces();
+
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ if(namespaces != null) {
+ for (String ns : namespaces) {
+ Set<String> childSensitiveTypes = new HashSet<>();
+ String senstiveType = checkSensitivity(site, ns, childSensitiveTypes);
+ values.add(new HBaseResourceEntity(ns, ns, null, null, senstiveType, childSensitiveTypes));
+ }
+ }
+ response.setObj(values);
+ return response;
+ }
+
+ @Path("/tables")
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public HBaseMetadataBrowseWebResponse getTables(@QueryParam("site") String site, @QueryParam("namespace") String namespace){
+ // delegate to HiveMetadataDAO
+ HBaseMetadataBrowseWebResponse response = new HBaseMetadataBrowseWebResponse();
+ List<String> tables = null;
+ List<HBaseResourceEntity> values = new ArrayList<>();
+ try {
+ Map<String, Object> config = getAppConfig(site, HBASE_APPLICATION);
+ Configuration conf = convert(config);
+ HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf);
+ tables = dao.getTables(namespace);
+ }catch(Exception ex){
+ LOG.error("fail getting databases", ex);
+ response.setException(EagleExceptionWrapper.wrap(ex));
+ }
+ if(tables != null) {
+ for (String table : tables) {
+ String resource = String.format("%s:%s", namespace, table);
+ Set<String> childSensitiveTypes = new HashSet<>();
+ String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes);
+ values.add(new HBaseResourceEntity(resource, namespace, table, null, sensitiveType, childSensitiveTypes));
+ }
+ }
+ response.setObj(values);
+ return response;
+ }
+
+ @Path("/columns")
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON)
+ public HBaseMetadataBrowseWebResponse getColumns(@QueryParam("site") String site, @QueryParam("namespace") String namespace, @QueryParam("table") String table){
+ // delegate to HiveMetadataDAO
+ HBaseMetadataBrowseWebResponse response = new HBaseMetadataBrowseWebResponse();
+ List<String> columns = null;
+ List<HBaseResourceEntity> values = new ArrayList<>();
+ try {
+ Map<String, Object> config = getAppConfig(site, HBASE_APPLICATION);
+ Configuration conf = convert(config);
+ HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf);
+ String tableName = String.format("%s:%s", namespace, table);
+ columns = dao.getColumnFamilies(tableName);
+ }catch(Exception ex){
+ LOG.error("fail getting databases", ex);
+ response.setException(EagleExceptionWrapper.wrap(ex));
+ }
+ if(columns != null) {
+ for (String col : columns) {
+ String resource = String.format("%s:%s:%s", namespace, table, col);
+ Set<String> childSensitiveTypes = new HashSet<>();
+ String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes);
+ values.add(new HBaseResourceEntity(resource, namespace, table, col, sensitiveType, childSensitiveTypes));
+ }
+ }
+ response.setObj(values);
+ return response;
+ }
+
+
+
+ String checkSensitivity(String site, String resource, Set<String> childSensitiveTypes) {
+ Map<String, Map<String, String>> maps = getAllSensitivities();
+ String sensitiveType = null;
+ if (maps != null && maps.get(site) != null) {
+ Map<String, String> map = maps.get(site);
+ for (String r : map.keySet()) {
+ Pattern pattern = Pattern.compile("^" + resource);
+ boolean isMatched = Pattern.matches(r, resource);
+ if (isMatched) {
+ sensitiveType = map.get(r);
+ }
+ else if (pattern.matcher(r).find()){
+ childSensitiveTypes.add(map.get(r));
+ }
+ }
+ }
+ return sensitiveType;
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java
new file mode 100644
index 0000000..929989f
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HBaseMetadataBrowseWebResponse.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.service.security.hbase;
+
+
+import org.apache.eagle.security.entity.HBaseResourceEntity;
+
+import java.util.List;
+
+public class HBaseMetadataBrowseWebResponse {
+ private String exception;
+ private List<HBaseResourceEntity> obj;
+
+ public List<HBaseResourceEntity> getObj() {
+ return obj;
+ }
+
+ public void setObj(List<HBaseResourceEntity> obj) {
+ this.obj = obj;
+ }
+
+ public String getException() {
+ return exception;
+ }
+
+ public void setException(String exception) {
+ this.exception = exception;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
deleted file mode 100644
index 69be2e5..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResource.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. See the NOTICE file distributed with
- * * this work for additional information regarding copyright ownership.
- * * The ASF licenses this file to You under the Apache License, Version 2.0
- * * (the "License"); you may not use this file except in compliance with
- * * the License. You may obtain a copy of the License at
- * * <p/>
- * * http://www.apache.org/licenses/LICENSE-2.0
- * * <p/>
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License.
- *
- */
-package org.apache.eagle.service.security.hbase;
-
-import com.google.inject.Inject;
-import com.typesafe.config.Config;
-import org.apache.eagle.metadata.model.ApplicationEntity;
-import org.apache.eagle.metadata.service.ApplicationEntityService;
-import org.apache.eagle.security.entity.HbaseResourceEntity;
-import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
-import org.apache.eagle.security.service.HBaseSensitivityEntity;
-import org.apache.eagle.security.service.ISecurityMetadataDAO;
-import org.apache.eagle.security.service.MetadataDaoFactory;
-import org.apache.eagle.service.common.EagleExceptionWrapper;
-import org.apache.eagle.service.security.hbase.dao.HbaseMetadataDAOImpl;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.ws.rs.*;
-import javax.ws.rs.core.MediaType;
-import java.util.*;
-import java.util.regex.Pattern;
-
-@Path("/hbaseResource")
-public class HbaseMetadataBrowseWebResource {
- private static Logger LOG = LoggerFactory.getLogger(HbaseMetadataBrowseWebResource.class);
- final public static String HBASE_APPLICATION = "HBaseAuditLogApplication";
-
- private ApplicationEntityService entityService;
- private ISecurityMetadataDAO dao;
-
- @Inject
- public HbaseMetadataBrowseWebResource(ApplicationEntityService entityService, Config eagleServerConfig){
- this.entityService = entityService;
- this.dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig);
- }
-
- private Map<String, Map<String, String>> getAllSensitivities(){
- Map<String, Map<String, String>> all = new HashMap<>();
- Collection<HBaseSensitivityEntity> entities = dao.listHBaseSensitivities();
- for(HBaseSensitivityEntity entity : entities){
- if(!all.containsKey(entity.getSite())){
- all.put(entity.getSite(), new HashMap<>());
- }
- all.get(entity.getSite()).put(entity.getHbaseResource(), entity.getSensitivityType());
- }
- return all;
- }
-
- private Map<String, Object> getAppConfig(String site, String appType){
- ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType);
- return entity.getConfiguration();
- }
-
-
- private Configuration convert(Map<String, Object> originalConfig) throws Exception {
- Configuration config = new Configuration();
- for (Map.Entry<String, Object> entry : originalConfig.entrySet()) {
- config.set(entry.getKey(), entry.getValue().toString());
- }
- return config;
- }
-
- @Path("/namespaces")
- @GET
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public HbaseMetadataBrowseWebResponse getNamespace(@QueryParam("site") String site) {
- List<String> namespaces = null;
- List<HbaseResourceEntity> values = new ArrayList<>();
- HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse();
- try {
- Map<String, Object> config = getAppConfig(site, HBASE_APPLICATION);
- Configuration conf = convert(config);
- HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
- namespaces = dao.getNamespaces();
-
- } catch (Exception e) {
- e.printStackTrace();
- }
- if(namespaces != null) {
- for (String ns : namespaces) {
- Set<String> childSensitiveTypes = new HashSet<>();
- String senstiveType = checkSensitivity(site, ns, childSensitiveTypes);
- values.add(new HbaseResourceEntity(ns, ns, null, null, senstiveType, childSensitiveTypes));
- }
- }
- response.setObj(values);
- return response;
- }
-
- @Path("/tables")
- @GET
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public HbaseMetadataBrowseWebResponse getTables(@QueryParam("site") String site, @QueryParam("namespace") String namespace){
- // delegate to HiveMetadataDAO
- HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse();
- List<String> tables = null;
- List<HbaseResourceEntity> values = new ArrayList<>();
- try {
- Map<String, Object> config = getAppConfig(site, HBASE_APPLICATION);
- Configuration conf = convert(config);
- HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
- tables = dao.getTables(namespace);
- }catch(Exception ex){
- LOG.error("fail getting databases", ex);
- response.setException(EagleExceptionWrapper.wrap(ex));
- }
- if(tables != null) {
- for (String table : tables) {
- String resource = String.format("%s:%s", namespace, table);
- Set<String> childSensitiveTypes = new HashSet<>();
- String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes);
- values.add(new HbaseResourceEntity(resource, namespace, table, null, sensitiveType, childSensitiveTypes));
- }
- }
- response.setObj(values);
- return response;
- }
-
- @Path("/columns")
- @GET
- @Consumes(MediaType.APPLICATION_JSON)
- @Produces(MediaType.APPLICATION_JSON)
- public HbaseMetadataBrowseWebResponse getColumns(@QueryParam("site") String site, @QueryParam("namespace") String namespace, @QueryParam("table") String table){
- // delegate to HiveMetadataDAO
- HbaseMetadataBrowseWebResponse response = new HbaseMetadataBrowseWebResponse();
- List<String> columns = null;
- List<HbaseResourceEntity> values = new ArrayList<>();
- try {
- Map<String, Object> config = getAppConfig(site, HBASE_APPLICATION);
- Configuration conf = convert(config);
- HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
- String tableName = String.format("%s:%s", namespace, table);
- columns = dao.getColumnFamilies(tableName);
- }catch(Exception ex){
- LOG.error("fail getting databases", ex);
- response.setException(EagleExceptionWrapper.wrap(ex));
- }
- if(columns != null) {
- for (String col : columns) {
- String resource = String.format("%s:%s:%s", namespace, table, col);
- Set<String> childSensitiveTypes = new HashSet<>();
- String sensitiveType = checkSensitivity(site, resource, childSensitiveTypes);
- values.add(new HbaseResourceEntity(resource, namespace, table, col, sensitiveType, childSensitiveTypes));
- }
- }
- response.setObj(values);
- return response;
- }
-
-
-
- String checkSensitivity(String site, String resource, Set<String> childSensitiveTypes) {
- Map<String, Map<String, String>> maps = getAllSensitivities();
- String sensitiveType = null;
- if (maps != null && maps.get(site) != null) {
- Map<String, String> map = maps.get(site);
- for (String r : map.keySet()) {
- Pattern pattern = Pattern.compile("^" + resource);
- boolean isMatched = Pattern.matches(r, resource);
- if (isMatched) {
- sensitiveType = map.get(r);
- }
- else if (pattern.matcher(r).find()){
- childSensitiveTypes.add(map.get(r));
- }
- }
- }
- return sensitiveType;
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java
deleted file mode 100644
index 960a411..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/HbaseMetadataBrowseWebResponse.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.service.security.hbase;
-
-
-import org.apache.eagle.security.entity.HbaseResourceEntity;
-
-import java.util.List;
-
-public class HbaseMetadataBrowseWebResponse {
- private String exception;
- private List<HbaseResourceEntity> obj;
-
- public List<HbaseResourceEntity> getObj() {
- return obj;
- }
-
- public void setObj(List<HbaseResourceEntity> obj) {
- this.obj = obj;
- }
-
- public String getException() {
- return exception;
- }
-
- public void setException(String exception) {
- this.exception = exception;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java
new file mode 100644
index 0000000..b1a9c20
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HBaseMetadataDAOImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.security.hbase.dao;
+
+
+import org.apache.eagle.security.util.HadoopSecurityUtil;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.*;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class HBaseMetadataDAOImpl {
+ private final static Logger LOG = LoggerFactory.getLogger(HBaseMetadataDAOImpl.class);
+
+ private Configuration hBaseConfiguration;
+
+ public HBaseMetadataDAOImpl(Configuration config) {
+ this.hBaseConfiguration = HBaseConfiguration.create();
+ this.hBaseConfiguration.addResource(config);
+ //this.hBaseConfiguration.set("hbase.zookeeper.quorum", this.config.getZkQuorum());
+ //this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", this.config.getZkClientPort());
+ }
+
+ private HBaseAdmin getHBaseAdmin() throws IOException {
+ HadoopSecurityUtil.login(hBaseConfiguration);
+ return new HBaseAdmin(this.hBaseConfiguration);
+ }
+
+ public List<String> getNamespaces() throws IOException {
+ List ret = new ArrayList();
+ HBaseAdmin admin = this.getHBaseAdmin();
+ NamespaceDescriptor [] ns = admin.listNamespaceDescriptors();
+ for(NamespaceDescriptor n : ns) {
+ ret.add(n.getName());
+ }
+ closeHBaseConnection(admin);
+ return ret;
+ }
+
+ public List<String> getTables(String namespace) throws IOException {
+ List ret = new ArrayList();
+ HBaseAdmin admin = this.getHBaseAdmin();
+ TableName[] tables = admin.listTableNamesByNamespace(namespace);
+
+ for(TableName tableName : tables) {
+ ret.add(tableName.getQualifierAsString());
+ }
+ closeHBaseConnection(admin);
+ return ret;
+ }
+
+ public List<String> getColumnFamilies(String tableName) throws IOException {
+ List ret = new ArrayList();
+ HBaseAdmin admin = this.getHBaseAdmin();
+ HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName.getBytes());
+
+ HColumnDescriptor [] cfs = tableDescriptor.getColumnFamilies();
+ for(HColumnDescriptor cf : cfs) {
+ ret.add(cf.getNameAsString());
+ }
+ closeHBaseConnection(admin);
+ return ret;
+ }
+
+ private void closeHBaseConnection(HBaseAdmin admin) {
+ if(admin != null) {
+ try {
+ admin.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java
deleted file mode 100644
index ed52bd7..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/dao/HbaseMetadataDAOImpl.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.security.hbase.dao;
-
-
-import org.apache.eagle.security.util.HadoopSecurityUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.*;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-public class HbaseMetadataDAOImpl {
- private final static Logger LOG = LoggerFactory.getLogger(HbaseMetadataDAOImpl.class);
-
- private Configuration hBaseConfiguration;
-
- public HbaseMetadataDAOImpl(Configuration config) {
- this.hBaseConfiguration = HBaseConfiguration.create();
- this.hBaseConfiguration.addResource(config);
- //this.hBaseConfiguration.set("hbase.zookeeper.quorum", this.config.getZkQuorum());
- //this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", this.config.getZkClientPort());
- }
-
- private HBaseAdmin getHBaseAdmin() throws IOException {
- HadoopSecurityUtil.login(hBaseConfiguration);
- return new HBaseAdmin(this.hBaseConfiguration);
- }
-
- public List<String> getNamespaces() throws IOException {
- List ret = new ArrayList();
- HBaseAdmin admin = this.getHBaseAdmin();
- NamespaceDescriptor [] ns = admin.listNamespaceDescriptors();
- for(NamespaceDescriptor n : ns) {
- ret.add(n.getName());
- }
- closeHbaseConnection(admin);
- return ret;
- }
-
- public List<String> getTables(String namespace) throws IOException {
- List ret = new ArrayList();
- HBaseAdmin admin = this.getHBaseAdmin();
- TableName[] tables = admin.listTableNamesByNamespace(namespace);
-
- for(TableName tableName : tables) {
- ret.add(tableName.getQualifierAsString());
- }
- closeHbaseConnection(admin);
- return ret;
- }
-
- public List<String> getColumnFamilies(String tableName) throws IOException {
- List ret = new ArrayList();
- HBaseAdmin admin = this.getHBaseAdmin();
- HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName.getBytes());
-
- HColumnDescriptor [] cfs = tableDescriptor.getColumnFamilies();
- for(HColumnDescriptor cf : cfs) {
- ret.add(cf.getNameAsString());
- }
- closeHbaseConnection(admin);
- return ret;
- }
-
- private void closeHbaseConnection(HBaseAdmin admin) {
- if(admin != null) {
- try {
- admin.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java
new file mode 100644
index 0000000..2b34670
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseActionResolver.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.service.security.hbase.resolver;
+
+
+import com.typesafe.config.Config;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+import org.apache.eagle.security.resolver.AbstractCommandResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+
+
+public class HBaseActionResolver extends AbstractCommandResolver {
+ public HBaseActionResolver(ApplicationEntityService entityService, Config eagleServerConfig){
+
+ }
+ private final static Logger LOG = LoggerFactory.getLogger(HBaseActionResolver.class);
+ private final String [] cmdStrs = {"ADMIN", "READ", "WRITE", "CREATE", "EXECUTE"};
+
+ @Override
+ public void init() {
+ this.setCommands(Arrays.asList(cmdStrs));
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java
new file mode 100644
index 0000000..7405f25
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseMetadataResolver.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.security.hbase.resolver;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+import org.apache.eagle.service.alert.resolver.AttributeResolvable;
+import org.apache.eagle.service.alert.resolver.AttributeResolveException;
+import org.apache.eagle.service.alert.resolver.BadAttributeResolveRequestException;
+import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest;
+import org.apache.eagle.service.security.hbase.HBaseMetadataBrowseWebResource;
+import org.apache.eagle.service.security.hbase.dao.HBaseMetadataDAOImpl;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+
+public class HBaseMetadataResolver implements AttributeResolvable<GenericAttributeResolveRequest,String> {
+ private ApplicationEntityService entityService;
+ public HBaseMetadataResolver(ApplicationEntityService entityService, Config eagleServerConfig){
+ this.entityService = entityService;
+ }
+
+ private Map<String, Object> getAppConfig(String site, String appType){
+ ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType);
+ return entity.getConfiguration();
+ }
+
+ private Configuration convert(Map<String, Object> originalConfig) throws Exception {
+ Configuration config = new Configuration();
+ for (Map.Entry<String, Object> entry : originalConfig.entrySet()) {
+ config.set(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return config;
+ }
+
+ @Override
+ public List<String> resolve(GenericAttributeResolveRequest request) throws AttributeResolveException {
+ String query = request.getQuery().trim();
+ String site = request.getSite().trim();
+ String[] subResources = query.split(":");
+
+ try {
+ Map<String, Object> config = getAppConfig(site, HBaseMetadataBrowseWebResource.HBASE_APPLICATION);
+ Configuration conf = convert(config);
+ HBaseMetadataDAOImpl dao = new HBaseMetadataDAOImpl(conf);
+
+ switch (subResources.length) {
+ case 1:
+ if(query.endsWith(":"))
+ return filterAndCombineAttribue(query, dao.getTables(subResources[0]), null);
+ else
+ return filterAndCombineAttribue("", dao.getNamespaces(), subResources[0]);
+ case 2:
+ if(query.endsWith(":"))
+ return filterAndCombineAttribue(query, dao.getColumnFamilies(subResources[1]), null);
+ else
+ return filterAndCombineAttribue(subResources[0], dao.getTables(subResources[0]), subResources[1]);
+ case 3:
+ return filterAndCombineAttribue(String.format("%s:%s", subResources[0], subResources[1]), dao.getColumnFamilies(subResources[1]), subResources[2]);
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ @Override
+ public void validateRequest(GenericAttributeResolveRequest request) throws BadAttributeResolveRequestException {
+ String query = request.getQuery();
+ String site = request.getSite();
+ if (query == null || query.startsWith(":") || !Pattern.matches("[a-zA-Z:]+", query) || site == null || site.length() == 0) {
+ throw new BadAttributeResolveRequestException("hBase resource must be {\"site\":\"${site}\", \"query\"=\"{namespace}:{table}:{columnFamily}\"");
+ }
+ }
+
+ public List<String> filterAndCombineAttribue(String prefix, List<String> attrs, String target) {
+ List<String> result = new ArrayList<>();
+ String format = (prefix == "" ? "%s%s" : "%s:%s");
+
+ if(target == null) {
+ for (String attr : attrs){
+ result.add(String.format("%s%s", prefix, attr));
+ }
+ } else {
+ Pattern pattern = Pattern.compile("^" + target, Pattern.CASE_INSENSITIVE);
+ for (String attr : attrs) {
+ if (pattern.matcher(attr).find()) {
+ result.add(String.format(format, prefix, attr));
+ }
+ }
+ if (result.size() == 0) {
+ for (String attr : attrs) {
+ result.add(String.format(format, prefix, attr));
+ }
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public Class<GenericAttributeResolveRequest> getRequestClass() {
+ return GenericAttributeResolveRequest.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java
new file mode 100644
index 0000000..483ba82
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseRequestResolver.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.service.security.hbase.resolver;
+
+
+import com.typesafe.config.Config;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+import org.apache.eagle.security.resolver.AbstractCommandResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+
+public class HBaseRequestResolver extends AbstractCommandResolver {
+ private final static Logger LOG = LoggerFactory.getLogger(HBaseRequestResolver.class);
+
+ public HBaseRequestResolver(ApplicationEntityService entityService, Config eagleServerConfig){
+
+ }
+
+ private final String [] master = {"createTable", "modifyTable", "deleteTable", "truncateTable", "addColumn", "modifyColumn",
+ "deleteColumn", "enableTable", "disableTable", "disableAclTable", "move", "assign", "unassign",
+ "regionOffline", "balance", "balanceSwitch", "shutdown", "stopMaster", "snapshot",
+ "listSnapshot", "cloneSnapshot", "restoreSnapshot", "deleteSnapshot", "createNamespace",
+ "deleteNamespace", "modifyNamespace", "getNamespaceDescriptor", "listNamespaceDescriptors*",
+ "flushTable", "getTableDescriptors*", "getTableNames*", "setUserQuota", "setTableQuota",
+ "setNamespaceQuota"};
+ private final String [] region = {"openRegion", "closeRegion", "flush", "split", "compact", "getClosestRowBefore", "getOp", "exists",
+ "put", "delete", "batchMutate", "checkAndPut", "checkAndPutAfterRowLock", "checkAndDelete",
+ "checkAndDeleteAfterRowLock", "incrementColumnValue", "append", "appendAfterRowLock","increment",
+ "incrementAfterRowLock", "scan", "bulkLoadHFile", "prepareBulkLoad",
+ "cleanupBulkLoad"};
+ private final String [] endpoint= {"invoke"};
+ private final String [] accessController = {"grant", "revoke", "getUserPermissions"};
+ private final String [] regionServer = {"stopRegionServer", "mergeRegions", "rollWALWriterRequest", "replicateLogEntries"};
+
+ @Override
+ public void init(){
+ List<String> cmds = new ArrayList<>();
+ cmds.addAll(Arrays.asList(master));
+ cmds.addAll(Arrays.asList(region));
+ cmds.addAll(Arrays.asList(regionServer));
+ cmds.addAll(Arrays.asList(endpoint));
+ cmds.addAll(Arrays.asList(accessController));
+ this.setCommands(cmds);
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java
new file mode 100644
index 0000000..3692150
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HBaseSensitivityTypeResolver.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.service.security.hbase.resolver;
+
+import com.typesafe.config.Config;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+import org.apache.eagle.security.resolver.AbstractSensitivityTypeResolver;
+import org.apache.eagle.security.service.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public class HBaseSensitivityTypeResolver extends AbstractSensitivityTypeResolver {
+ private final static Logger LOG = LoggerFactory.getLogger(HBaseSensitivityTypeResolver.class);
+ private ISecurityMetadataDAO dao;
+
+ public HBaseSensitivityTypeResolver(ApplicationEntityService entityService, Config eagleServerConfig){
+ // todo I know this is ugly, but push abstraction to later
+ dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig);
+ }
+
+ @Override
+ public void init() {
+ Map<String, Map<String, String>> maps = getAllSensitivities();
+ this.setSensitivityMaps(maps);
+ }
+
+ private Map<String, Map<String, String>> getAllSensitivities(){
+ Map<String, Map<String, String>> all = new HashMap<>();
+ Collection<HBaseSensitivityEntity> entities = dao.listHBaseSensitivities();
+ for(HBaseSensitivityEntity entity : entities){
+ if(!all.containsKey(entity.getSite())){
+ all.put(entity.getSite(), new HashMap<>());
+ }
+ all.get(entity.getSite()).put(entity.getHBaseResource(), entity.getSensitivityType());
+ }
+ return all;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java
deleted file mode 100644
index 3baa1da..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseActionResolver.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.service.security.hbase.resolver;
-
-
-import com.typesafe.config.Config;
-import org.apache.eagle.metadata.service.ApplicationEntityService;
-import org.apache.eagle.security.resolver.AbstractCommandResolver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-
-
-public class HbaseActionResolver extends AbstractCommandResolver {
- public HbaseActionResolver(ApplicationEntityService entityService, Config eagleServerConfig){
-
- }
- private final static Logger LOG = LoggerFactory.getLogger(HbaseActionResolver.class);
- private final String [] cmdStrs = {"ADMIN", "READ", "WRITE", "CREATE", "EXECUTE"};
-
- @Override
- public void init() {
- this.setCommands(Arrays.asList(cmdStrs));
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
deleted file mode 100644
index 6dd82e4..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseMetadataResolver.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.security.hbase.resolver;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.metadata.model.ApplicationEntity;
-import org.apache.eagle.metadata.service.ApplicationEntityService;
-import org.apache.eagle.service.alert.resolver.AttributeResolvable;
-import org.apache.eagle.service.alert.resolver.AttributeResolveException;
-import org.apache.eagle.service.alert.resolver.BadAttributeResolveRequestException;
-import org.apache.eagle.service.alert.resolver.GenericAttributeResolveRequest;
-import org.apache.eagle.service.security.hbase.HbaseMetadataBrowseWebResource;
-import org.apache.eagle.service.security.hbase.dao.HbaseMetadataDAOImpl;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Pattern;
-
-
-public class HbaseMetadataResolver implements AttributeResolvable<GenericAttributeResolveRequest,String> {
- private ApplicationEntityService entityService;
- public HbaseMetadataResolver(ApplicationEntityService entityService, Config eagleServerConfig){
- this.entityService = entityService;
- }
-
- private Map<String, Object> getAppConfig(String site, String appType){
- ApplicationEntity entity = entityService.getBySiteIdAndAppType(site, appType);
- return entity.getConfiguration();
- }
-
- private Configuration convert(Map<String, Object> originalConfig) throws Exception {
- Configuration config = new Configuration();
- for (Map.Entry<String, Object> entry : originalConfig.entrySet()) {
- config.set(entry.getKey().toString(), entry.getValue().toString());
- }
- return config;
- }
-
- @Override
- public List<String> resolve(GenericAttributeResolveRequest request) throws AttributeResolveException {
- String query = request.getQuery().trim();
- String site = request.getSite().trim();
- String[] subResources = query.split(":");
-
- try {
- Map<String, Object> config = getAppConfig(site, HbaseMetadataBrowseWebResource.HBASE_APPLICATION);
- Configuration conf = convert(config);
- HbaseMetadataDAOImpl dao = new HbaseMetadataDAOImpl(conf);
-
- switch (subResources.length) {
- case 1:
- if(query.endsWith(":"))
- return filterAndCombineAttribue(query, dao.getTables(subResources[0]), null);
- else
- return filterAndCombineAttribue("", dao.getNamespaces(), subResources[0]);
- case 2:
- if(query.endsWith(":"))
- return filterAndCombineAttribue(query, dao.getColumnFamilies(subResources[1]), null);
- else
- return filterAndCombineAttribue(subResources[0], dao.getTables(subResources[0]), subResources[1]);
- case 3:
- return filterAndCombineAttribue(String.format("%s:%s", subResources[0], subResources[1]), dao.getColumnFamilies(subResources[1]), subResources[2]);
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- return null;
- }
-
- @Override
- public void validateRequest(GenericAttributeResolveRequest request) throws BadAttributeResolveRequestException {
- String query = request.getQuery();
- String site = request.getSite();
- if (query == null || query.startsWith(":") || !Pattern.matches("[a-zA-Z:]+", query) || site == null || site.length() == 0) {
- throw new BadAttributeResolveRequestException("hBase resource must be {\"site\":\"${site}\", \"query\"=\"{namespace}:{table}:{columnFamily}\"");
- }
- }
-
- public List<String> filterAndCombineAttribue(String prefix, List<String> attrs, String target) {
- List<String> result = new ArrayList<>();
- String format = (prefix == "" ? "%s%s" : "%s:%s");
-
- if(target == null) {
- for (String attr : attrs){
- result.add(String.format("%s%s", prefix, attr));
- }
- } else {
- Pattern pattern = Pattern.compile("^" + target, Pattern.CASE_INSENSITIVE);
- for (String attr : attrs) {
- if (pattern.matcher(attr).find()) {
- result.add(String.format(format, prefix, attr));
- }
- }
- if (result.size() == 0) {
- for (String attr : attrs) {
- result.add(String.format(format, prefix, attr));
- }
- }
- }
- return result;
- }
-
- @Override
- public Class<GenericAttributeResolveRequest> getRequestClass() {
- return GenericAttributeResolveRequest.class;
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java
deleted file mode 100644
index 8467f40..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseRequestResolver.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.service.security.hbase.resolver;
-
-
-import com.typesafe.config.Config;
-import org.apache.eagle.metadata.service.ApplicationEntityService;
-import org.apache.eagle.security.resolver.AbstractCommandResolver;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-
-public class HbaseRequestResolver extends AbstractCommandResolver {
- private final static Logger LOG = LoggerFactory.getLogger(HbaseRequestResolver.class);
-
- public HbaseRequestResolver(ApplicationEntityService entityService, Config eagleServerConfig){
-
- }
-
- private final String [] master = {"createTable", "modifyTable", "deleteTable", "truncateTable", "addColumn", "modifyColumn",
- "deleteColumn", "enableTable", "disableTable", "disableAclTable", "move", "assign", "unassign",
- "regionOffline", "balance", "balanceSwitch", "shutdown", "stopMaster", "snapshot",
- "listSnapshot", "cloneSnapshot", "restoreSnapshot", "deleteSnapshot", "createNamespace",
- "deleteNamespace", "modifyNamespace", "getNamespaceDescriptor", "listNamespaceDescriptors*",
- "flushTable", "getTableDescriptors*", "getTableNames*", "setUserQuota", "setTableQuota",
- "setNamespaceQuota"};
- private final String [] region = {"openRegion", "closeRegion", "flush", "split", "compact", "getClosestRowBefore", "getOp", "exists",
- "put", "delete", "batchMutate", "checkAndPut", "checkAndPutAfterRowLock", "checkAndDelete",
- "checkAndDeleteAfterRowLock", "incrementColumnValue", "append", "appendAfterRowLock","increment",
- "incrementAfterRowLock", "scan", "bulkLoadHFile", "prepareBulkLoad",
- "cleanupBulkLoad"};
- private final String [] endpoint= {"invoke"};
- private final String [] accessController = {"grant", "revoke", "getUserPermissions"};
- private final String [] regionServer = {"stopRegionServer", "mergeRegions", "rollWALWriterRequest", "replicateLogEntries"};
-
- @Override
- public void init(){
- List<String> cmds = new ArrayList<>();
- cmds.addAll(Arrays.asList(master));
- cmds.addAll(Arrays.asList(region));
- cmds.addAll(Arrays.asList(regionServer));
- cmds.addAll(Arrays.asList(endpoint));
- cmds.addAll(Arrays.asList(accessController));
- this.setCommands(cmds);
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java b/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java
deleted file mode 100644
index b5ad203..0000000
--- a/eagle-security/eagle-security-hbase-web/src/main/java/org/apache/eagle/service/security/hbase/resolver/HbaseSensitivityTypeResolver.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.service.security.hbase.resolver;
-
-import com.typesafe.config.Config;
-import org.apache.eagle.metadata.service.ApplicationEntityService;
-import org.apache.eagle.security.resolver.AbstractSensitivityTypeResolver;
-import org.apache.eagle.security.service.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-public class HbaseSensitivityTypeResolver extends AbstractSensitivityTypeResolver {
- private final static Logger LOG = LoggerFactory.getLogger(HbaseSensitivityTypeResolver.class);
- private ISecurityMetadataDAO dao;
-
- public HbaseSensitivityTypeResolver(ApplicationEntityService entityService, Config eagleServerConfig){
- // todo I know this is ugly, but push abstraction to later
- dao = MetadataDaoFactory.getMetadataDAO(eagleServerConfig);
- }
-
- @Override
- public void init() {
- Map<String, Map<String, String>> maps = getAllSensitivities();
- this.setSensitivityMaps(maps);
- }
-
- private Map<String, Map<String, String>> getAllSensitivities(){
- Map<String, Map<String, String>> all = new HashMap<>();
- Collection<HBaseSensitivityEntity> entities = dao.listHBaseSensitivities();
- for(HBaseSensitivityEntity entity : entities){
- if(!all.containsKey(entity.getSite())){
- all.put(entity.getSite(), new HashMap<>());
- }
- all.get(entity.getSite()).put(entity.getHbaseResource(), entity.getSensitivityType());
- }
- return all;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java
new file mode 100644
index 0000000..9f90770
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHBaseMetadataResolver.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.service.security.hbase;
+
+
+import org.junit.Test;
+
+public class TestHBaseMetadataResolver {
+
+ @Test
+ public void testSplit() {
+ String query = "hbase:";
+ String[] subResources = query.split(":");
+
+ query = "hbase:meta";
+ subResources = query.split(":");
+
+ query = "hbase:meta:";
+ subResources = query.split(":");
+
+ query = "hbase:meta:cf";
+ subResources = query.split(":");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java b/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java
deleted file mode 100644
index 8cd0c32..0000000
--- a/eagle-security/eagle-security-hbase-web/src/test/java/org/apache/eagle/service/security/hbase/TestHbaseMetadataResolver.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.service.security.hbase;
-
-
-import org.junit.Test;
-
-public class TestHbaseMetadataResolver {
-
- @Test
- public void testSplit() {
- String query = "hbase:";
- String[] subResources = query.split(":");
-
- query = "hbase:meta";
- subResources = query.split(":");
-
- query = "hbase:meta:";
- subResources = query.split(":");
-
- query = "hbase:meta:cf";
- subResources = query.split(":");
-
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
index 17dec20..22afeb3 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/TopologyExtractorFactory.java
@@ -21,12 +21,11 @@ package org.apache.eagle.topology.extractor;
import backtype.storm.spout.SpoutOutputCollector;
import org.apache.eagle.topology.TopologyCheckAppConfig;
import org.apache.eagle.topology.TopologyConstants;
-import org.apache.eagle.topology.extractor.hbase.HbaseTopologyCrawler;
+import org.apache.eagle.topology.extractor.hbase.HBaseTopologyCrawler;
import org.apache.eagle.topology.extractor.hdfs.HdfsTopologyCrawler;
import org.apache.eagle.topology.extractor.mr.MRTopologyCrawler;
import org.apache.eagle.topology.resolver.TopologyRackResolver;
-import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver;
import org.slf4j.Logger;
import java.lang.reflect.Constructor;
@@ -69,7 +68,7 @@ public class TopologyExtractorFactory {
}
static {
- registerTopologyExtractor(TopologyConstants.TopologyType.HBASE.name(), HbaseTopologyCrawler.class);
+ registerTopologyExtractor(TopologyConstants.TopologyType.HBASE.name(), HBaseTopologyCrawler.class);
registerTopologyExtractor(TopologyConstants.TopologyType.HDFS.name(), HdfsTopologyCrawler.class);
registerTopologyExtractor(TopologyConstants.TopologyType.MR.name(), MRTopologyCrawler.class);
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java
new file mode 100644
index 0000000..1ee1fa1
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyCrawler.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor.hbase;
+
+import backtype.storm.spout.SpoutOutputCollector;
+import backtype.storm.tuple.Values;
+import org.apache.eagle.topology.TopologyCheckAppConfig;
+import org.apache.eagle.topology.TopologyCheckMessageId;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
+import org.apache.eagle.topology.extractor.TopologyCrawler;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HBaseTopologyCrawler implements TopologyCrawler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HBaseTopologyCrawler.class);
+
+ private HBaseTopologyEntityParser parser;
+ private SpoutOutputCollector collector;
+ private String site;
+
+ public HBaseTopologyCrawler(TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) {
+ this.site = config.dataExtractorConfig.site;
+ this.parser = new HBaseTopologyEntityParser(this.site, config.hBaseConfig, rackResolver);
+ this.collector = collector;
+ }
+
+ @Override
+ public void extract() {
+ long updateTimestamp = System.currentTimeMillis();
+ TopologyEntityParserResult result = parser.parse(updateTimestamp);;
+
+ if (result == null || result.getMetrics().isEmpty()) {
+ LOG.warn("No data fetched");
+ result = new TopologyEntityParserResult();
+ }
+ TopologyCheckMessageId messageId = new TopologyCheckMessageId(TopologyConstants.TopologyType.HBASE, updateTimestamp);
+ this.collector.emit(new Values(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME, result), messageId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java
new file mode 100644
index 0000000..a73fbf8
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HBaseTopologyEntityParser.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.topology.extractor.hbase;
+
+import org.apache.eagle.app.utils.HadoopSecurityUtil;
+import org.apache.eagle.topology.TopologyCheckAppConfig;
+import org.apache.eagle.topology.TopologyConstants;
+import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
+import org.apache.eagle.topology.entity.HBaseServiceTopologyAPIEntity;
+import org.apache.eagle.topology.extractor.TopologyEntityParser;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.apache.eagle.topology.utils.EntityBuilderHelper;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.ClusterStatus;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.ServerLoad;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.eagle.topology.TopologyConstants.*;
+
+public class HBaseTopologyEntityParser implements TopologyEntityParser {
+
+ private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HBaseTopologyEntityParser.class);
+ private Configuration hBaseConfiguration;
+ private String site;
+ private Boolean kerberosEnable = false;
+ private TopologyRackResolver rackResolver;
+
+ public HBaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) {
+ this.site = site;
+ this.rackResolver = resolver;
+ this.hBaseConfiguration = HBaseConfiguration.create();
+ this.hBaseConfiguration.set("hbase.zookeeper.quorum", hBaseConfig.zkQuorum);
+ this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", hBaseConfig.zkClientPort);
+ this.hBaseConfiguration.set("zookeeper.znode.parent", hBaseConfig.zkRoot);
+ this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes);
+ // kerberos authentication
+ if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null
+ && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) {
+ this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal);
+ this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab);
+ this.kerberosEnable = true;
+ this.hBaseConfiguration.set("hbase.security.authentication", "kerberos");
+ this.hBaseConfiguration.set("hbase.master.kerberos.principal", hBaseConfig.hbaseMasterPrincipal);
+ }
+ }
+
+ private HBaseAdmin getHBaseAdmin() throws IOException {
+ if (this.kerberosEnable) {
+ HadoopSecurityUtil.login(hBaseConfiguration);
+ }
+ return new HBaseAdmin(this.hBaseConfiguration);
+ }
+
+
+ @Override
+ public TopologyEntityParserResult parse(long timestamp) {
+ final TopologyEntityParserResult result = new TopologyEntityParserResult();
+ int activeRatio = 0;
+ try {
+ doParse(timestamp, result);
+ activeRatio++;
+ } catch (Exception ex) {
+ LOG.error(ex.getMessage(), ex);
+ }
+ result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE, activeRatio, site, timestamp));
+ return result;
+ }
+
+ private void doParse(long timestamp, TopologyEntityParserResult result) throws IOException {
+ long deadServers = 0;
+ long liveServers = 0;
+ HBaseAdmin admin = null;
+ try {
+ admin = getHBaseAdmin();
+ ClusterStatus status = admin.getClusterStatus();
+ deadServers = status.getDeadServers();
+ liveServers = status.getServersSize();
+
+ for (ServerName liveServer : status.getServers()) {
+ ServerLoad load = status.getLoad(liveServer);
+ result.getSlaveNodes().add(parseServer(liveServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_LIVE_STATUS, timestamp));
+ }
+ for (ServerName deadServer : status.getDeadServerNames()) {
+ ServerLoad load = status.getLoad(deadServer);
+ result.getSlaveNodes().add(parseServer(deadServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_DEAD_STATUS, timestamp));
+ }
+ ServerName master = status.getMaster();
+ if (master != null) {
+ ServerLoad load = status.getLoad(master);
+ result.getMasterNodes().add(parseServer(master, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_ACTIVE_STATUS, timestamp));
+ }
+ for (ServerName backupMaster : status.getBackupMasters()) {
+ ServerLoad load = status.getLoad(backupMaster);
+ result.getMasterNodes().add(parseServer(backupMaster, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_STANDBY_STATUS, timestamp));
+ }
+ double liveRatio = liveServers * 1d / (liveServers + deadServers);
+ result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE, liveRatio, site, timestamp));
+ LOG.info("live servers: {}, dead servers: {}", liveServers, deadServers);
+ } finally {
+ if (admin != null) {
+ try {
+ admin.close();
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }
+ }
+
+ private HBaseServiceTopologyAPIEntity parseServer(ServerName serverName, ServerLoad serverLoad, String role, String status, long timestamp) {
+ if (serverName == null) {
+ return null;
+ }
+ HBaseServiceTopologyAPIEntity entity = createEntity(role, serverName.getHostname(), timestamp);
+ parseServerLoad(entity, serverLoad);
+ entity.setStatus(status);
+ return entity;
+ }
+
+ private void parseServerLoad(HBaseServiceTopologyAPIEntity entity, ServerLoad load) {
+ if (load == null) {
+ return;
+ }
+ entity.setMaxHeapMB(load.getMaxHeapMB());
+ entity.setUsedHeapMB(load.getUsedHeapMB());
+ entity.setNumRegions(load.getNumberOfRegions());
+ entity.setNumRequests(load.getNumberOfRequests());
+ }
+
+ private HBaseServiceTopologyAPIEntity createEntity(String roleType, String hostname, long timestamp) {
+ HBaseServiceTopologyAPIEntity entity = new HBaseServiceTopologyAPIEntity();
+ Map<String, String> tags = new HashMap<String, String>();
+ entity.setTags(tags);
+ tags.put(SITE_TAG, site);
+ tags.put(ROLE_TAG, roleType);
+ tags.put(HOSTNAME_TAG, hostname);
+ String rack = rackResolver.resolve(hostname);
+ tags.put(RACK_TAG, rack);
+ entity.setLastUpdateTime(timestamp);
+ entity.setTimestamp(timestamp);
+ return entity;
+ }
+
+ @Override
+ public TopologyConstants.TopologyType getTopologyType() {
+ return TopologyType.HBASE;
+ }
+
+ @Override
+ public TopologyConstants.HadoopVersion getHadoopVersion() {
+ return HadoopVersion.V2;
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java
deleted file mode 100644
index 04a4aa1..0000000
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyCrawler.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.topology.extractor.hbase;
-
-import backtype.storm.spout.SpoutOutputCollector;
-import backtype.storm.tuple.Values;
-import org.apache.eagle.topology.TopologyCheckAppConfig;
-import org.apache.eagle.topology.TopologyCheckMessageId;
-import org.apache.eagle.topology.TopologyConstants;
-import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
-import org.apache.eagle.topology.extractor.TopologyCrawler;
-import org.apache.eagle.topology.resolver.TopologyRackResolver;
-import org.apache.eagle.topology.utils.EntityBuilderHelper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-public class HbaseTopologyCrawler implements TopologyCrawler {
-
- private static final Logger LOG = LoggerFactory.getLogger(HbaseTopologyCrawler.class);
-
- private HbaseTopologyEntityParser parser;
- private SpoutOutputCollector collector;
- private String site;
-
- public HbaseTopologyCrawler(TopologyCheckAppConfig config, TopologyRackResolver rackResolver, SpoutOutputCollector collector) {
- this.site = config.dataExtractorConfig.site;
- this.parser = new HbaseTopologyEntityParser(this.site, config.hBaseConfig, rackResolver);
- this.collector = collector;
- }
-
- @Override
- public void extract() {
- long updateTimestamp = System.currentTimeMillis();
- TopologyEntityParserResult result = parser.parse(updateTimestamp);;
-
- if (result == null || result.getMetrics().isEmpty()) {
- LOG.warn("No data fetched");
- result = new TopologyEntityParserResult();
- }
- TopologyCheckMessageId messageId = new TopologyCheckMessageId(TopologyConstants.TopologyType.HBASE, updateTimestamp);
- this.collector.emit(new Values(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME, result), messageId);
- }
-}
[16/17] eagle git commit: [EAGLE-993] add duplicate removal settings
in policy definition
Posted by ja...@apache.org.
[EAGLE-993] add duplicate removal settings in policy definition
https://issues.apache.org/jira/browse/EAGLE-993
Author: Zhao, Qingwen <qi...@apache.org>
Closes #926 from qingwen220/EAGLE-993.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/8da06636
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/8da06636
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/8da06636
Branch: refs/heads/branch-0.5
Commit: 8da06636b8e3834f7bebf6b1af923ad24b7d6d8b
Parents: 47f00f1
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Fri May 5 13:02:20 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Fri May 5 13:02:20 2017 +0800
----------------------------------------------------------------------
eagle-assembly/src/main/doc/metadata-ddl.sql | 7 -
.../src/assembly/alert-assembly.xml | 3 +-
.../engine/coordinator/AlertDeduplication.java | 71 +++++++
.../engine/coordinator/PolicyDefinition.java | 13 +-
.../engine/coordinator/PublishmentType.java | 21 +-
.../publisher/email/AlertEmailGenerator.java | 4 +-
.../publisher/impl/AbstractPublishPlugin.java | 17 +-
.../publisher/impl/AlertEmailPublisher.java | 2 -
.../publisher/impl/AlertPublisherImpl.java | 43 ++---
.../publisher/impl/DefaultDeduplicator.java | 10 +-
.../template/VelocityAlertTemplateEngine.java | 8 +-
.../alert/engine/runner/AlertPublisherBolt.java | 18 +-
.../main/resources/ALERT_INLINED_TEMPLATE.vm | 2 +-
.../VelocityAlertTemplateEngineTest.java | 2 +-
.../metadata/impl/JdbcMetadataDaoImpl.java | 2 +-
.../metadata/impl/JdbcMetadataHandler.java | 30 +++
.../jpm/mr/history/JHFEventReaderBaseTest.java | 74 +++++++
.../src/test/resources/application.conf | 74 +++++++
.../resources/job_1479206441898_508949_conf.xml | 191 +++++++++++++++++--
19 files changed, 515 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-assembly/src/main/doc/metadata-ddl.sql
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/doc/metadata-ddl.sql b/eagle-assembly/src/main/doc/metadata-ddl.sql
index 0334623..3312576 100644
--- a/eagle-assembly/src/main/doc/metadata-ddl.sql
+++ b/eagle-assembly/src/main/doc/metadata-ddl.sql
@@ -164,10 +164,3 @@ CREATE TABLE IF NOT EXISTS analysis_email (
modifiedtime bigint(20) DEFAULT NULL,
UNIQUE (siteId, userId)
);
-
-INSERT INTO publishment_type(id, content) VALUES
-('Kafka', '{"name":"Kafka","type":"org.apache.eagle.alert.engine.publisher.impl.AlertKafkaPublisher","description":null,"fields":[{"name":"kafka_broker","value":"sandbox.hortonworks.com:6667"},{"name":"topic"}]}'),
-('Email', '{"name":"Email","type":"org.apache.eagle.alert.engine.publisher.impl.AlertEmailPublisher","description":null,"fields":[{"name":"subject"},{"name":"sender"}, {"name":"recipients"}]}'),
-('Slack', '{"name":"Slack","type":"org.apache.eagle.alert.engine.publisher.impl.AlertSlackPublisher","description":null,"fields":[{"name":"token"},{"name":"channels"}, {"name":"severitys"}, {"name":"urltemplate"}]}'),
-('HBaseStorage', '{"name":"HBaseStorage","type":"org.apache.eagle.alert.app.AlertEagleStorePlugin","description":null,"fields":[]}'),
-('JDBCStorage', '{"name":"JDBCStorage","type":"org.apache.eagle.alert.engine.publisher.impl.AlertEagleStorePlugin","description":null,"fields":[]}');
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-assembly/src/assembly/alert-assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-assembly/src/assembly/alert-assembly.xml b/eagle-core/eagle-alert-parent/eagle-alert/alert-assembly/src/assembly/alert-assembly.xml
index 9f25ec0..b361f99 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-assembly/src/assembly/alert-assembly.xml
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-assembly/src/assembly/alert-assembly.xml
@@ -9,7 +9,8 @@
distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
License for the specific language governing permissions and ~ limitations
- under the License. -->
+ under the License.
+-->
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDeduplication.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDeduplication.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDeduplication.java
new file mode 100644
index 0000000..78fef7a
--- /dev/null
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/AlertDeduplication.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.alert.engine.coordinator;
+
+import org.apache.commons.collections.ListUtils;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.util.List;
+import java.util.Objects;
+
+public class AlertDeduplication {
+ private String dedupIntervalMin;
+ private List<String> dedupFields;
+
+ public String getDedupIntervalMin() {
+ return dedupIntervalMin;
+ }
+
+ public void setDedupIntervalMin(String dedupIntervalMin) {
+ this.dedupIntervalMin = dedupIntervalMin;
+ }
+
+ public List<String> getDedupFields() {
+ return dedupFields;
+ }
+
+ public void setDedupFields(List<String> dedupFields) {
+ this.dedupFields = dedupFields;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder()
+ .append(dedupFields)
+ .append(dedupIntervalMin)
+ .build();
+ }
+
+ @Override
+ public boolean equals(Object that) {
+ if (that == this) {
+ return true;
+ }
+ if (!(that instanceof AlertDeduplication)) {
+ return false;
+ }
+ AlertDeduplication another = (AlertDeduplication) that;
+ if (ListUtils.isEqualList(another.dedupFields, this.dedupFields)
+ && Objects.equals(another.dedupIntervalMin, this.dedupIntervalMin)) {
+ return true;
+ }
+ return false;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
index c377e41..5004513 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PolicyDefinition.java
@@ -43,6 +43,7 @@ public class PolicyDefinition implements Serializable {
private Definition stateDefinition;
private PolicyStatus policyStatus = PolicyStatus.ENABLED;
private AlertDefinition alertDefinition;
+ private AlertDeduplication deduplication;
// one stream only have one partition in one policy, since we don't support stream alias
private List<StreamPartition> partitionSpec = new ArrayList<StreamPartition>();
@@ -147,6 +148,7 @@ public class PolicyDefinition implements Serializable {
.append(policyStatus)
.append(parallelismHint)
.append(alertDefinition)
+ .append(deduplication)
.build();
}
@@ -172,7 +174,8 @@ public class PolicyDefinition implements Serializable {
&& CollectionUtils.isEqualCollection(another.partitionSpec, this.partitionSpec)
&& another.policyStatus.equals(this.policyStatus)
&& another.parallelismHint == this.parallelismHint
- && Objects.equals(another.alertDefinition, alertDefinition)) {
+ && Objects.equals(another.alertDefinition, alertDefinition)
+ && Objects.equals(another.deduplication, deduplication)) {
return true;
}
return false;
@@ -202,6 +205,14 @@ public class PolicyDefinition implements Serializable {
this.siteId = siteId;
}
+ public AlertDeduplication getDeduplication() {
+ return deduplication;
+ }
+
+ public void setDeduplication(AlertDeduplication deduplication) {
+ this.deduplication = deduplication;
+ }
+
@JsonIgnoreProperties(ignoreUnknown = true)
public static class Definition implements Serializable {
private static final long serialVersionUID = -622366527887848346L;
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
index f7025f2..3119ee6 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/PublishmentType.java
@@ -26,17 +26,6 @@ import java.util.*;
@JsonIgnoreProperties(ignoreUnknown = true)
public class PublishmentType {
private String name;
-
- @Override
- public String toString() {
- return "PublishmentType{"
- + "name='" + name + '\''
- + ", type='" + type + '\''
- + ", description='" + description + '\''
- + ", fields=" + fields
- + '}';
- }
-
private String type;
private String description;
private List<Map<String, String>> fields = new LinkedList<>();
@@ -73,7 +62,15 @@ public class PublishmentType {
this.fields = fields;
}
-
+ @Override
+ public String toString() {
+ return "PublishmentType{"
+ + "name='" + name + '\''
+ + ", type='" + type + '\''
+ + ", description='" + description + '\''
+ + ", fields=" + fields
+ + '}';
+ }
@Override
public boolean equals(Object obj) {
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
index 1bcac17..a57941e 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/email/AlertEmailGenerator.java
@@ -139,7 +139,9 @@ public class AlertEmailGenerator {
alertContext.put(PublishConstants.ALERT_EMAIL_ALERT_DATA_DESC, generateAlertDataDesc(event));
alertContext.put(PublishConstants.ALERT_EMAIL_ALERT_CATEGORY, event.getCategory());
alertContext.put(PublishConstants.ALERT_EMAIL_ALERT_SEVERITY, event.getSeverity().toString());
- alertContext.put(PublishConstants.ALERT_EMAIL_TIME, DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()));
+ alertContext.put(PublishConstants.ALERT_EMAIL_TIME, String.format("%s %s",
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()),
+ DateTimeUtil.CURRENT_TIME_ZONE.getID()));
alertContext.put(PublishConstants.ALERT_EMAIL_STREAM_ID, event.getStreamId());
alertContext.put(PublishConstants.ALERT_EMAIL_CREATOR, event.getCreatedBy());
alertContext.put(PublishConstants.ALERT_EMAIL_VERSION, Version.version);
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AbstractPublishPlugin.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AbstractPublishPlugin.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AbstractPublishPlugin.java
index b155bb8..c5c9e04 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AbstractPublishPlugin.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AbstractPublishPlugin.java
@@ -29,6 +29,7 @@ import org.apache.eagle.alert.engine.publisher.dedup.DedupCache;
import org.apache.eagle.alert.engine.publisher.dedup.ExtendedDeduplicator;
import org.slf4j.Logger;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,8 +72,14 @@ public abstract class AbstractPublishPlugin implements AlertPublishPlugin {
getLogger().error(String.format("initialize extended deduplicator %s failed", spec.getClassName()), t);
}
} else {
- this.deduplicator = new DefaultDeduplicator(publishment.getDedupIntervalMin(),
- publishment.getDedupFields(), publishment.getDedupStateField(), publishment.getDedupStateCloseValue(), dedupCache);
+ if (publishment.getDedupIntervalMin() != null && !publishment.getDedupIntervalMin().isEmpty()) {
+ this.deduplicator = new DefaultDeduplicator(
+ publishment.getDedupIntervalMin(),
+ publishment.getDedupFields(),
+ publishment.getDedupStateField(),
+ publishment.getDedupStateCloseValue(),
+ dedupCache);
+ }
this.pubName = publishment.getName();
}
String serializerClz = publishment.getSerializer();
@@ -98,7 +105,11 @@ public abstract class AbstractPublishPlugin implements AlertPublishPlugin {
@Override
public List<AlertStreamEvent> dedup(AlertStreamEvent event) {
- return deduplicator.dedup(event);
+ if (null != deduplicator) {
+ return deduplicator.dedup(event);
+ } else {
+ return Collections.singletonList(event);
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
index 152a9f1..f40680c 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertEmailPublisher.java
@@ -40,7 +40,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import static org.apache.eagle.alert.service.MetadataServiceClientImpl.*;
import static org.apache.eagle.common.mail.AlertEmailConstants.*;
public class AlertEmailPublisher extends AbstractPublishPlugin implements AlertPublishPluginProvider {
@@ -215,7 +214,6 @@ public class AlertEmailPublisher extends AbstractPublishPlugin implements AlertP
.name("Email")
.type(AlertEmailPublisher.class)
.description("Email alert publisher")
- .field("subject")
.field("sender")
.field("recipients")
.build();
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
index 5b902f9..e38799f 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/AlertPublisherImpl.java
@@ -43,7 +43,10 @@ public class AlertPublisherImpl implements AlertPublisher {
private final String name;
- private volatile Map<PublishPartition, AlertPublishPlugin> publishPluginMapping = new ConcurrentHashMap<>(1);
+ // <publishId, PublishPlugin>
+ private volatile Map<String, AlertPublishPlugin> publishPluginMapping = new ConcurrentHashMap<>(1);
+ //private volatile Map<PublishPartition, AlertPublishPlugin> publishPluginMapping = new ConcurrentHashMap<>(1);
+
private Config config;
private Map conf;
@@ -73,11 +76,11 @@ public class AlertPublisherImpl implements AlertPublisher {
private void notifyAlert(PublishPartition partition, AlertStreamEvent event) {
// remove the column values for publish plugin match
partition.getColumnValues().clear();
- if (!publishPluginMapping.containsKey(partition)) {
+ if (!publishPluginMapping.containsKey(partition.getPublishId())) {
LOG.warn("PublishPartition {} is not found in publish plugin map", partition);
return;
}
- AlertPublishPlugin plugin = publishPluginMapping.get(partition);
+ AlertPublishPlugin plugin = publishPluginMapping.get(partition.getPublishId());
if (plugin == null) {
LOG.warn("PublishPartition {} has problems while initializing publish plugin", partition);
return;
@@ -120,7 +123,7 @@ public class AlertPublisherImpl implements AlertPublisher {
}
// copy and swap to avoid concurrency issue
- Map<PublishPartition, AlertPublishPlugin> newPublishMap = new HashMap<>(publishPluginMapping);
+ Map<String, AlertPublishPlugin> newPublishMap = new HashMap<>(publishPluginMapping);
// added
for (Publishment publishment : added) {
@@ -128,9 +131,7 @@ public class AlertPublisherImpl implements AlertPublisher {
AlertPublishPlugin plugin = AlertPublishPluginsFactory.createNotificationPlugin(publishment, config, conf);
if (plugin != null) {
- for (PublishPartition p : getPublishPartitions(publishment)) {
- newPublishMap.put(p, plugin);
- }
+ newPublishMap.put(publishment.getName(), plugin);
} else {
LOG.error("OnPublishChange alertPublisher {} failed due to invalid format", publishment);
}
@@ -138,16 +139,9 @@ public class AlertPublisherImpl implements AlertPublisher {
//removed
List<AlertPublishPlugin> toBeClosed = new ArrayList<>();
for (Publishment publishment : removed) {
- AlertPublishPlugin plugin = null;
- for (PublishPartition p : getPublishPartitions(publishment)) {
- if (plugin == null) {
- plugin = newPublishMap.remove(p);
- } else {
- newPublishMap.remove(p);
- }
- }
- if (plugin != null) {
- toBeClosed.add(plugin);
+ AlertPublishPlugin publishPlugin = newPublishMap.remove(publishment.getName());
+ if (publishPlugin != null) {
+ toBeClosed.add(publishPlugin);
}
}
// updated
@@ -155,16 +149,11 @@ public class AlertPublisherImpl implements AlertPublisher {
// for updated publishment, need to init them too
AlertPublishPlugin newPlugin = AlertPublishPluginsFactory.createNotificationPlugin(publishment, config, conf);
if (newPlugin != null) {
- AlertPublishPlugin plugin = null;
- for (PublishPartition p : getPublishPartitions(publishment)) {
- if (plugin == null) {
- plugin = newPublishMap.get(p);
- }
- newPublishMap.put(p, newPlugin);
- }
- if (plugin != null) {
- toBeClosed.add(plugin);
+ AlertPublishPlugin oldPlugin = newPublishMap.get(publishment.getName());
+ if (oldPlugin != null) {
+ toBeClosed.add(oldPlugin);
}
+ newPublishMap.put(publishment.getName(), newPlugin);
} else {
LOG.error("OnPublishChange alertPublisher {} failed due to invalid format", publishment);
}
@@ -199,7 +188,7 @@ public class AlertPublisherImpl implements AlertPublisher {
try {
p.close();
} catch (Exception e) {
- LOG.error(String.format("Error when close publish plugin {}!", p.getClass().getCanonicalName()), e);
+ LOG.error("Error when close publish plugin {}!", p.getClass().getCanonicalName(), e);
}
}
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
index ac99db3..54d551e 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
@@ -20,6 +20,7 @@ package org.apache.eagle.alert.engine.publisher.impl;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.alert.engine.coordinator.AlertDeduplication;
import org.apache.eagle.alert.engine.coordinator.StreamDefinition;
import org.apache.eagle.alert.engine.model.AlertStreamEvent;
import org.apache.eagle.alert.engine.publisher.AlertDeduplicator;
@@ -59,6 +60,13 @@ public class DefaultDeduplicator implements AlertDeduplicator {
this.dedupIntervalSec = intervalMin;
}
+ public DefaultDeduplicator(AlertDeduplication alertDeduplication) {
+ this.customDedupFields = alertDeduplication.getDedupFields();
+ this.dedupIntervalSec = Integer.parseInt(alertDeduplication.getDedupIntervalMin()) * 60;
+ this.withoutStatesCache = CacheBuilder.newBuilder().expireAfterWrite(
+ this.dedupIntervalSec, TimeUnit.SECONDS).build();
+ }
+
public DefaultDeduplicator(String intervalMin, List<String> customDedupFields,
String dedupStateField, String dedupStateCloseValue, DedupCache dedupCache) {
setDedupIntervalMin(intervalMin);
@@ -81,7 +89,7 @@ public class DefaultDeduplicator implements AlertDeduplicator {
* @param key
* @return
*/
- public List<AlertStreamEvent> checkDedup(AlertStreamEvent event, EventUniq key, String stateFiledValue) {
+ private List<AlertStreamEvent> checkDedup(AlertStreamEvent event, EventUniq key, String stateFiledValue) {
if (StringUtils.isBlank(stateFiledValue)) {
// without state field, we cannot determine whether it is duplicated
// without custom filed values, we cannot determine whether it is duplicated
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngine.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngine.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngine.java
index 87a067f..c0b765a 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngine.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngine.java
@@ -151,9 +151,13 @@ public class VelocityAlertTemplateEngine implements AlertTemplateEngine {
context.put(AlertContextFields.ALERT_ID, event.getAlertId());
context.put(AlertContextFields.CREATED_BY, event.getCreatedBy());
context.put(AlertContextFields.CREATED_TIMESTAMP, event.getCreatedTime());
- context.put(AlertContextFields.CREATED_TIME, DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()));
+ context.put(AlertContextFields.CREATED_TIME, String.format("%s %s",
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()),
+ DateTimeUtil.CURRENT_TIME_ZONE.getID()));
context.put(AlertContextFields.ALERT_TIMESTAMP, event.getTimestamp());
- context.put(AlertContextFields.ALERT_TIME, DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getTimestamp()));
+ context.put(AlertContextFields.ALERT_TIME, String.format("%s %s",
+ DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getTimestamp()),
+ DateTimeUtil.CURRENT_TIME_ZONE.getID()));
context.put(AlertContextFields.ALERT_SCHEMA, event.getSchema());
context.put(AlertContextFields.ALERT_EVENT, event);
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
index 44a5fe9..d6829d6 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/runner/AlertPublisherBolt.java
@@ -33,6 +33,7 @@ import org.apache.eagle.alert.engine.publisher.AlertPublisher;
import org.apache.eagle.alert.engine.publisher.AlertStreamFilter;
import org.apache.eagle.alert.engine.publisher.PipeStreamFilter;
import org.apache.eagle.alert.engine.publisher.impl.AlertPublisherImpl;
+import org.apache.eagle.alert.engine.publisher.impl.DefaultDeduplicator;
import org.apache.eagle.alert.engine.publisher.template.AlertTemplateEngine;
import org.apache.eagle.alert.engine.publisher.template.AlertTemplateProvider;
import org.apache.eagle.alert.utils.AlertConstants;
@@ -43,6 +44,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPublishSpecListener {
@@ -51,6 +53,7 @@ public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPubli
private volatile Map<String, Publishment> cachedPublishments = new HashMap<>();
private volatile Map<String, PolicyDefinition> policyDefinitionMap;
private volatile Map<String, StreamDefinition> streamDefinitionMap;
+ private volatile Map<String, DefaultDeduplicator> deduplicatorMap = new ConcurrentHashMap<>();
private AlertTemplateEngine alertTemplateEngine;
private boolean logEventEnabled;
@@ -87,6 +90,13 @@ public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPubli
if (logEventEnabled) {
LOG.info("Alert publish bolt {}/{} with partition {} received event: {}", this.getBoltId(), this.context.getThisTaskId(), partition, event);
}
+ if (deduplicatorMap != null && deduplicatorMap.containsKey(event.getPolicyId())) {
+ List<AlertStreamEvent> eventList = deduplicatorMap.get(event.getPolicyId()).dedup(event);
+ if (eventList == null || eventList.isEmpty()) {
+ collector.ack(input);
+ return;
+ }
+ }
AlertStreamEvent filteredEvent = alertFilter.filter(event);
if (filteredEvent != null) {
alertPublisher.nextEvent(partition, filteredEvent);
@@ -139,7 +149,7 @@ public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPubli
}
@Override
- public void onAlertPolicyChange(Map<String, PolicyDefinition> pds, Map<String, StreamDefinition> sds) {
+ public synchronized void onAlertPolicyChange(Map<String, PolicyDefinition> pds, Map<String, StreamDefinition> sds) {
List<String> policyToRemove = new ArrayList<>();
if (this.policyDefinitionMap != null) {
policyToRemove.addAll(this.policyDefinitionMap.keySet().stream().filter(policyId -> !pds.containsKey(policyId)).collect(Collectors.toList()));
@@ -151,6 +161,9 @@ public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPubli
for (Map.Entry<String, PolicyDefinition> entry : pds.entrySet()) {
try {
this.alertTemplateEngine.register(entry.getValue());
+ if (entry.getValue().getDeduplication() != null) {
+ this.deduplicatorMap.put(entry.getKey(), new DefaultDeduplicator(entry.getValue().getDeduplication()));
+ }
} catch (Throwable throwable) {
LOG.error("Failed to register policy {} in template engine", entry.getKey(), throwable);
}
@@ -159,6 +172,9 @@ public class AlertPublisherBolt extends AbstractStreamBolt implements AlertPubli
for (String policyId : policyToRemove) {
try {
this.alertTemplateEngine.unregister(policyId);
+ if (deduplicatorMap != null && deduplicatorMap.containsKey(policyId)) {
+ deduplicatorMap.remove(policyId);
+ }
} catch (Throwable throwable) {
LOG.error("Failed to unregister policy {} from template engine", policyId, throwable);
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_INLINED_TEMPLATE.vm
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_INLINED_TEMPLATE.vm b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_INLINED_TEMPLATE.vm
index 0e3d5fe..70013c3 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_INLINED_TEMPLATE.vm
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/resources/ALERT_INLINED_TEMPLATE.vm
@@ -145,7 +145,7 @@
<td class="content-block"
style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; vertical-align: top; margin: 0; padding: 0 0 20px;"
valign="top">
- <small>CATEGORY:</small> <strong style="color: $alertColor">#if($alert["alertCategory"]) $alert["alertCategory"] #else N/A #end</strong> <small>TIME:</small> <strong>$alert["alertTime"]</strong>
+ <small>CATEGORY:</small> <strong style="color: $alertColor">#if($alert["alertCategory"]) $alert["alertCategory"] #else N/A #end</strong> <small>CREATE TIME:</small> <strong>$alert["alertTime"]</strong>
</td>
</tr>
<tr style="font-family: 'Helvetica Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; margin: 0;">
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngineTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngineTest.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngineTest.java
index 7b1d494..e5ec474 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngineTest.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/test/java/org/apache/eagle/alert/engine/publisher/template/VelocityAlertTemplateEngineTest.java
@@ -36,7 +36,7 @@ public class VelocityAlertTemplateEngineTest {
templateEngine.init(ConfigFactory.load());
templateEngine.register(mockPolicy("testPolicy"));
AlertStreamEvent event = templateEngine.filter(mockAlertEvent("testPolicy"));
- Assert.assertEquals("Alert (2016-11-30 07:31:15): cpu usage on hadoop of cluster test_cluster at localhost is 0.98, " +
+ Assert.assertEquals("Alert (2016-11-30 07:31:15 UTC): cpu usage on hadoop of cluster test_cluster at localhost is 0.98, " +
"exceeding thread hold: 90%. (policy: testPolicy, description: Policy for monitoring cpu usage > 90%), " +
"definition: from HADOOP_JMX_METRIC_STREAM[site == \"test_cluster\" and metric == \"cpu.usage\" and value > 0.9] " +
"select site, metric, host, role, value insert into capacityUsageAlert", event.getBody());
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java
index e0b5c9d..6427d8c 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataDaoImpl.java
@@ -227,7 +227,7 @@ public class JdbcMetadataDaoImpl implements IMetadataDao {
@Override
public OpResult removePolicy(String policyId) {
- return handler.removeById(PolicyDefinition.class.getSimpleName(), policyId);
+ return handler.removePolicyById(PolicyDefinition.class.getSimpleName(), policyId);
}
@Override
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataHandler.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataHandler.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataHandler.java
index a9e3c5e..7fffa55 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataHandler.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-metadata-parent/alert-metadata/src/main/java/org/apache/eagle/alert/metadata/impl/JdbcMetadataHandler.java
@@ -455,6 +455,36 @@ public class JdbcMetadataHandler {
return result;
}
+ public OpResult removePolicyById(String clzName, String policyId) {
+ Connection connection = null;
+ PreparedStatement statement = null;
+ OpResult result = new OpResult();
+ try {
+ String tb = getTableName(clzName);
+ connection = dataSource.getConnection();
+ connection.setAutoCommit(false);
+ statement = connection.prepareStatement(String.format(DELETE_STATEMENT, tb));
+ statement.setString(1, policyId);
+ int status = statement.executeUpdate();
+ LOG.info("delete {} policy {} from {}", status, policyId, tb);
+ closeResource(null, statement, null);
+
+ statement = connection.prepareStatement(DELETE_PUBLISHMENT_STATEMENT);
+ statement.setString(1, policyId);
+ status = statement.executeUpdate();
+ LOG.info("delete {} records from policy_publishment", status);
+
+ connection.commit();
+ connection.setAutoCommit(true);
+ } catch (SQLException e) {
+ e.printStackTrace();
+ } finally {
+ closeResource(null, statement, connection);
+ }
+ LOG.info(result.message);
+ return result;
+ }
+
public OpResult removeById(String clzName, String key) {
Connection connection = null;
PreparedStatement statement = null;
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/JHFEventReaderBaseTest.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/JHFEventReaderBaseTest.java b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/JHFEventReaderBaseTest.java
new file mode 100644
index 0000000..9c8146d
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/java/org/apache/eagle/jpm/mr/history/JHFEventReaderBaseTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.mr.history;
+
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilter;
+import org.apache.eagle.jpm.mr.history.crawler.JobHistoryContentFilterBuilder;
+import org.apache.eagle.jpm.mr.history.parser.JHFMRVer2EventReader;
+import org.apache.eagle.jpm.mr.history.parser.JobConfigurationCreationServiceListener;
+import org.apache.eagle.jpm.mr.historyentity.JobBaseAPIEntity;
+import org.apache.eagle.jpm.mr.historyentity.JobConfigurationAPIEntity;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+public class JHFEventReaderBaseTest {
+
+ @Test
+ public void testParseConfiguration() throws Exception {
+ Configuration conf = new org.apache.hadoop.conf.Configuration();
+ conf.addResource("job_1479206441898_508949_conf.xml");
+
+ final JobHistoryContentFilterBuilder builder = JobHistoryContentFilterBuilder.newBuilder().acceptJobFile().acceptJobConfFile();
+ List<String> confKeyPatterns = new ArrayList<>();
+ confKeyPatterns.add(Constants.JobConfiguration.CASCADING_JOB);
+ confKeyPatterns.add(Constants.JobConfiguration.HIVE_JOB);
+ confKeyPatterns.add(Constants.JobConfiguration.PIG_JOB);
+ confKeyPatterns.add(Constants.JobConfiguration.SCOOBI_JOB);
+ for (String key : confKeyPatterns) {
+ builder.includeJobKeyPatterns(Pattern.compile(key));
+ }
+ JobHistoryContentFilter filter = builder.build();
+
+ MRHistoryJobConfig appConfig = MRHistoryJobConfig.newInstance(ConfigFactory.load());
+ Map<String, String> tags = new HashMap<>();
+ tags.put("site", "sandbox");
+ tags.put("jobId", "job_1490593856016_152289");
+ tags.put("jobType", "HIVE");
+ tags.put("jobDefId", "INSERT OVERWRITE TABLE kyl...'2017-04-06')))(Stage-1)");
+ JHFMRVer2EventReader reader = new JHFMRVer2EventReader(tags, conf, filter, appConfig);
+ reader.addListener(new JobConfigurationCreationServiceListener(appConfig.getEagleServiceConfig()) {
+ @Override
+ public void jobEntityCreated(JobBaseAPIEntity entity) throws Exception {
+ Assert.assertTrue(null != entity);
+ Assert.assertTrue(entity instanceof JobConfigurationAPIEntity);
+ JobConfigurationAPIEntity configurationAPIEntity = (JobConfigurationAPIEntity) entity;
+ Assert.assertTrue(configurationAPIEntity.getJobConfig().getConfig().size() == 1);
+ }
+ });
+ reader.parseConfiguration();
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application.conf b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application.conf
new file mode 100644
index 0000000..00b14a8
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/application.conf
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ "appId":"mrHistoryJob",
+ "mode":"LOCAL",
+ "workers" : 3,
+ "siteId" : "sandbox",
+ application.storm.nimbusHost=localhost
+
+ "stormConfig" : {
+ "mrHistoryJobSpoutTasks" : 6,
+ "jobKafkaSinkTasks" : 1,
+ "taskAttemptKafkaSinkTasks" : 1
+ },
+
+ "zookeeper" : {
+ "zkQuorum" : "sandbox.hortonworks.com:2181",
+ "zkRoot" : "/test_mrjobhistory",
+ "zkSessionTimeoutMs" : 15000,
+ "zkRetryTimes" : 3,
+ "zkRetryInterval" : 20000
+ },
+
+ "endpointConfig" : {
+ "timeZone" : "UTC",
+ "mrHistoryServerUrl" : "http://sandbox.hortonworks.com:19888",
+ "basePath" : "/mr-history/done",
+ "hdfs" : {
+ fs.defaultFS : "hdfs://sandbox.hortonworks.com:8020",
+ #if not need, then do not set
+ # hdfs.kerberos.principal = ,
+ # hdfs.keytab.file =
+ # ....
+ }
+ },
+
+ "service": {
+ "host": "localhost",
+ "port": 9090,
+ "username": "admin",
+ "password": "secret",
+ "readTimeOutSeconds" : 10,
+ context = "/rest"
+ },
+
+ "dataSinkConfig": {
+ "topic" : "map_reduce_failed_job",
+ "brokerList" : "sandbox.hortonworks.com:6667",
+ "serializerClass" : "kafka.serializer.StringEncoder",
+ "keySerializerClass" : "kafka.serializer.StringEncoder"
+ "producerType" : "async",
+ "numBatchMessages" : "4096",
+ "maxQueueBufferMs" : "5000",
+ "requestRequiredAcks" : "0"
+ },
+
+ "MRConfigureKeys" : {
+ "jobNameKey" : "eagle.job.name",
+ "jobConfigKey" : "mapreduce.map.output.compress,mapreduce.map.output.compress.codec,mapreduce.output.fileoutputformat.compress,mapreduce.output.fileoutputformat.compress.type,mapreduce.output.fileoutputformat.compress.codec,mapred.output.format.class, dataplatform.etl.info,mapreduce.map.memory.mb,mapreduce.reduce.memory.mb,mapreduce.map.java.opts,mapreduce.reduce.java.opts"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/8da06636/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml
index 6d22996..b670a3f 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml
+++ b/eagle-jpm/eagle-jpm-mr-history/src/test/resources/job_1479206441898_508949_conf.xml
@@ -1,18 +1,177 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with ~
+ this work for additional information regarding copyright ownership. ~ The
+ ASF licenses this file to You under the Apache License, Version 2.0 ~ (the
+ "License"); you may not use this file except in compliance with ~ the License.
+ You may obtain a copy of the License at ~ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~ ~ Unless required by applicable law or agreed to in writing, software ~
+ distributed under the License is distributed on an "AS IS" BASIS, ~ WITHOUT
+ WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ~ See the
+ License for the specific language governing permissions and ~ limitations
+ under the License.
-->
-<?xml version="1.0" encoding="UTF-8" standalone="no"?><configuration>
+<configuration>
+ <property>
+ <name>hive.optimize.skewjoin.compiletime</name>
+ <value>false</value>
+ <source>programatically</source>
+ <source>
+ org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@67236f24
+ </source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>hive.query.string</name>
+ <value>
+ select a.phone_number from customer_details a, call_detail_records b where a.phone_number=b.phone_number
+ </value>
+ <source>programatically</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>120</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>mapreduce.reduce.markreset.buffer.percent</name>
+ <value>0.0</value>
+ <source>mapred-default.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>file.client-write-packet-size</name>
+ <value>65536</value>
+ <source>core-default.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>
+ hadoop.http.authentication.simple.anonymous.allowed
+ </name>
+ <value>true</value>
+ <source>core-site.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>hive.querylog.location</name>
+ <value>/tmp/hive</value>
+ <source>programatically</source>
+ <source>
+ org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@67236f24
+ </source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>yarn.timeline-service.leveldb-timeline-store.path</name>
+ <value>/hadoop/yarn/timeline</value>
+ <source>yarn-site.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.proxy-user-privileges.enabled</name>
+ <value>false</value>
+ <source>yarn-default.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>hive.exec.script.allow.partial.consumption</name>
+ <value>false</value>
+ <source>programatically</source>
+ <source>
+ org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@67236f24
+ </source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>hive.server2.global.init.file.location</name>
+ <value>/etc/hive/conf</value>
+ <source>programatically</source>
+ <source>
+ org.apache.hadoop.hive.conf.LoopingByteArrayInputStream@67236f24
+ </source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>
+ yarn.resourcemanager.work-preserving-recovery.scheduling-wait-ms
+ </name>
+ <value>10000</value>
+ <source>yarn-site.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>dfs.datanode.slow.io.warning.threshold.ms</name>
+ <value>300</value>
+ <source>hdfs-default.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>hive.support.concurrency</name>
+ <value>true</value>
+ <source>file:/etc/hive/conf/hive-site.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>mapreduce.reduce.shuffle.merge.percent</name>
+ <value>0.66</value>
+ <source>mapred-site.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
+ <property>
+ <name>mapreduce.task.skip.start.attempts</name>
+ <value>2</value>
+ <source>mapred-default.xml</source>
+ <source>job.xml</source>
+ <source>
+ hdfs://sandbox.hortonworks.com:8020/mr-history/done/2017/04/07/000000/job_1491539903031_0002_conf.xml
+ </source>
+ </property>
</configuration>
\ No newline at end of file
[04/17] eagle git commit: [MINOR] fix metix heatmap chart size bug
Posted by ja...@apache.org.
[MINOR] fix metix heatmap chart size bug
[MINOR] fix metix heatmap chart size bug
Author: koone <lu...@126.com>
Closes #918 from koone/fixbug.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/d182819d
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/d182819d
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/d182819d
Branch: refs/heads/branch-0.5
Commit: d182819d8d72276d3ae29b3904007d0f01ce8c15
Parents: ecb5129
Author: koone <lu...@126.com>
Authored: Mon Apr 17 15:01:14 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Mon Apr 17 15:01:14 2017 -0700
----------------------------------------------------------------------
.../src/main/webapp/app/apps/hbase/ctrls/backupMasterListCtrl.js | 4 +---
.../src/main/webapp/app/apps/hbase/ctrls/regionListCtrl.js | 2 +-
.../src/main/webapp/app/apps/hdfs/ctrl/datanodeListCtrl.js | 2 +-
.../src/main/webapp/app/apps/hdfs/ctrl/namenodeListCtrl.js | 2 +-
4 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/d182819d/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/backupMasterListCtrl.js
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/backupMasterListCtrl.js b/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/backupMasterListCtrl.js
index 2330885..7c289c3 100644
--- a/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/backupMasterListCtrl.js
+++ b/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/backupMasterListCtrl.js
@@ -118,13 +118,11 @@
}
master_status.push([x, y, 0, master.tags.hostname, master.tags.rack, master.status || "-"])
});
- for(var i = 0;i < split; i++){
+ for(var i = 0;i <= split; i++){
master_status_category.push(i);
}
$scope.healthStatusSeries = [getCommonHeatMapSeries("Health", master_status)];
- console.log($scope.healthStatusSeries);
$scope.healthStatusOption = getHealthHeatMapOption();
- console.log($scope.healthStatusOption);
$scope.healthStatusCategory = master_status_category;
$scope.heatmapHeight = {
'height': getHeight(y)
http://git-wip-us.apache.org/repos/asf/eagle/blob/d182819d/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/regionListCtrl.js
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/regionListCtrl.js b/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/regionListCtrl.js
index 401ef15..d83bdf2 100644
--- a/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/regionListCtrl.js
+++ b/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/ctrls/regionListCtrl.js
@@ -121,7 +121,7 @@
regionServer_status.push([x, y, 0, regionServer.tags.hostname, regionServer.tags.rack, regionServer.usedHeapMB, regionServer.maxHeapMB, regionServer.status || "-"]);
});
- for(var i = 0;i < split; i++){
+ for(var i = 0;i <= split; i++){
regionServer_status_category.push(i);
}
$scope.healthStatusSeries = [getCommonHeatMapSeries("Health", regionServer_status)];
http://git-wip-us.apache.org/repos/asf/eagle/blob/d182819d/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/datanodeListCtrl.js
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/datanodeListCtrl.js b/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/datanodeListCtrl.js
index 7664b7c..3db293a 100644
--- a/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/datanodeListCtrl.js
+++ b/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/datanodeListCtrl.js
@@ -121,7 +121,7 @@
}
datanode_status.push([x, y, 0, datanode.tags.hostname, datanode.tags.rack, datanode.status || "-"])
});
- for(var i = 0;i < split; i++){
+ for(var i = 0;i <= split; i++){
datanode_status_category.push(i);
}
$scope.healthStatusSeries = [getCommonHeatMapSeries("Health", datanode_status)];
http://git-wip-us.apache.org/repos/asf/eagle/blob/d182819d/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/namenodeListCtrl.js
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/namenodeListCtrl.js b/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/namenodeListCtrl.js
index d0d41bf..9baf5c0 100644
--- a/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/namenodeListCtrl.js
+++ b/eagle-metric/eagle-hdfs-web/src/main/webapp/app/apps/hdfs/ctrl/namenodeListCtrl.js
@@ -119,7 +119,7 @@
}
namenode_status.push([x, y, 0, namenode.tags.hostname, namenode.tags.rack, namenode.usedCapacityTB, namenode.status || "-"])
});
- for(var i = 0;i < split; i++){
+ for(var i = 0;i <= split; i++){
namenode_status_category.push(i);
}
$scope.healthStatusSeries = [getCommonHeatMapSeries("Health", namenode_status)];
[06/17] eagle git commit: HBase Naming that unify `Hbase` and `HBase`
into `HBase`
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
deleted file mode 100644
index ce977a9..0000000
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/extractor/hbase/HbaseTopologyEntityParser.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.eagle.topology.extractor.hbase;
-
-import org.apache.eagle.app.utils.HadoopSecurityUtil;
-import org.apache.eagle.topology.TopologyCheckAppConfig;
-import org.apache.eagle.topology.TopologyConstants;
-import org.apache.eagle.topology.extractor.TopologyEntityParserResult;
-import org.apache.eagle.topology.entity.HBaseServiceTopologyAPIEntity;
-import org.apache.eagle.topology.extractor.TopologyEntityParser;
-import org.apache.eagle.topology.resolver.TopologyRackResolver;
-import org.apache.eagle.topology.utils.EntityBuilderHelper;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.ServerLoad;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.slf4j.Logger;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.eagle.topology.TopologyConstants.*;
-
-public class HbaseTopologyEntityParser implements TopologyEntityParser {
-
- private static final Logger LOG = org.slf4j.LoggerFactory.getLogger(HbaseTopologyEntityParser.class);
- private Configuration hBaseConfiguration;
- private String site;
- private Boolean kerberosEnable = false;
- private TopologyRackResolver rackResolver;
-
- public HbaseTopologyEntityParser(String site, TopologyCheckAppConfig.HBaseConfig hBaseConfig, TopologyRackResolver resolver) {
- this.site = site;
- this.rackResolver = resolver;
- this.hBaseConfiguration = HBaseConfiguration.create();
- this.hBaseConfiguration.set("hbase.zookeeper.quorum", hBaseConfig.zkQuorum);
- this.hBaseConfiguration.set("hbase.zookeeper.property.clientPort", hBaseConfig.zkClientPort);
- this.hBaseConfiguration.set("zookeeper.znode.parent", hBaseConfig.zkRoot);
- this.hBaseConfiguration.set("hbase.client.retries.number", hBaseConfig.zkRetryTimes);
- // kerberos authentication
- if (hBaseConfig.eaglePrincipal != null && hBaseConfig.eagleKeytab != null
- && !hBaseConfig.eaglePrincipal.isEmpty() && !hBaseConfig.eagleKeytab.isEmpty()) {
- this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_PRINCIPAL_KEY, hBaseConfig.eaglePrincipal);
- this.hBaseConfiguration.set(HadoopSecurityUtil.EAGLE_KEYTAB_FILE_KEY, hBaseConfig.eagleKeytab);
- this.kerberosEnable = true;
- this.hBaseConfiguration.set("hbase.security.authentication", "kerberos");
- this.hBaseConfiguration.set("hbase.master.kerberos.principal", hBaseConfig.hbaseMasterPrincipal);
- }
- }
-
- private HBaseAdmin getHBaseAdmin() throws IOException {
- if (this.kerberosEnable) {
- HadoopSecurityUtil.login(hBaseConfiguration);
- }
- return new HBaseAdmin(this.hBaseConfiguration);
- }
-
-
- @Override
- public TopologyEntityParserResult parse(long timestamp) {
- final TopologyEntityParserResult result = new TopologyEntityParserResult();
- int activeRatio = 0;
- try {
- doParse(timestamp, result);
- activeRatio++;
- } catch (Exception ex) {
- LOG.error(ex.getMessage(), ex);
- }
- result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.HMASTER_ROLE, activeRatio, site, timestamp));
- return result;
- }
-
- private void doParse(long timestamp, TopologyEntityParserResult result) throws IOException {
- long deadServers = 0;
- long liveServers = 0;
- HBaseAdmin admin = null;
- try {
- admin = getHBaseAdmin();
- ClusterStatus status = admin.getClusterStatus();
- deadServers = status.getDeadServers();
- liveServers = status.getServersSize();
-
- for (ServerName liveServer : status.getServers()) {
- ServerLoad load = status.getLoad(liveServer);
- result.getSlaveNodes().add(parseServer(liveServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_LIVE_STATUS, timestamp));
- }
- for (ServerName deadServer : status.getDeadServerNames()) {
- ServerLoad load = status.getLoad(deadServer);
- result.getSlaveNodes().add(parseServer(deadServer, load, TopologyConstants.REGIONSERVER_ROLE, TopologyConstants.REGIONSERVER_DEAD_STATUS, timestamp));
- }
- ServerName master = status.getMaster();
- if (master != null) {
- ServerLoad load = status.getLoad(master);
- result.getMasterNodes().add(parseServer(master, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_ACTIVE_STATUS, timestamp));
- }
- for (ServerName backupMaster : status.getBackupMasters()) {
- ServerLoad load = status.getLoad(backupMaster);
- result.getMasterNodes().add(parseServer(backupMaster, load, TopologyConstants.HMASTER_ROLE, TopologyConstants.HMASTER_STANDBY_STATUS, timestamp));
- }
- double liveRatio = liveServers * 1d / (liveServers + deadServers);
- result.getMetrics().add(EntityBuilderHelper.generateMetric(TopologyConstants.REGIONSERVER_ROLE, liveRatio, site, timestamp));
- LOG.info("live servers: {}, dead servers: {}", liveServers, deadServers);
- } finally {
- if (admin != null) {
- try {
- admin.close();
- } catch (IOException e) {
- LOG.error(e.getMessage(), e);
- }
- }
- }
- }
-
- private HBaseServiceTopologyAPIEntity parseServer(ServerName serverName, ServerLoad serverLoad, String role, String status, long timestamp) {
- if (serverName == null) {
- return null;
- }
- HBaseServiceTopologyAPIEntity entity = createEntity(role, serverName.getHostname(), timestamp);
- parseServerLoad(entity, serverLoad);
- entity.setStatus(status);
- return entity;
- }
-
- private void parseServerLoad(HBaseServiceTopologyAPIEntity entity, ServerLoad load) {
- if (load == null) {
- return;
- }
- entity.setMaxHeapMB(load.getMaxHeapMB());
- entity.setUsedHeapMB(load.getUsedHeapMB());
- entity.setNumRegions(load.getNumberOfRegions());
- entity.setNumRequests(load.getNumberOfRequests());
- }
-
- private HBaseServiceTopologyAPIEntity createEntity(String roleType, String hostname, long timestamp) {
- HBaseServiceTopologyAPIEntity entity = new HBaseServiceTopologyAPIEntity();
- Map<String, String> tags = new HashMap<String, String>();
- entity.setTags(tags);
- tags.put(SITE_TAG, site);
- tags.put(ROLE_TAG, roleType);
- tags.put(HOSTNAME_TAG, hostname);
- String rack = rackResolver.resolve(hostname);
- tags.put(RACK_TAG, rack);
- entity.setLastUpdateTime(timestamp);
- entity.setTimestamp(timestamp);
- return entity;
- }
-
- @Override
- public TopologyConstants.TopologyType getTopologyType() {
- return TopologyType.HBASE;
- }
-
- @Override
- public TopologyConstants.HadoopVersion getHadoopVersion() {
- return HadoopVersion.V2;
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
index aa36c53..2e7a23c 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
+++ b/eagle-topology-check/eagle-topology-app/src/main/resources/META-INF/providers/org.apache.eagle.topology.TopologyCheckAppProvider.xml
@@ -81,17 +81,17 @@
<property>
<name>dataSourceConfig.hbase.zkQuorum</name>
<displayName>HBase Zookeeper Quorum</displayName>
- <description>hbase zookeeper quorum (optional)</description>
+ <description>hBase zookeeper quorum (optional)</description>
</property>
<property>
<name>dataSourceConfig.hbase.zkZnodeParent</name>
- <displayName>Hbase Zookeeper Znode Parent Root</displayName>
- <description>hbase zookeeper znode parent root (optional)</description>
+ <displayName>HBase Zookeeper Znode Parent Root</displayName>
+ <description>hBase zookeeper znode parent root (optional)</description>
</property>
<property>
<name>dataSourceConfig.hbase.zkPropertyClientPort</name>
- <displayName>Hbase Zookeeper Client Port</displayName>
- <description>hbase zookeeper client port (optional)</description>
+ <displayName>HBase Zookeeper Client Port</displayName>
+ <description>hBase zookeeper client port (optional)</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHBaseTopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHBaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHBaseTopologyCrawler.java
new file mode 100644
index 0000000..8fbedb3
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHBaseTopologyCrawler.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.topology;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.topology.extractor.hbase.HBaseTopologyCrawler;
+import org.apache.eagle.topology.resolver.TopologyRackResolver;
+import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestHBaseTopologyCrawler {
+
+ @Test @Ignore
+ public void test() {
+ Config config = ConfigFactory.load();
+
+ TopologyCheckAppConfig topologyCheckAppConfig = TopologyCheckAppConfig.newInstance(config);
+ TopologyRackResolver rackResolver = new DefaultTopologyRackResolver();
+ HBaseTopologyCrawler crawler = new HBaseTopologyCrawler(topologyCheckAppConfig, rackResolver, null);
+ crawler.extract();
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHbaseTopologyCrawler.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHbaseTopologyCrawler.java b/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHbaseTopologyCrawler.java
deleted file mode 100644
index 11b6c15..0000000
--- a/eagle-topology-check/eagle-topology-app/src/test/java/org/apache/eagle/topology/TestHbaseTopologyCrawler.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.topology;
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import org.apache.eagle.topology.extractor.hbase.HbaseTopologyCrawler;
-import org.apache.eagle.topology.resolver.TopologyRackResolver;
-import org.apache.eagle.topology.resolver.impl.DefaultTopologyRackResolver;
-import org.junit.Ignore;
-import org.junit.Test;
-
-public class TestHbaseTopologyCrawler {
-
- @Test @Ignore
- public void test() {
- Config config = ConfigFactory.load();
-
- TopologyCheckAppConfig topologyCheckAppConfig = TopologyCheckAppConfig.newInstance(config);
- TopologyRackResolver rackResolver = new DefaultTopologyRackResolver();
- HbaseTopologyCrawler crawler = new HbaseTopologyCrawler(topologyCheckAppConfig, rackResolver, null);
- crawler.extract();
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
index 189af22..7be82e4 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
@@ -24,7 +24,7 @@ public class TopologyConstants {
public static final String SYSTEM_INSTANCE_SERVICE_NAME = "SystemServiceInstance";
public static final String HDFS_INSTANCE_SERVICE_NAME = "HdfsServiceInstance";
- public static final String HBASE_INSTANCE_SERVICE_NAME = "HbaseServiceInstance";
+ public static final String HBASE_INSTANCE_SERVICE_NAME = "HBaseServiceInstance";
public static final String MR_INSTANCE_SERVICE_NAME = "MRServiceInstance";
public static final String JN_INSTANCE_SERVICE_NAME = "JNServiceInstance";
public static final String GENERIC_METRIC_SERVICE = "GenericMetricService";
[03/17] eagle git commit: [EAGLE-982] The log length has exceeded the
limit of 4 MB in Travis
Posted by ja...@apache.org.
[EAGLE-982] The log length has exceeded the limit of 4 MB in Travis
Maven throw too many warnings for checking `Code Style` in `Travis`. Should we disable it for a while until we fix those `Code Style` issues?
```
The log length has exceeded the limit of 4 MB (this usually means that the test suite is raising the same exception over and over).
The job has been terminated
```
(https://issues.apache.org/jira/browse/EAGLE-982)
Author: asdf2014 <15...@qq.com>
Author: Benedict Jin <16...@cnsuning.com>
Closes #897 from asdf2014/travis_log.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/ecb5129e
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/ecb5129e
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/ecb5129e
Branch: refs/heads/branch-0.5
Commit: ecb5129ec5b1ccc2be528a3ae98d94754a3b92df
Parents: fc7281d
Author: asdf2014 <15...@qq.com>
Authored: Mon Apr 17 14:10:36 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Mon Apr 17 14:10:36 2017 -0700
----------------------------------------------------------------------
.travis.yml | 18 ++++++++++++++++--
pom.xml | 2 +-
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/ecb5129e/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index cfe5014..509f795 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,20 @@
# limitations under the License.
language: java
-script: mvn clean test
+
jdk:
- - oraclejdk8
\ No newline at end of file
+ - oraclejdk8
+
+before_install: sudo echo "MAVEN_OPTS='-Xmx2048m -Xms1024m -Dorg.slf4j.simpleLogger.defaultLogLevel=error'" > ~/.mavenrc
+
+script: mvn clean test
+
+env:
+ global:
+ - MAVEN_OPTS: "-Xmx2048m -Xms1024m -Dorg.slf4j.simpleLogger.defaultLogLevel=error"
+
+sudo: required
+
+cache:
+ directories:
+ - $HOME/.m2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/ecb5129e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7862e6f..5115602 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1126,7 +1126,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>${maven-surefire.version}</version>
<configuration>
- <argLine>-XX:-UseGCOverheadLimit</argLine>
+ <argLine>-Xmx2048m -Xms1024m -XX:MaxPermSize=512m -XX:-UseGCOverheadLimit</argLine>
<forkMode>always</forkMode>
</configuration>
<dependencies>
[10/17] eagle git commit: [MINOR] Add license and exclude file end
with sdl.
Posted by ja...@apache.org.
[MINOR] Add license and exclude file end with sdl.
Add license and exclude file end with sdl.
Author: chitin <ch...@gmail.com>
Closes #921 from chitin/fix998.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/8b402e7c
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/8b402e7c
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/8b402e7c
Branch: refs/heads/branch-0.5
Commit: 8b402e7cd003e13d95cff19b75b6d205b45f3299
Parents: 3b66c0d
Author: chitin <ch...@gmail.com>
Authored: Tue Apr 18 19:22:13 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Tue Apr 18 19:22:13 2017 +0800
----------------------------------------------------------------------
eagle-external/eagle-cdh/README.md | 19 +++++++++++
eagle-external/eagle-cdh/assembly.xml | 36 +++++++++-----------
eagle-external/eagle-cdh/src/scripts/control.sh | 1 -
pom.xml | 1 +
4 files changed, 36 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/8b402e7c/eagle-external/eagle-cdh/README.md
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/README.md b/eagle-external/eagle-cdh/README.md
index d626aa4..b21e93a 100644
--- a/eagle-external/eagle-cdh/README.md
+++ b/eagle-external/eagle-cdh/README.md
@@ -1,3 +1,22 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
EAGLE Parcel and CSD for Cloudera's CDH 5
=============
This Eagle Parcel and CSD greatly simplifies setting up a Eagle cluster on your Cloudera cluster.
http://git-wip-us.apache.org/repos/asf/eagle/blob/8b402e7c/eagle-external/eagle-cdh/assembly.xml
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/assembly.xml b/eagle-external/eagle-cdh/assembly.xml
index 5eb9daf..523d9b0 100644
--- a/eagle-external/eagle-cdh/assembly.xml
+++ b/eagle-external/eagle-cdh/assembly.xml
@@ -1,23 +1,19 @@
-<!--
-# Copyright (c) 2015 Rakuten Marketing, LLC
-# Licensed to Rakuten Marketing, LLC under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. Rakuten Marketing licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Author: Aiman Najjar <ai...@rakuten.com>
--->
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
<assembly>
<id>assemble</id>
<formats>
http://git-wip-us.apache.org/repos/asf/eagle/blob/8b402e7c/eagle-external/eagle-cdh/src/scripts/control.sh
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/src/scripts/control.sh b/eagle-external/eagle-cdh/src/scripts/control.sh
index ad4230c..c7e82c6 100644
--- a/eagle-external/eagle-cdh/src/scripts/control.sh
+++ b/eagle-external/eagle-cdh/src/scripts/control.sh
@@ -1,5 +1,4 @@
#!/bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
http://git-wip-us.apache.org/repos/asf/eagle/blob/8b402e7c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5115602..f463221 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1323,6 +1323,7 @@
<!-- Java Object serial file -->
<exclude>**/test/resources/appinfobyteint</exclude>
<exclude>**/test/resources/appinfobytelong</exclude>
+ <exclude>*.sdl</exclude>
</excludes>
</configuration>
</execution>
[02/17] eagle git commit: [EAGLE-1007] Fix memory leak in
ExecutionRuntimeManager
Posted by ja...@apache.org.
[EAGLE-1007] Fix memory leak in ExecutionRuntimeManager
https://issues.apache.org/jira/browse/EAGLE-1007
Author: Zhao, Qingwen <qi...@apache.org>
Closes #917 from qingwen220/EAGLE-1007.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/fc7281d7
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/fc7281d7
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/fc7281d7
Branch: refs/heads/branch-0.5
Commit: fc7281d715f57f2f911079d24f5c09b1f79c50cf
Parents: d20f2e1
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Mon Apr 17 20:42:16 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Mon Apr 17 20:42:16 2017 +0800
----------------------------------------------------------------------
.../app/environment/AbstractEnvironment.java | 10 +++++
.../ExecutionRuntimeManagerTest.java | 45 ++++++++++++++++++++
2 files changed, 55 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/fc7281d7/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/AbstractEnvironment.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/AbstractEnvironment.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/AbstractEnvironment.java
index ec2ffe1..5e3a38b 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/AbstractEnvironment.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/environment/AbstractEnvironment.java
@@ -62,6 +62,16 @@ public abstract class AbstractEnvironment implements Environment {
.append(this.config()).build();
}
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof AbstractEnvironment) {
+ Environment environment = (Environment) obj;
+ return this.getClass().equals(obj.getClass())
+ && this.config.equals(environment.config());
+ }
+ return false;
+ }
+
public StreamProvider stream() {
return streamProvider;
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/fc7281d7/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/ExecutionRuntimeManagerTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/ExecutionRuntimeManagerTest.java b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/ExecutionRuntimeManagerTest.java
new file mode 100644
index 0000000..eacda9f
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-base/src/test/java/org/apache/eagle/app/environment/ExecutionRuntimeManagerTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.app.environment;
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import org.apache.eagle.app.environment.impl.StormEnvironment;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.Map;
+
+public class ExecutionRuntimeManagerTest {
+
+ @Test
+ public void test() throws NoSuchFieldException, IllegalAccessException {
+ Config config = ConfigFactory.load();
+ ExecutionRuntimeManager manager = ExecutionRuntimeManager.getInstance();
+ manager.getRuntime(StormEnvironment.class, config);
+ manager.getRuntime(StormEnvironment.class, config);
+
+ Field field = manager.getClass().getDeclaredField("executionRuntimeCache");
+ field.setAccessible(true);
+ Map<Environment, ExecutionRuntime> executionRuntimeCache = (Map<Environment, ExecutionRuntime>) field.get(manager);
+
+ Assert.assertTrue(executionRuntimeCache.size() == 1);
+
+ }
+}
[08/17] eagle git commit: HBase Naming that unify `Hbase` and `HBase`
into `HBase`
Posted by ja...@apache.org.
HBase Naming that unify `Hbase` and `HBase` into `HBase`
HBase Naming that unify `Hbase` and `HBase` into `HBase`
(https://issues.apache.org/jira/browse/EAGLE-992)
Author: asdf2014 <15...@qq.com>
Closes #905 from asdf2014/hbase_naming.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/cc5d23ba
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/cc5d23ba
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/cc5d23ba
Branch: refs/heads/branch-0.5
Commit: cc5d23baed2a30bafc31bba713cc1dbffed86971
Parents: 391c672
Author: asdf2014 <15...@qq.com>
Authored: Tue Apr 18 01:05:26 2017 -0700
Committer: Jay <jh...@gmail.com>
Committed: Tue Apr 18 01:05:26 2017 -0700
----------------------------------------------------------------------
.../apache/eagle/common/config/EagleConfig.java | 2 +-
.../eagle/common/config/EagleConfigFactory.java | 2 +-
.../eagle/service/hbase/EmbeddedHBase.java | 141 ++++++++++++++
.../eagle/service/hbase/EmbeddedHbase.java | 141 --------------
.../eagle/service/hbase/EmbeddedHBaseTest.java | 28 +++
.../eagle/service/hbase/EmbeddedHbaseTest.java | 28 ---
.../eagle/audit/listener/TestAuditSupport.java | 2 +-
.../eagle/service/client/ClientTestBase.java | 6 +-
.../log/entity/TestHBaseWritePerformance.java | 100 ++++++++++
.../log/entity/TestHbaseWritePerformance.java | 100 ----------
.../apache/eagle/query/TestHBaseLogReader2.java | 4 +-
.../EagleServiceSelfCheckAPIEntity.java | 8 +-
.../EagleServiceSelfCheckResource.java | 4 +-
.../storage/hbase/HBaseEntitySchemaManager.java | 2 +-
.../security/entity/HBaseResourceEntity.java | 105 ++++++++++
.../security/entity/HbaseResourceEntity.java | 105 ----------
.../service/HBaseSensitivityEntity.java | 4 +-
.../security/service/InMemMetadataDaoImpl.java | 3 +-
.../service/JDBCSecurityMetadataDAO.java | 4 +-
.../audit/JDBCSecurityMetadataDAOTest.java | 2 +-
.../audit/TestMetaDataAccessConfigRepo.java | 2 +-
.../eagle-security-hbase-auditlog/README.md | 2 +-
.../hbase/HBaseAuditLogApplication.java | 2 +-
.../security/hbase/HBaseAuditLogObject.java | 28 +++
.../security/hbase/HBaseAuditLogParser.java | 148 +++++++++++++++
.../security/hbase/HBaseAuditLogParserBolt.java | 4 +-
.../HBaseResourceSensitivityDataJoinBolt.java | 80 ++++++++
.../hbase/HBaseSensitivityDataEnrichLCM.java | 2 +-
.../security/hbase/HbaseAuditLogObject.java | 28 ---
.../security/hbase/HbaseAuditLogParser.java | 148 ---------------
.../HbaseResourceSensitivityDataJoinBolt.java | 80 --------
....security.hbase.HBaseAuditLogAppProvider.xml | 4 +-
.../security/hbase/TestHBaseAuditLogParser.java | 44 +++++
.../security/hbase/TestHbaseAuditLogParser.java | 44 -----
.../hbase/HBaseMetadataBrowseWebResource.java | 189 ++++++++++++++++++
.../hbase/HBaseMetadataBrowseWebResponse.java | 45 +++++
.../hbase/HbaseMetadataBrowseWebResource.java | 190 -------------------
.../hbase/HbaseMetadataBrowseWebResponse.java | 45 -----
.../hbase/dao/HBaseMetadataDAOImpl.java | 95 ++++++++++
.../hbase/dao/HbaseMetadataDAOImpl.java | 95 ----------
.../hbase/resolver/HBaseActionResolver.java | 41 ++++
.../hbase/resolver/HBaseMetadataResolver.java | 130 +++++++++++++
.../hbase/resolver/HBaseRequestResolver.java | 67 +++++++
.../resolver/HBaseSensitivityTypeResolver.java | 58 ++++++
.../hbase/resolver/HbaseActionResolver.java | 41 ----
.../hbase/resolver/HbaseMetadataResolver.java | 130 -------------
.../hbase/resolver/HbaseRequestResolver.java | 67 -------
.../resolver/HbaseSensitivityTypeResolver.java | 58 ------
.../hbase/TestHBaseMetadataResolver.java | 41 ++++
.../hbase/TestHbaseMetadataResolver.java | 41 ----
.../extractor/TopologyExtractorFactory.java | 5 +-
.../extractor/hbase/HBaseTopologyCrawler.java | 58 ++++++
.../hbase/HBaseTopologyEntityParser.java | 176 +++++++++++++++++
.../extractor/hbase/HbaseTopologyCrawler.java | 61 ------
.../hbase/HbaseTopologyEntityParser.java | 176 -----------------
....eagle.topology.TopologyCheckAppProvider.xml | 10 +-
.../topology/TestHBaseTopologyCrawler.java | 40 ++++
.../topology/TestHbaseTopologyCrawler.java | 40 ----
.../eagle/topology/TopologyConstants.java | 2 +-
59 files changed, 1651 insertions(+), 1657 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java
index 14e2451..008a7e6 100755
--- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfig.java
@@ -29,7 +29,7 @@ public interface EagleConfig {
HTableInterface getHTable(String tableName);
- Configuration getHbaseConf();
+ Configuration getHBaseConf();
String getStorageType();
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java
index e95e65c..6ad655a 100755
--- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/config/EagleConfigFactory.java
@@ -153,7 +153,7 @@ public class EagleConfigFactory implements EagleConfig {
}
@Override
- public Configuration getHbaseConf() {
+ public Configuration getHBaseConf() {
return hbaseConf;
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java
new file mode 100644
index 0000000..e2942f6
--- /dev/null
+++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHBase.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.hbase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+@Deprecated
+public class EmbeddedHBase {
+ private HBaseTestingUtility util;
+ private MiniHBaseCluster hbaseCluster;
+ private static EmbeddedHBase hbase;
+ private int port;
+ private String znode;
+ private static int DEFAULT_PORT = 2181;
+ private static String DEFAULT_ZNODE = "/hbase-unsecure";
+ private static final Logger LOG = LoggerFactory.getLogger(EmbeddedHBase.class);
+
+ private EmbeddedHBase(int port, String znode) {
+ this.port = port;
+ this.znode = znode;
+ }
+
+ private EmbeddedHBase(int port) {
+ this(port, DEFAULT_ZNODE);
+ }
+
+ public static EmbeddedHBase getInstance(Configuration conf) {
+ if (hbase == null) {
+ synchronized (EmbeddedHBase.class) {
+ if (hbase == null) {
+ hbase = new EmbeddedHBase();
+ hbase.start(conf);
+ }
+ }
+ }
+ return hbase;
+ }
+
+ public static EmbeddedHBase getInstance() {
+ return getInstance(null);
+ }
+
+ public EmbeddedHBase() {
+ this(DEFAULT_PORT, DEFAULT_ZNODE);
+ }
+
+ public void start() {
+ start(null);
+ }
+
+ public void start(Configuration confMap) {
+ try {
+ util = new HBaseTestingUtility();
+ Configuration conf = util.getConfiguration();
+ if (confMap != null) {
+ conf.addResource(confMap);
+ }
+ conf.setInt("test.hbase.zookeeper.property.clientPort", port);
+ conf.set("zookeeper.znode.parent", znode);
+ conf.setInt("hbase.zookeeper.property.maxClientCnxns", 200);
+
+ conf.setInt("hbase.master.info.port", -1);//avoid port clobbering
+ // start mini hbase cluster
+ hbaseCluster = util.startMiniCluster();
+ Configuration config = hbaseCluster.getConf();
+
+ config.set("zookeeper.session.timeout", "120000");
+ config.set("hbase.zookeeper.property.tickTime", "6000");
+ config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
+ config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
+ config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ shutdown();
+ }
+ });
+ } catch (Throwable t) {
+ LOG.error("Got an exception: ", t);
+ }
+ }
+
+ public void shutdown() {
+ try {
+ util.shutdownMiniCluster();
+ } catch (Throwable t) {
+ LOG.info("Got an exception, " + t, t.getCause());
+ try {
+ util.shutdownMiniCluster();
+ } catch (Throwable t1) {
+ // ignored
+ }
+ }
+ }
+
+ public void createTable(String tableName, String cf) {
+ try {
+ util.createTable(tableName, cf);
+ } catch (IOException ex) {
+ LOG.warn("Create table failed, probably table already existed, table name: " + tableName);
+ }
+ }
+
+ public void deleteTable(String tableName) {
+ try {
+ util.deleteTable(tableName);
+ } catch (Exception ex) {
+ LOG.warn("Delete table failed, probably table not existed, table name: " + tableName);
+ }
+ }
+
+ public static void main(String[] args) {
+ EmbeddedHBase hbase = new EmbeddedHBase(12181);
+ hbase.start();
+ for (String table : new Tables().getTables()) {
+ hbase.createTable(table, "f");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
deleted file mode 100644
index 0aeac2c..0000000
--- a/eagle-core/eagle-embed/eagle-embed-hbase/src/main/java/org/apache/eagle/service/hbase/EmbeddedHbase.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.service.hbase;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.MiniHBaseCluster;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-@Deprecated
-public class EmbeddedHbase {
- private HBaseTestingUtility util;
- private MiniHBaseCluster hbaseCluster;
- private static EmbeddedHbase hbase;
- private int port;
- private String znode;
- private static int DEFAULT_PORT = 2181;
- private static String DEFAULT_ZNODE = "/hbase-unsecure";
- private static final Logger LOG = LoggerFactory.getLogger(EmbeddedHbase.class);
-
- private EmbeddedHbase(int port, String znode) {
- this.port = port;
- this.znode = znode;
- }
-
- private EmbeddedHbase(int port) {
- this(port, DEFAULT_ZNODE);
- }
-
- public static EmbeddedHbase getInstance(Configuration conf) {
- if (hbase == null) {
- synchronized (EmbeddedHbase.class) {
- if (hbase == null) {
- hbase = new EmbeddedHbase();
- hbase.start(conf);
- }
- }
- }
- return hbase;
- }
-
- public static EmbeddedHbase getInstance() {
- return getInstance(null);
- }
-
- public EmbeddedHbase() {
- this(DEFAULT_PORT, DEFAULT_ZNODE);
- }
-
- public void start() {
- start(null);
- }
-
- public void start(Configuration confMap) {
- try {
- util = new HBaseTestingUtility();
- Configuration conf = util.getConfiguration();
- if (confMap != null) {
- conf.addResource(confMap);
- }
- conf.setInt("test.hbase.zookeeper.property.clientPort", port);
- conf.set("zookeeper.znode.parent", znode);
- conf.setInt("hbase.zookeeper.property.maxClientCnxns", 200);
-
- conf.setInt("hbase.master.info.port", -1);//avoid port clobbering
- // start mini hbase cluster
- hbaseCluster = util.startMiniCluster();
- Configuration config = hbaseCluster.getConf();
-
- config.set("zookeeper.session.timeout", "120000");
- config.set("hbase.zookeeper.property.tickTime", "6000");
- config.set(HConstants.HBASE_CLIENT_PAUSE, "3000");
- config.set(HConstants.HBASE_CLIENT_RETRIES_NUMBER, "1");
- config.set(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, "60000");
-
- Runtime.getRuntime().addShutdownHook(new Thread() {
- @Override
- public void run() {
- shutdown();
- }
- });
- } catch (Throwable t) {
- LOG.error("Got an exception: ", t);
- }
- }
-
- public void shutdown() {
- try {
- util.shutdownMiniCluster();
- } catch (Throwable t) {
- LOG.info("Got an exception, " + t, t.getCause());
- try {
- util.shutdownMiniCluster();
- } catch (Throwable t1) {
- // ignored
- }
- }
- }
-
- public void createTable(String tableName, String cf) {
- try {
- util.createTable(tableName, cf);
- } catch (IOException ex) {
- LOG.warn("Create table failed, probably table already existed, table name: " + tableName);
- }
- }
-
- public void deleteTable(String tableName) {
- try {
- util.deleteTable(tableName);
- } catch (Exception ex) {
- LOG.warn("Delete table failed, probably table not existed, table name: " + tableName);
- }
- }
-
- public static void main(String[] args) {
- EmbeddedHbase hbase = new EmbeddedHbase(12181);
- hbase.start();
- for (String table : new Tables().getTables()) {
- hbase.createTable(table, "f");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java
new file mode 100644
index 0000000..c0f4e4c
--- /dev/null
+++ b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHBaseTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.hbase;
+
+import org.junit.Test;
+
+import java.io.IOException;
+
+public class EmbeddedHBaseTest extends TestHBaseBase {
+ @Test
+ public void testHBaseCreateTable() throws IOException {
+ hbase.createTable("test_hbase_table","f");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java b/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java
deleted file mode 100644
index ee9d32d..0000000
--- a/eagle-core/eagle-embed/eagle-embed-hbase/src/test/java/org/apache/eagle/service/hbase/EmbeddedHbaseTest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.service.hbase;
-
-import org.junit.Test;
-
-import java.io.IOException;
-
-public class EmbeddedHbaseTest extends TestHBaseBase {
- @Test
- public void testHBaseCreateTable() throws IOException {
- hbase.createTable("test_hbase_table","f");
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java b/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java
index 6e7b5cf..d219516 100644
--- a/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java
+++ b/eagle-core/eagle-query/eagle-audit-base/src/test/java/org/apache/eagle/audit/listener/TestAuditSupport.java
@@ -54,7 +54,7 @@ public class TestAuditSupport {
}
@Test
- public void addAuditProxyHbase() throws Exception {
+ public void addAuditProxyHBase() throws Exception {
AuditSupport auditSupport = new AuditSupport(this);
Field mapField = AuditSupport.class.getDeclaredField("map");
mapField.setAccessible(true);
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java
index adfd2e2..77d7c3d 100644
--- a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/ClientTestBase.java
@@ -16,16 +16,16 @@
*/
package org.apache.eagle.service.client;
-import org.apache.eagle.service.hbase.EmbeddedHbase;
+import org.apache.eagle.service.hbase.EmbeddedHBase;
public class ClientTestBase {
//protected static EmbeddedServer server;
- protected static EmbeddedHbase hbase;
+ protected static EmbeddedHBase hbase;
//@BeforeClass
public static void startup() throws Exception {
- //hbase = EmbeddedHbase.getInstance();
+ //hbase = EmbeddedHBase.getInstance();
//String webappDirLocation = "../../../eagle-webservice/target/eagle-service";
//server = EmbeddedServer.getInstance(webappDirLocation);
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java b/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java
new file mode 100644
index 0000000..75b55ba
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHBaseWritePerformance.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.log.entity;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.eagle.common.config.EagleConfigFactory;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Put;
+import org.junit.Test;
+
+public class TestHBaseWritePerformance {
+
+ public static void main(String[] args) throws IOException {
+
+ HTableInterface tbl = EagleConfigFactory.load().getHTable("unittest");
+
+ int putSize = 1000;
+ List<Put> list = new ArrayList<Put>(putSize);
+ for (int i = 0; i < putSize; ++i) {
+ byte[] v = Integer.toString(i).getBytes();
+ Put p = new Put(v);
+ p.add("f".getBytes(), "a".getBytes(), 100, v);
+ list.add(p);
+ }
+
+ // Case 1
+ System.out.println("Case 1: autoflush = true, individual put");
+ tbl.setAutoFlush(true);
+ long startTime = System.currentTimeMillis();
+ for (int i = 0; i < 1; ++i) {
+ for (Put p : list) {
+ tbl.put(p);
+ }
+ tbl.flushCommits();
+ }
+ long endTime = System.currentTimeMillis();
+ System.out.println("Case 1: " + (endTime - startTime) + " ms");
+
+
+ // Case 2
+ System.out.println("Case 2: autoflush = true, multi-put");
+ tbl.setAutoFlush(true);
+ startTime = System.currentTimeMillis();
+ for (int i = 0; i < 1; ++i) {
+ tbl.put(list);
+ tbl.flushCommits();
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println("Case 2: " + (endTime - startTime) + " ms");
+
+
+ // Case 3
+ System.out.println("Case 3: autoflush = false, multi-put");
+ tbl.setAutoFlush(false);
+ startTime = System.currentTimeMillis();
+ for (int i = 0; i < 1; ++i) {
+ tbl.put(list);
+ tbl.flushCommits();
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println("Case 3: " + (endTime - startTime) + " ms");
+
+
+ // Case 4
+ System.out.println("Case 4: autoflush = false, individual put");
+ tbl.setAutoFlush(true);
+ startTime = System.currentTimeMillis();
+ for (int i = 0; i < 1; ++i) {
+ for (Put p : list) {
+ tbl.put(p);
+ }
+ tbl.flushCommits();
+ }
+ endTime = System.currentTimeMillis();
+ System.out.println("Case 4: " + (endTime - startTime) + " ms");
+
+ }
+
+ @Test
+ public void test() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java b/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java
deleted file mode 100644
index 81f1dce..0000000
--- a/eagle-core/eagle-query/eagle-entity-base/src/test/java/org/apache/eagle/log/entity/TestHbaseWritePerformance.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.log.entity;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.eagle.common.config.EagleConfigFactory;
-import org.apache.hadoop.hbase.client.HTableInterface;
-import org.apache.hadoop.hbase.client.Put;
-import org.junit.Test;
-
-public class TestHbaseWritePerformance {
-
- public static void main(String[] args) throws IOException {
-
- HTableInterface tbl = EagleConfigFactory.load().getHTable("unittest");
-
- int putSize = 1000;
- List<Put> list = new ArrayList<Put>(putSize);
- for (int i = 0; i < putSize; ++i) {
- byte[] v = Integer.toString(i).getBytes();
- Put p = new Put(v);
- p.add("f".getBytes(), "a".getBytes(), 100, v);
- list.add(p);
- }
-
- // Case 1
- System.out.println("Case 1: autoflush = true, individual put");
- tbl.setAutoFlush(true);
- long startTime = System.currentTimeMillis();
- for (int i = 0; i < 1; ++i) {
- for (Put p : list) {
- tbl.put(p);
- }
- tbl.flushCommits();
- }
- long endTime = System.currentTimeMillis();
- System.out.println("Case 1: " + (endTime - startTime) + " ms");
-
-
- // Case 2
- System.out.println("Case 2: autoflush = true, multi-put");
- tbl.setAutoFlush(true);
- startTime = System.currentTimeMillis();
- for (int i = 0; i < 1; ++i) {
- tbl.put(list);
- tbl.flushCommits();
- }
- endTime = System.currentTimeMillis();
- System.out.println("Case 2: " + (endTime - startTime) + " ms");
-
-
- // Case 3
- System.out.println("Case 3: autoflush = false, multi-put");
- tbl.setAutoFlush(false);
- startTime = System.currentTimeMillis();
- for (int i = 0; i < 1; ++i) {
- tbl.put(list);
- tbl.flushCommits();
- }
- endTime = System.currentTimeMillis();
- System.out.println("Case 3: " + (endTime - startTime) + " ms");
-
-
- // Case 4
- System.out.println("Case 4: autoflush = false, individual put");
- tbl.setAutoFlush(true);
- startTime = System.currentTimeMillis();
- for (int i = 0; i < 1; ++i) {
- for (Put p : list) {
- tbl.put(p);
- }
- tbl.flushCommits();
- }
- endTime = System.currentTimeMillis();
- System.out.println("Case 4: " + (endTime - startTime) + " ms");
-
- }
-
- @Test
- public void test() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java b/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
index 6513452..6406b13 100755
--- a/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
+++ b/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
@@ -24,7 +24,7 @@ import org.apache.eagle.log.entity.SearchCondition;
import org.apache.eagle.log.entity.meta.EntityDefinition;
import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
-import org.apache.eagle.service.hbase.EmbeddedHbase;
+import org.apache.eagle.service.hbase.EmbeddedHBase;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
import org.junit.Test;
@@ -35,7 +35,7 @@ import java.util.*;
public class TestHBaseLogReader2 {
private final static Logger LOG = LoggerFactory.getLogger(TestHBaseLogReader2.class);
- private static EmbeddedHbase hbase = EmbeddedHbase.getInstance();
+ private static EmbeddedHBase hbase = EmbeddedHBase.getInstance();
@SuppressWarnings("serial")
@Test
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java
index 0e52a2e..a66c057 100644
--- a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java
+++ b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckAPIEntity.java
@@ -32,16 +32,16 @@ public class EagleServiceSelfCheckAPIEntity {
public void setEnv(String env) {
this.env = env;
}
- public String getHbaseZookeeperQuorum() {
+ public String getHBaseZookeeperQuorum() {
return hbaseZookeeperQuorum;
}
- public void setHbaseZookeeperQuorum(String hbaseZookeeperQuorum) {
+ public void setHBaseZookeeperQuorum(String hbaseZookeeperQuorum) {
this.hbaseZookeeperQuorum = hbaseZookeeperQuorum;
}
- public String getHbaseZookeeperClientPort() {
+ public String getHBaseZookeeperClientPort() {
return hbaseZookeeperClientPort;
}
- public void setHbaseZookeeperClientPort(String hbaseZookeeperClientPort) {
+ public void setHBaseZookeeperClientPort(String hbaseZookeeperClientPort) {
this.hbaseZookeeperClientPort = hbaseZookeeperClientPort;
}
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java
index 88ab613..0eed64c 100644
--- a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java
+++ b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/selfcheck/EagleServiceSelfCheckResource.java
@@ -30,8 +30,8 @@ public class EagleServiceSelfCheckResource {
@Produces(MediaType.APPLICATION_JSON)
public EagleServiceSelfCheckAPIEntity selfCheck(){
EagleServiceSelfCheckAPIEntity entity = new EagleServiceSelfCheckAPIEntity();
- entity.setHbaseZookeeperQuorum(EagleConfigFactory.load().getZKQuorum());
- entity.setHbaseZookeeperClientPort(EagleConfigFactory.load().getZKPort());
+ entity.setHBaseZookeeperQuorum(EagleConfigFactory.load().getZKQuorum());
+ entity.setHBaseZookeeperClientPort(EagleConfigFactory.load().getZKPort());
entity.setEnv(EagleConfigFactory.load().getEnv());
return entity;
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
index 8e4005c..eaa5d3c 100644
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/HBaseEntitySchemaManager.java
@@ -60,7 +60,7 @@ public class HBaseEntitySchemaManager {
LOG.debug("Auto create table disabled, skip creating table");
return;
}
- Configuration conf = EagleConfigFactory.load().getHbaseConf();
+ Configuration conf = EagleConfigFactory.load().getHBaseConf();
try {
admin = new HBaseAdmin(conf);
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java
new file mode 100644
index 0000000..a9421e4
--- /dev/null
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HBaseResourceEntity.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.security.entity;
+
+import com.google.common.base.Objects;
+
+import java.io.Serializable;
+import java.util.Set;
+
+public class HBaseResourceEntity implements Serializable {
+ private String resource;
+ private String namespace;
+ private String table;
+ private String columnFamily;
+ private String sensitiveType;
+ private Set<String> childSensitiveTypes;
+
+
+
+ public HBaseResourceEntity(String resource, String ns, String table, String cf, String sensitiveType, Set<String> childSensitiveTypes) {
+ this.resource = resource;
+ this.namespace = ns;
+ this.table = table;
+ this.columnFamily = cf;
+ this.sensitiveType = sensitiveType;
+ this.childSensitiveTypes = childSensitiveTypes;
+ }
+
+ public String getResource() {
+ return resource;
+ }
+
+ public void setResource(String resource) {
+ this.resource = resource;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+
+ public void setNamespace(String namespace) {
+ this.namespace = namespace;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public String getColumnFamily() {
+ return columnFamily;
+ }
+
+ public void setColumnFamily(String columnFamily) {
+ this.columnFamily = columnFamily;
+ }
+
+ public String getSensitiveType() {
+ return sensitiveType;
+ }
+
+ public void setSensitiveType(String sensitiveType) {
+ this.sensitiveType = sensitiveType;
+ }
+
+ public Set<String> getChildSensitiveTypes() {
+ return childSensitiveTypes;
+ }
+
+ public void setChildSensitiveTypes(Set<String> childSensitiveTypes) {
+ this.childSensitiveTypes = childSensitiveTypes;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+ final HBaseResourceEntity other = (HBaseResourceEntity) obj;
+ return Objects.equal(this.resource, other.resource)
+ && this.namespace.equals(other.namespace)
+ && this.table.equals(other.table)
+ && this.columnFamily.equals(other.columnFamily)
+ && this.sensitiveType.equals(other.sensitiveType)
+ && this.childSensitiveTypes.containsAll(other.childSensitiveTypes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java
deleted file mode 100644
index deb8364..0000000
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/entity/HbaseResourceEntity.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.eagle.security.entity;
-
-import com.google.common.base.Objects;
-
-import java.io.Serializable;
-import java.util.Set;
-
-public class HbaseResourceEntity implements Serializable {
- private String resource;
- private String namespace;
- private String table;
- private String columnFamily;
- private String sensitiveType;
- private Set<String> childSensitiveTypes;
-
-
-
- public HbaseResourceEntity(String resource, String ns, String table, String cf, String sensitiveType, Set<String> childSensitiveTypes) {
- this.resource = resource;
- this.namespace = ns;
- this.table = table;
- this.columnFamily = cf;
- this.sensitiveType = sensitiveType;
- this.childSensitiveTypes = childSensitiveTypes;
- }
-
- public String getResource() {
- return resource;
- }
-
- public void setResource(String resource) {
- this.resource = resource;
- }
-
- public String getNamespace() {
- return namespace;
- }
-
- public void setNamespace(String namespace) {
- this.namespace = namespace;
- }
-
- public String getTable() {
- return table;
- }
-
- public void setTable(String table) {
- this.table = table;
- }
-
- public String getColumnFamily() {
- return columnFamily;
- }
-
- public void setColumnFamily(String columnFamily) {
- this.columnFamily = columnFamily;
- }
-
- public String getSensitiveType() {
- return sensitiveType;
- }
-
- public void setSensitiveType(String sensitiveType) {
- this.sensitiveType = sensitiveType;
- }
-
- public Set<String> getChildSensitiveTypes() {
- return childSensitiveTypes;
- }
-
- public void setChildSensitiveTypes(Set<String> childSensitiveTypes) {
- this.childSensitiveTypes = childSensitiveTypes;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == null || getClass() != obj.getClass()) {
- return false;
- }
- final HbaseResourceEntity other = (HbaseResourceEntity) obj;
- return Objects.equal(this.resource, other.resource)
- && this.namespace.equals(other.namespace)
- && this.table.equals(other.table)
- && this.columnFamily.equals(other.columnFamily)
- && this.sensitiveType.equals(other.sensitiveType)
- && this.childSensitiveTypes.containsAll(other.childSensitiveTypes);
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
index d46ef13..eff5092 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/HBaseSensitivityEntity.java
@@ -32,11 +32,11 @@ public class HBaseSensitivityEntity {
this.site = site;
}
- public String getHbaseResource() {
+ public String getHBaseResource() {
return hbaseResource;
}
- public void setHbaseResource(String hbaseResource) {
+ public void setHBaseResource(String hbaseResource) {
this.hbaseResource = hbaseResource;
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
index 359cbe6..898f2a9 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/InMemMetadataDaoImpl.java
@@ -17,7 +17,6 @@
package org.apache.eagle.security.service;
import com.google.inject.Inject;
-import com.typesafe.config.Config;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.slf4j.Logger;
@@ -54,7 +53,7 @@ public class InMemMetadataDaoImpl implements ISecurityMetadataDAO {
@Override
public synchronized OpResult addHBaseSensitivity(Collection<HBaseSensitivityEntity> h) {
for (HBaseSensitivityEntity e : h) {
- Pair p = new ImmutablePair<>(e.getSite(), e.getHbaseResource());
+ Pair p = new ImmutablePair<>(e.getSite(), e.getHBaseResource());
hBaseSensitivityEntities.put(p, e);
}
return new OpResult();
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java
index e089bf9..07523d2 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/service/JDBCSecurityMetadataDAO.java
@@ -124,7 +124,7 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO {
try {
HBaseSensitivityEntity entity = new HBaseSensitivityEntity();
entity.setSite(rs.getString(1));
- entity.setHbaseResource(rs.getString(2));
+ entity.setHBaseResource(rs.getString(2));
entity.setSensitivityType(rs.getString(3));
return entity;
}catch(Exception ex){ throw new IllegalStateException(ex);}
@@ -138,7 +138,7 @@ public class JDBCSecurityMetadataDAO implements ISecurityMetadataDAO {
HBaseSensitivityEntity e = (HBaseSensitivityEntity)entity;
try {
statement.setString(1, e.getSite());
- statement.setString(2, e.getHbaseResource());
+ statement.setString(2, e.getHBaseResource());
statement.setString(3, e.getSensitivityType());
}catch(Exception ex){
throw new IllegalStateException(ex);
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java
index 03ae58c..18c89a3 100644
--- a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java
+++ b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/JDBCSecurityMetadataDAOTest.java
@@ -55,7 +55,7 @@ public class JDBCSecurityMetadataDAOTest {
public void testJDBCSecurityMetadataDAO(){
HBaseSensitivityEntity entity = new HBaseSensitivityEntity();
entity.setSite("test_site");
- entity.setHbaseResource("test_hbaseResource");
+ entity.setHBaseResource("test_hbaseResource");
metadataDAO.addHBaseSensitivity(Collections.singletonList(entity));
Collection<HBaseSensitivityEntity> entities = metadataDAO.listHBaseSensitivities();
Assert.assertEquals(1,entities.size());
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
index 325663b..092ee73 100644
--- a/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
+++ b/eagle-security/eagle-security-common/src/test/java/org/apache/eagle/security/crawler/audit/TestMetaDataAccessConfigRepo.java
@@ -51,7 +51,7 @@ public class TestMetaDataAccessConfigRepo {
Assert.assertTrue(hbaseConfig.getString("hbase.zookeeper.property.clientPort").equals("2181"));
}
- String appConfigStr = "classification.hbase.zookeeper.property.clientPort=2181\nclassification.hbase.zookeeper.quorum=sandbox.hortonworks.com\n\napp.envContextConfig.env=storm\napp.envContextConfig.mode=cluster\napp.dataSourceConfig.topic=sandbox_hbase_security_log\napp.dataSourceConfig.zkConnection=127.0.0.1:2181\napp.dataSourceConfig.zkConnectionTimeoutMS=15000\napp.dataSourceConfig.brokerZkPath=/brokers\napp.dataSourceConfig.fetchSize=1048586\napp.dataSourceConfig.transactionZKServers=127.0.0.1\napp.dataSourceConfig.transactionZKPort=2181\napp.dataSourceConfig.transactionZKRoot=/consumers\napp.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer\napp.dataSourceConfig.transactionStateUpdateMS=2000\napp.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.parse.HbaseAuditLogKafkaDeserializer\napp.eagleProps.site=sandbox\napp.eagleProps.application=hbaseSecurityLog\napp.eagleProps.dataJoinPollIntervalSec=30\napp.eagleProps.mailHost=mailHost.com\napp.ea
gleProps.mailSmtpPort=25\napp.eagleProps.mailDebug=true\napp.eagleProps.eagleService.host=localhost\napp.eagleProps.eagleService.port=9099\napp.eagleProps.eagleService.username=admin\napp.eagleProps.eagleService.password=secret";
+ String appConfigStr = "classification.hbase.zookeeper.property.clientPort=2181\nclassification.hbase.zookeeper.quorum=sandbox.hortonworks.com\n\napp.envContextConfig.env=storm\napp.envContextConfig.mode=cluster\napp.dataSourceConfig.topic=sandbox_hbase_security_log\napp.dataSourceConfig.zkConnection=127.0.0.1:2181\napp.dataSourceConfig.zkConnectionTimeoutMS=15000\napp.dataSourceConfig.brokerZkPath=/brokers\napp.dataSourceConfig.fetchSize=1048586\napp.dataSourceConfig.transactionZKServers=127.0.0.1\napp.dataSourceConfig.transactionZKPort=2181\napp.dataSourceConfig.transactionZKRoot=/consumers\napp.dataSourceConfig.consumerGroupId=eagle.hbasesecurity.consumer\napp.dataSourceConfig.transactionStateUpdateMS=2000\napp.dataSourceConfig.deserializerClass=org.apache.eagle.security.hbase.parse.HBaseAuditLogKafkaDeserializer\napp.eagleProps.site=sandbox\napp.eagleProps.application=hbaseSecurityLog\napp.eagleProps.dataJoinPollIntervalSec=30\napp.eagleProps.mailHost=mailHost.com\napp.ea
gleProps.mailSmtpPort=25\napp.eagleProps.mailDebug=true\napp.eagleProps.eagleService.host=localhost\napp.eagleProps.eagleService.port=9099\napp.eagleProps.eagleService.username=admin\napp.eagleProps.eagleService.password=secret";
config = ConfigFactory.parseString(appConfigStr, options);
Config appConfig = null;
if(config.hasPath(EagleConfigConstants.APP_CONFIG)) {
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/README.md
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/README.md b/eagle-security/eagle-security-hbase-auditlog/README.md
index b0a6f90..d70c417 100644
--- a/eagle-security/eagle-security-hbase-auditlog/README.md
+++ b/eagle-security/eagle-security-hbase-auditlog/README.md
@@ -68,7 +68,7 @@ download logstash 2.3.x
#### 2.3 run eagle webservice
find eagle-webservice project, run it
#### 2.4 run eagle-security-hdfs-authlog topology
-find org.apache.eagle.security.hbase.HbaseAuditLogMonitoringMain, run it
+find org.apache.eagle.security.hbase.HBaseAuditLogMonitoringMain, run it
### Step 3: alert engine: consume parsed log
#### 3.1 run alert engine
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java
index 11a22e5..2008f0b 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogApplication.java
@@ -55,7 +55,7 @@ public class HBaseAuditLogApplication extends StormApplication {
BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));
- HbaseResourceSensitivityDataJoinBolt joinBolt = new HbaseResourceSensitivityDataJoinBolt(config);
+ HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config);
BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java
new file mode 100644
index 0000000..c09073a
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogObject.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.hbase;
+
+
+public class HBaseAuditLogObject {
+ public long timestamp;
+ public String user = "";
+ public String scope = "";
+ public String action = "";
+ public String host = "";
+ public String request = "";
+ public String status = "";
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java
new file mode 100644
index 0000000..b033a82
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParser.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import java.io.Serializable;
+import java.util.LinkedList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.security.util.LogParseUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class HBaseAuditLogParser implements Serializable {
+ private final static Logger LOG = LoggerFactory.getLogger(HBaseAuditLogParser.class);
+
+ private final static int LOGDATE_INDEX = 1;
+ private final static int LOGLEVEL_INDEX = 2;
+ private final static int LOGATTRS_INDEX = 3;
+ private final static String ALLOWED = "allowed";
+ private final static String DENIED = "denied";
+ private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*)");
+ private final static Pattern loggerContextPattern = Pattern.compile("\\w+:\\s*\\((.*)\\s*\\)");
+ private final static Pattern allowedPattern = Pattern.compile(ALLOWED);
+
+
+ public HBaseAuditLogObject parse(String logLine) {
+ if(logLine == null || logLine.isEmpty()) return null;
+
+ HBaseAuditLogObject ret = new HBaseAuditLogObject();
+ String timestamp = "";
+ String user = "";
+ String scope = "";
+ String action = "";
+ String ip = "";
+ String request = "";
+ String family = "";
+ String context = "";
+
+ Matcher loggerMatcher = loggerPattern.matcher(logLine);
+ if(loggerMatcher.find()) {
+ try {
+ timestamp = loggerMatcher.group(LOGDATE_INDEX);
+ String [] attrs = loggerMatcher.group(LOGATTRS_INDEX).split(";");
+ ret.status = allowedPattern.matcher(attrs[0]).find() ? ALLOWED : DENIED;
+ try {
+ ip = attrs[2].split(":")[1].trim();
+ } catch (Exception e) {
+ ip = "";
+ }
+ try {
+ request = attrs[3].split(":")[1].trim();
+ } catch (Exception e) {
+ request = "";
+ }
+ try {
+ context = attrs[4].trim();
+ } catch (Exception e) {
+ context = "";
+ }
+
+ Matcher contextMatcher = loggerContextPattern.matcher(context.replaceAll("\\s+",""));
+ if(contextMatcher.find()) {
+ boolean paramsOpen = false;
+
+ List<String> kvs = new LinkedList<String>(Arrays.asList(contextMatcher.group(1).split(",")));
+
+ while (!kvs.isEmpty()) {
+ String kv = kvs.get(0);
+
+ if (kv.split("=").length < 2) {
+ kvs.remove(0);
+ continue;
+ }
+
+ String k = kv.split("=")[0];
+ String v = kv.split("=")[1];
+
+ if (paramsOpen && kv.substring(kv.length() - 1).equals("]")) {
+ paramsOpen = false;
+ v = v.substring(0, v.length() - 1);
+ }
+
+ switch (k) {
+ case "user":
+ user = v;
+ break;
+ case "scope":
+ scope = v;
+ break;
+ case "family":
+ family = v;
+ break;
+ case "action":
+ action = v;
+ break;
+ case "params":
+ kvs.add(v.substring(1) + "=" + kv.split("=")[2]);
+ paramsOpen = true;
+ break;
+ default: break;
+ }
+
+ kvs.remove(0);
+ }
+ }
+
+ if(StringUtils.isNotEmpty(family)) {
+ if(!scope.contains(":")) scope = "default:" + scope;
+ scope = String.format("%s:%s", scope, family);
+ }
+ if(StringUtils.isNotEmpty(ip)) {
+ ret.host = ip.substring(1);
+ }
+ ret.timestamp = DateTimeUtil.humanDateToMilliseconds(timestamp);
+ ret.scope = scope;
+ ret.action = action;
+ ret.user = LogParseUtil.parseUserFromUGI(user);
+ ret.request = request;
+ return ret;
+ } catch(Exception e) {
+ LOG.error("Got exception when parsing audit log:" + logLine + ", exception:" + e.getMessage(), e);
+ }
+ }
+ return null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
index ffed0ef..36fe33a 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseAuditLogParserBolt.java
@@ -34,7 +34,7 @@ import java.util.*;
public class HBaseAuditLogParserBolt extends BaseRichBolt {
private static Logger LOG = LoggerFactory.getLogger(HBaseAuditLogParserBolt.class);
private OutputCollector collector;
- private static final HbaseAuditLogParser parser = new HbaseAuditLogParser();
+ private static final HBaseAuditLogParser parser = new HBaseAuditLogParser();
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
@@ -45,7 +45,7 @@ public class HBaseAuditLogParserBolt extends BaseRichBolt {
public void execute(Tuple input) {
String logLine = input.getString(0);
try {
- HbaseAuditLogObject entity = parser.parse(logLine);
+ HBaseAuditLogObject entity = parser.parse(logLine);
Map<String, Object> map = new TreeMap<>();
map.put("action", entity.action);
map.put("host", entity.host);
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java
new file mode 100644
index 0000000..8b5461e
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseResourceSensitivityDataJoinBolt.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.security.hbase;
+
+import backtype.storm.topology.OutputFieldsDeclarer;
+import backtype.storm.tuple.Fields;
+import backtype.storm.tuple.Tuple;
+import com.typesafe.config.Config;
+import org.apache.eagle.security.service.HBaseSensitivityEntity;
+import org.apache.eagle.security.enrich.AbstractDataEnrichBolt;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.regex.Pattern;
+
+public class HBaseResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<HBaseSensitivityEntity, String> {
+ private final static Logger LOG = LoggerFactory.getLogger(HBaseResourceSensitivityDataJoinBolt.class);
+
+ public HBaseResourceSensitivityDataJoinBolt(Config config) {
+ super(config, new HBaseSensitivityDataEnrichLCM(config));
+ }
+
+ @Override
+ public void executeWithEnrich(Tuple input, Map<String, HBaseSensitivityEntity> map) {
+ try {
+ Map<String, Object> event = (Map<String, Object>) input.getValue(0);
+ LOG.info(">>>> event: " + event + " >>>> map: " + map);
+
+ String resource = (String) event.get("scope");
+
+ HBaseSensitivityEntity sensitivityEntity = null;
+
+ if (map != null && resource != "") {
+ for (String key : map.keySet()) {
+ Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
+ if (pattern.matcher(resource).find()) {
+ sensitivityEntity = map.get(key);
+ break;
+ }
+ }
+ }
+ Map<String, Object> newEvent = new TreeMap<String, Object>(event);
+ newEvent.put("sensitivityType", sensitivityEntity == null
+ ? "NA" : sensitivityEntity.getSensitivityType());
+ newEvent.put("scope", resource);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("After hbase resource sensitivity lookup: " + newEvent);
+ }
+ LOG.info("After hbase resource sensitivity lookup: " + newEvent);
+ // push to Kafka sink
+ collector.emit(Arrays.asList(newEvent.get("user"), newEvent));
+ } catch (Exception ex) {
+ LOG.error("error joining data, ignore it", ex);
+ } finally {
+ collector.ack(input);
+ }
+ }
+
+ @Override
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ declarer.declare(new Fields("user", "message"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
index 2cb1dec..9745706 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HBaseSensitivityDataEnrichLCM.java
@@ -38,6 +38,6 @@ public class HBaseSensitivityDataEnrichLCM extends AbstractDataEnrichLCM<HBaseSe
@Override
public String getCacheKey(HBaseSensitivityEntity entity) {
- return entity.getHbaseResource();
+ return entity.getHBaseResource();
}
}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java
deleted file mode 100644
index ed9367d..0000000
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogObject.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.hbase;
-
-
-public class HbaseAuditLogObject {
- public long timestamp;
- public String user = "";
- public String scope = "";
- public String action = "";
- public String host = "";
- public String request = "";
- public String status = "";
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java
deleted file mode 100644
index bff9c0b..0000000
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseAuditLogParser.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-package org.apache.eagle.security.hbase;
-
-import java.io.Serializable;
-import java.util.LinkedList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.eagle.common.DateTimeUtil;
-import org.apache.eagle.security.util.LogParseUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-public class HbaseAuditLogParser implements Serializable {
- private final static Logger LOG = LoggerFactory.getLogger(HbaseAuditLogParser.class);
-
- private final static int LOGDATE_INDEX = 1;
- private final static int LOGLEVEL_INDEX = 2;
- private final static int LOGATTRS_INDEX = 3;
- private final static String ALLOWED = "allowed";
- private final static String DENIED = "denied";
- private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*)");
- private final static Pattern loggerContextPattern = Pattern.compile("\\w+:\\s*\\((.*)\\s*\\)");
- private final static Pattern allowedPattern = Pattern.compile(ALLOWED);
-
-
- public HbaseAuditLogObject parse(String logLine) {
- if(logLine == null || logLine.isEmpty()) return null;
-
- HbaseAuditLogObject ret = new HbaseAuditLogObject();
- String timestamp = "";
- String user = "";
- String scope = "";
- String action = "";
- String ip = "";
- String request = "";
- String family = "";
- String context = "";
-
- Matcher loggerMatcher = loggerPattern.matcher(logLine);
- if(loggerMatcher.find()) {
- try {
- timestamp = loggerMatcher.group(LOGDATE_INDEX);
- String [] attrs = loggerMatcher.group(LOGATTRS_INDEX).split(";");
- ret.status = allowedPattern.matcher(attrs[0]).find() ? ALLOWED : DENIED;
- try {
- ip = attrs[2].split(":")[1].trim();
- } catch (Exception e) {
- ip = "";
- }
- try {
- request = attrs[3].split(":")[1].trim();
- } catch (Exception e) {
- request = "";
- }
- try {
- context = attrs[4].trim();
- } catch (Exception e) {
- context = "";
- }
-
- Matcher contextMatcher = loggerContextPattern.matcher(context.replaceAll("\\s+",""));
- if(contextMatcher.find()) {
- boolean paramsOpen = false;
-
- List<String> kvs = new LinkedList<String>(Arrays.asList(contextMatcher.group(1).split(",")));
-
- while (!kvs.isEmpty()) {
- String kv = kvs.get(0);
-
- if (kv.split("=").length < 2) {
- kvs.remove(0);
- continue;
- }
-
- String k = kv.split("=")[0];
- String v = kv.split("=")[1];
-
- if (paramsOpen && kv.substring(kv.length() - 1).equals("]")) {
- paramsOpen = false;
- v = v.substring(0, v.length() - 1);
- }
-
- switch (k) {
- case "user":
- user = v;
- break;
- case "scope":
- scope = v;
- break;
- case "family":
- family = v;
- break;
- case "action":
- action = v;
- break;
- case "params":
- kvs.add(v.substring(1) + "=" + kv.split("=")[2]);
- paramsOpen = true;
- break;
- default: break;
- }
-
- kvs.remove(0);
- }
- }
-
- if(StringUtils.isNotEmpty(family)) {
- if(!scope.contains(":")) scope = "default:" + scope;
- scope = String.format("%s:%s", scope, family);
- }
- if(StringUtils.isNotEmpty(ip)) {
- ret.host = ip.substring(1);
- }
- ret.timestamp = DateTimeUtil.humanDateToMilliseconds(timestamp);
- ret.scope = scope;
- ret.action = action;
- ret.user = LogParseUtil.parseUserFromUGI(user);
- ret.request = request;
- return ret;
- } catch(Exception e) {
- LOG.error("Got exception when parsing audit log:" + logLine + ", exception:" + e.getMessage(), e);
- }
- }
- return null;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java b/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
deleted file mode 100644
index a1545c2..0000000
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/java/org/apache/eagle/security/hbase/HbaseResourceSensitivityDataJoinBolt.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.eagle.security.hbase;
-
-import backtype.storm.topology.OutputFieldsDeclarer;
-import backtype.storm.tuple.Fields;
-import backtype.storm.tuple.Tuple;
-import com.typesafe.config.Config;
-import org.apache.eagle.security.service.HBaseSensitivityEntity;
-import org.apache.eagle.security.enrich.AbstractDataEnrichBolt;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.regex.Pattern;
-
-public class HbaseResourceSensitivityDataJoinBolt extends AbstractDataEnrichBolt<HBaseSensitivityEntity, String> {
- private final static Logger LOG = LoggerFactory.getLogger(HbaseResourceSensitivityDataJoinBolt.class);
-
- public HbaseResourceSensitivityDataJoinBolt(Config config) {
- super(config, new HBaseSensitivityDataEnrichLCM(config));
- }
-
- @Override
- public void executeWithEnrich(Tuple input, Map<String, HBaseSensitivityEntity> map) {
- try {
- Map<String, Object> event = (Map<String, Object>) input.getValue(0);
- LOG.info(">>>> event: " + event + " >>>> map: " + map);
-
- String resource = (String) event.get("scope");
-
- HBaseSensitivityEntity sensitivityEntity = null;
-
- if (map != null && resource != "") {
- for (String key : map.keySet()) {
- Pattern pattern = Pattern.compile(key, Pattern.CASE_INSENSITIVE);
- if (pattern.matcher(resource).find()) {
- sensitivityEntity = map.get(key);
- break;
- }
- }
- }
- Map<String, Object> newEvent = new TreeMap<String, Object>(event);
- newEvent.put("sensitivityType", sensitivityEntity == null
- ? "NA" : sensitivityEntity.getSensitivityType());
- newEvent.put("scope", resource);
- if (LOG.isDebugEnabled()) {
- LOG.debug("After hbase resource sensitivity lookup: " + newEvent);
- }
- LOG.info("After hbase resource sensitivity lookup: " + newEvent);
- // push to Kafka sink
- collector.emit(Arrays.asList(newEvent.get("user"), newEvent));
- } catch (Exception ex) {
- LOG.error("error joining data, ignore it", ex);
- } finally {
- collector.ack(input);
- }
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- declarer.declare(new Fields("user", "message"));
- }
-}
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
index 1fbf863..16651aa 100644
--- a/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
+++ b/eagle-security/eagle-security-hbase-auditlog/src/main/resources/META-INF/providers/org.apache.eagle.security.hbase.HBaseAuditLogAppProvider.xml
@@ -18,7 +18,7 @@
<application>
<type>HBASE_AUDIT_LOG_MONITOR</type>
- <name>Hbase Audit Log Monitor</name>
+ <name>HBase Audit Log Monitor</name>
<appClass>org.apache.eagle.security.hbase.HBaseAuditLogApplication</appClass>
<configuration>
<property>
@@ -150,7 +150,7 @@
<streams>
<stream>
<streamId>hbase_audit_log_stream</streamId>
- <description>Hbase Audit Log Stream</description>
+ <description>HBase Audit Log Stream</description>
<validate>true</validate>
<timeseries>true</timeseries>
<columns>
http://git-wip-us.apache.org/repos/asf/eagle/blob/cc5d23ba/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java
new file mode 100644
index 0000000..945ea50
--- /dev/null
+++ b/eagle-security/eagle-security-hbase-auditlog/src/test/java/org/apache/eagle/security/hbase/TestHBaseAuditLogParser.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.eagle.security.hbase;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TestHBaseAuditLogParser {
+ HBaseAuditLogParser parser = new HBaseAuditLogParser();
+
+ @Test
+ public void test() throws Exception {
+ String log = "2015-08-11 13:31:03,729 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Table permission granted; remote address: /127.0.0.1; request: get; context: (user=eagle,scope=hbase:namespace,family=info, action=READ)";
+ HBaseAuditLogObject obj = parser.parse(log);
+ Assert.assertEquals(obj.action, "READ");
+ Assert.assertEquals(obj.host, "127.0.0.1");
+ Assert.assertEquals(obj.scope, "hbase:namespace:info");
+ }
+
+ @Test
+ public void test2() throws Exception {
+ String log = "2015-08-04 12:29:03,073 TRACE SecurityLogger.org.apache.hadoop.hbase.security.access.AccessController: Access allowed for user eagle; reason: Global check allowed; remote address: ; request: preOpen; context: (user=eagle, scope=GLOBAL, family=, action=ADMIN)";
+ HBaseAuditLogObject obj = parser.parse(log);
+ Assert.assertEquals(obj.action, "ADMIN");
+ Assert.assertEquals(obj.host, "");
+ Assert.assertEquals(obj.scope, "GLOBAL");
+ }
+}
[13/17] eagle git commit: Add license on service.sdl
Posted by ja...@apache.org.
Add license on service.sdl
Add license on service.sdl, avoid using exclude.
Author: chitin <ch...@gmail.com>
Closes #924 from chitin/master3.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/84d40ae6
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/84d40ae6
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/84d40ae6
Branch: refs/heads/branch-0.5
Commit: 84d40ae6f7eb50214de231b4f9744a034cff1a4a
Parents: e3afe44
Author: chitin <ch...@gmail.com>
Authored: Wed Apr 19 17:01:21 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Wed Apr 19 17:01:21 2017 +0800
----------------------------------------------------------------------
eagle-external/eagle-cdh/src/descriptor/service.sdl | 15 +++++++++++++++
pom.xml | 1 -
2 files changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/84d40ae6/eagle-external/eagle-cdh/src/descriptor/service.sdl
----------------------------------------------------------------------
diff --git a/eagle-external/eagle-cdh/src/descriptor/service.sdl b/eagle-external/eagle-cdh/src/descriptor/service.sdl
index 786d864..ffe6a08 100644
--- a/eagle-external/eagle-cdh/src/descriptor/service.sdl
+++ b/eagle-external/eagle-cdh/src/descriptor/service.sdl
@@ -1,3 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
{
"name": "EAGLE",
"label": "Eagle",
http://git-wip-us.apache.org/repos/asf/eagle/blob/84d40ae6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f463221..5115602 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1323,7 +1323,6 @@
<!-- Java Object serial file -->
<exclude>**/test/resources/appinfobyteint</exclude>
<exclude>**/test/resources/appinfobytelong</exclude>
- <exclude>*.sdl</exclude>
</excludes>
</configuration>
</execution>
[12/17] eagle git commit: [EAGLE-1009] Fix `return` inside `finally`
block may result in losing exception
Posted by ja...@apache.org.
[EAGLE-1009] Fix `return` inside `finally` block may result in losing exception
`return` inside `finally` block will result in losing exception:
* If the finally block completes abruptly for reason S, then the try statement completes abruptly for reason S (and the throw of value V is discarded and forgotten).
* If the finally block completes abruptly for reason S, then the try statement completes abruptly for reason S (and reason R is discarded).
reference:
http://docs.oracle.com/javase/specs/jls/se8/html/jls-14.html#jls-14.20.2
https://issues.apache.org/jira/secure/attachment/12863778/FinallyTest.java
(https://issues.apache.org/jira/browse/EAGLE-1009)
Author: asdf2014 <15...@qq.com>
Closes #920 from asdf2014/return_inside_finally.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/e3afe444
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/e3afe444
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/e3afe444
Branch: refs/heads/branch-0.5
Commit: e3afe444f59d3117fbe9d778120a47c411e2aa99
Parents: b4d0711
Author: asdf2014 <15...@qq.com>
Authored: Wed Apr 19 17:00:32 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Wed Apr 19 17:00:32 2017 +0800
----------------------------------------------------------------------
.../apache/eagle/jpm/analyzer/util/Utils.java | 53 +++++++++++---------
.../security/hdfs/MAPRFSAuditLogParser.java | 20 +++-----
2 files changed, 36 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/e3afe444/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java
index 9c1a2c7..42a5cac 100644
--- a/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java
+++ b/eagle-jpm/eagle-jpm-analyzer/src/main/java/org/apache/eagle/jpm/analyzer/util/Utils.java
@@ -26,7 +26,6 @@ import org.apache.eagle.jpm.util.resourcefetch.connection.InputStreamUtils;
import org.codehaus.jackson.JsonParser;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.type.TypeReference;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -46,55 +45,59 @@ public class Utils {
public static List<JobMetaEntity> getJobMeta(Config config, String siteId, String jobDefId) {
List<JobMetaEntity> result = new ArrayList<>();
String url = "http://"
- + config.getString(Constants.HOST_PATH)
- + ":"
- + config.getInt(Constants.PORT_PATH)
- + config.getString(Constants.CONTEXT_PATH)
- + Constants.ANALYZER_PATH
- + Constants.JOB_META_ROOT_PATH
- + "/"
- + siteId
- + "/"
- + URLEncoder.encode(jobDefId);
+ + config.getString(Constants.HOST_PATH)
+ + ":"
+ + config.getInt(Constants.PORT_PATH)
+ + config.getString(Constants.CONTEXT_PATH)
+ + Constants.ANALYZER_PATH
+ + Constants.JOB_META_ROOT_PATH
+ + "/"
+ + siteId
+ + "/"
+ + URLEncoder.encode(jobDefId);
InputStream is = null;
try {
is = InputStreamUtils.getInputStream(url, null, org.apache.eagle.jpm.util.Constants.CompressionType.NONE);
LOG.info("get job meta from {}", url);
- result = ((RESTResponse<List<JobMetaEntity>>)OBJ_MAPPER.readValue(is, new TypeReference<RESTResponse<List<JobMetaEntity>>>(){})).getData();
+ result = ((RESTResponse<List<JobMetaEntity>>) OBJ_MAPPER.readValue(is,
+ new TypeReference<RESTResponse<List<JobMetaEntity>>>() {
+ })).getData();
} catch (Exception e) {
LOG.warn("failed to get job meta from {}", url, e);
} finally {
org.apache.eagle.jpm.util.Utils.closeInputStream(is);
- return result;
}
+ return result;
}
public static List<UserEmailEntity> getUserMail(Config config, String siteId, String userId) {
List<UserEmailEntity> result = new ArrayList<>();
String url = "http://"
- + config.getString(Constants.HOST_PATH)
- + ":"
- + config.getInt(Constants.PORT_PATH)
- + config.getString(Constants.CONTEXT_PATH)
- + Constants.ANALYZER_PATH
- + Constants.USER_META_ROOT_PATH
- + "/"
- + siteId
- + "/"
- + URLEncoder.encode(userId);
+ + config.getString(Constants.HOST_PATH)
+ + ":"
+ + config.getInt(Constants.PORT_PATH)
+ + config.getString(Constants.CONTEXT_PATH)
+ + Constants.ANALYZER_PATH
+ + Constants.USER_META_ROOT_PATH
+ + "/"
+ + siteId
+ + "/"
+ + URLEncoder.encode(userId);
InputStream is = null;
try {
is = InputStreamUtils.getInputStream(url, null, org.apache.eagle.jpm.util.Constants.CompressionType.NONE);
LOG.info("get user meta from {}", url);
- result = ((RESTResponse<List<UserEmailEntity>>)OBJ_MAPPER.readValue(is, new TypeReference<RESTResponse<List<UserEmailEntity>>>(){})).getData();
+ result = ((RESTResponse<List<UserEmailEntity>>) OBJ_MAPPER.readValue(is,
+ new TypeReference<RESTResponse<List<UserEmailEntity>>>() {
+ })).getData();
} catch (Exception e) {
LOG.warn("failed to get user meta from {}", url, e);
} finally {
org.apache.eagle.jpm.util.Utils.closeInputStream(is);
- return result;
}
+ return result;
}
public static <K, V extends Comparable<? super V>> List<Map.Entry<K, V>> sortByValue(Map<K, V> map) {
http://git-wip-us.apache.org/repos/asf/eagle/blob/e3afe444/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/MAPRFSAuditLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/MAPRFSAuditLogParser.java b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/MAPRFSAuditLogParser.java
index 1dd2c78..30804a3 100644
--- a/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/MAPRFSAuditLogParser.java
+++ b/eagle-security/eagle-security-common/src/main/java/org/apache/eagle/security/hdfs/MAPRFSAuditLogParser.java
@@ -22,19 +22,16 @@ import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.text.ParseException;
-
-
public final class MAPRFSAuditLogParser {
private final static Logger LOG = LoggerFactory.getLogger(MAPRFSAuditLogParser.class);
- public MAPRFSAuditLogParser(){
+ public MAPRFSAuditLogParser() {
}
public MAPRFSAuditLogObject parse(String log) throws JSONException {
JSONObject jsonObject = new JSONObject(log);
MAPRFSAuditLogObject entity = new MAPRFSAuditLogObject();
- try{
+ try {
String timestamp = jsonObject.getJSONObject("timestamp").getString("$date");
String cmd = jsonObject.getString("operation");
String user = jsonObject.getString("uid");
@@ -43,15 +40,15 @@ public final class MAPRFSAuditLogParser {
String volumeID = jsonObject.getString("volumeId");
String src;
String dst;
- if(jsonObject.has("srcFid")){
+ if (jsonObject.has("srcFid")) {
src = jsonObject.getString("srcFid");
- }else{
+ } else {
src = "null";
}
- if(jsonObject.has("dstFid")){
+ if (jsonObject.has("dstFid")) {
dst = jsonObject.getString("dstFid");
- }else{
+ } else {
dst = "null";
}
entity.user = user;
@@ -62,10 +59,9 @@ public final class MAPRFSAuditLogParser {
entity.status = status;
entity.volume = volumeID;
entity.timestamp = DateTimeUtil.maprhumanDateToMilliseconds(timestamp);
- } catch (Exception e){
+ } catch (Exception e) {
LOG.error("Failed to parse mapr audit log message", e);
- } finally {
- return entity;
}
+ return entity;
}
}
[11/17] eagle git commit: MINOR: rename 'HbaseServiceInstance' as
'HBaseServiceInstance'
Posted by ja...@apache.org.
MINOR: rename 'HbaseServiceInstance' as 'HBaseServiceInstance'
Author: Zhao, Qingwen <qi...@apache.org>
Author: Qingwen Zhao <qi...@gmail.com>
Closes #923 from qingwen220/minor.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/b4d07113
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/b4d07113
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/b4d07113
Branch: refs/heads/branch-0.5
Commit: b4d07113933aa79e8575715370b08ef79c287444
Parents: 8b402e7
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Wed Apr 19 14:57:47 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Wed Apr 19 14:57:47 2017 +0800
----------------------------------------------------------------------
.../org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java | 4 ++--
.../eagle-hbase-web/src/main/webapp/app/apps/hbase/index.js | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/b4d07113/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
index 11deefe..0e975fb 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/storm/JobHistorySpout.java
@@ -240,7 +240,7 @@ public class JobHistorySpout extends BaseRichSpout {
return;
}
- LOG.info("update process time stamp {}", minTimeStamp);
+ LOG.info("updating the latest updated process time {}", minTimeStamp);
Map<String, String> baseTags = new HashMap<String, String>() {
{
put("site", jobHistoryEndpointConfig.site);
@@ -266,7 +266,7 @@ public class JobHistorySpout extends BaseRichSpout {
int tried = 0;
while (tried <= MAX_RETRY_TIMES) {
try {
- LOG.info("start flushing JobProcessTimeStampEntity entities of total number " + entities.size());
+ LOG.info("start flushing {} JobProcessTimeStampEntity entities", entities.size());
client.create(entities);
LOG.info("finish flushing entities of total number " + entities.size());
break;
http://git-wip-us.apache.org/repos/asf/eagle/blob/b4d07113/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/index.js
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/index.js b/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/index.js
index 157cded..3120d5c 100644
--- a/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/index.js
+++ b/eagle-metric/eagle-hbase-web/src/main/webapp/app/apps/hbase/index.js
@@ -65,8 +65,8 @@
METRIC.STATUS_DEAD = "dead";
METRIC.QUERY_HBASE_METRICS = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]{*}&metricName=${metric}&pageSize=${limit}';
METRIC.QUERY_HBASE_METRICS_WITHTIME = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]{*}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}';
- METRIC.QUERY_HBASE_INSTANCE = '${baseURL}/rest/entities?query=HbaseServiceInstance[${condition}]{*}&pageSize=${limit}';
- METRIC.QUERY_HBASE_INSTANCE_AGG = "${baseURL}/rest/entities?query=HbaseServiceInstance[${condition}]<${groups}>{${field}}&pageSize=${limit}";
+ METRIC.QUERY_HBASE_INSTANCE = '${baseURL}/rest/entities?query=HBaseServiceInstance[${condition}]{*}&pageSize=${limit}';
+ METRIC.QUERY_HBASE_INSTANCE_AGG = "${baseURL}/rest/entities?query=HBaseServiceInstance[${condition}]<${groups}>{${field}}&pageSize=${limit}";
METRIC.QUERY_HBASE_METRICS_INTERVAL = '${baseURL}/rest/entities?query=GenericMetricService[${condition}]<${groups}>{${field}}${order}${top}&metricName=${metric}&pageSize=${limit}&startTime=${startTime}&endTime=${endTime}&intervalmin=${intervalMin}&timeSeries=true';
/**
* Fetch query content with current site application configuration