You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/11/22 07:18:58 UTC
[1/6] incubator-eagle git commit: [MINOR] Fix bug caused by EAGLE-791
Repository: incubator-eagle
Updated Branches:
refs/heads/master a267e731e -> 30b127b8d
[MINOR] Fix bug caused by EAGLE-791
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/b821439b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/b821439b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/b821439b
Branch: refs/heads/master
Commit: b821439bce99e03591136436bf71ef22e4d0d655
Parents: ddb45f5
Author: Hao Chen <ha...@apache.org>
Authored: Tue Nov 22 15:13:09 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Nov 22 15:13:09 2016 +0800
----------------------------------------------------------------------
eagle-core/eagle-query/pom.xml | 2 +-
eagle-security/eagle-security-hive/pom.xml | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b821439b/eagle-core/eagle-query/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/pom.xml b/eagle-core/eagle-query/pom.xml
index 119e1bd..ae91780 100644
--- a/eagle-core/eagle-query/pom.xml
+++ b/eagle-core/eagle-query/pom.xml
@@ -102,7 +102,7 @@
</profiles>
<build>
- <finalName>${project.name}-${project.version}</finalName>
+ <finalName>${project.artifactId}-${project.version}</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/b821439b/eagle-security/eagle-security-hive/pom.xml
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hive/pom.xml b/eagle-security/eagle-security-hive/pom.xml
index 37d0949..fbe0034 100644
--- a/eagle-security/eagle-security-hive/pom.xml
+++ b/eagle-security/eagle-security-hive/pom.xml
@@ -84,6 +84,12 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.pentaho</groupId>
+ <artifactId>pentaho-aggdesigner-algorithm</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
[5/6] incubator-eagle git commit: [MINOR] Sync remote to resolve bug
caused by EAGLE-791
Posted by ha...@apache.org.
[MINOR] Sync remote to resolve bug caused by EAGLE-791
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/229d6b32
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/229d6b32
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/229d6b32
Branch: refs/heads/master
Commit: 229d6b32f8e4d5e87c623e58ca9ac0a44346ff2a
Parents: 225d539 dc9fe0b
Author: Hao Chen <ha...@apache.org>
Authored: Tue Nov 22 15:16:37 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Nov 22 15:16:37 2016 +0800
----------------------------------------------------------------------
----------------------------------------------------------------------
[2/6] incubator-eagle git commit: [EAGLE-427] support timestamp in a
query statement
Posted by ha...@apache.org.
[EAGLE-427] support timestamp in a query statement
https://issues.apache.org/jira/browse/EAGLE-427
Author: Zhao, Qingwen <qi...@apache.org>
Closes #672 from qingwen220/EAGLE-427.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/225d5397
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/225d5397
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/225d5397
Branch: refs/heads/master
Commit: 225d53973f4016e9fb22cf95b3166b2ef27e7a8d
Parents: 7471ece
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Tue Nov 22 14:27:32 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Nov 22 15:15:08 2016 +0800
----------------------------------------------------------------------
.../org/apache/eagle/common/DateTimeUtil.java | 8 ++++++
.../entity/GenericEntityScanStreamReader.java | 4 +--
.../log/entity/GenericEntityStreamReaderMT.java | 12 ++++-----
...ricMetricEntityDecompactionStreamReader.java | 4 +--
.../eagle/log/entity/SearchCondition.java | 12 ++++-----
.../apache/eagle/query/TestHBaseLogReader2.java | 28 ++++++++------------
.../common/SplitFullScanEntityReader.java | 4 +--
.../service/generic/ListQueryResource.java | 10 +++----
.../eagle/storage/operation/CompiledQuery.java | 16 +++++------
.../GenericCoprocessorAggregateQuery.java | 8 +++---
.../GenericInMemoryAggregateQuery.java | 8 +++---
11 files changed, 57 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java
index 8d6814b..e7ef3f8 100644
--- a/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java
+++ b/eagle-core/eagle-common/src/main/java/org/apache/eagle/common/DateTimeUtil.java
@@ -161,4 +161,12 @@ public class DateTimeUtil {
Date d = sdf.parse(date);
return d.getTime();
}
+
+ public static long parseTimeStrToMilliseconds(String timeStr) throws ParseException {
+ try {
+ return Long.valueOf(timeStr);
+ } catch (Exception ex) {
+ return humanDateToSeconds(timeStr) * ONESECOND;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java
index 3f97e78..e0a7119 100755
--- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java
+++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityScanStreamReader.java
@@ -80,8 +80,8 @@ public class GenericEntityScanStreamReader extends StreamReader {
}
// Process the time range if needed
if(entityDef.isTimeSeries()){
- start = DateTimeUtil.humanDateToDate(condition.getStartTime());
- end = DateTimeUtil.humanDateToDate(condition.getEndTime());
+ start = new Date(condition.getStartTime());
+ end = new Date(condition.getEndTime());
}else{
start = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_START_HUMANTIME);
end = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_END_HUMANTIME);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java
index 1946d6c..bf72a36 100755
--- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java
+++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericEntityStreamReaderMT.java
@@ -43,8 +43,8 @@ public class GenericEntityStreamReaderMT extends StreamReader{
public GenericEntityStreamReaderMT(String serviceName, SearchCondition condition, int numThreads) throws Exception{
checkIsTimeSeries(serviceName);
checkNumThreads(numThreads);
- long queryStartTime = DateTimeUtil.humanDateToSeconds(condition.getStartTime())*1000;
- long queryEndTime = DateTimeUtil.humanDateToSeconds(condition.getEndTime())*1000;
+ long queryStartTime = condition.getStartTime();
+ long queryEndTime = condition.getEndTime();
long subStartTime = queryStartTime;
long subEndTime = 0;
long interval = (queryEndTime-queryStartTime) / numThreads;
@@ -56,11 +56,11 @@ public class GenericEntityStreamReaderMT extends StreamReader{
}else{
subEndTime = subStartTime + interval;
}
- String strStartTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(subStartTime);
- String strEndTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(subEndTime);
+ //String strStartTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(subStartTime);
+ //String strEndTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(subEndTime);
SearchCondition sc = new SearchCondition(condition);
- sc.setStartTime(strStartTime);
- sc.setEndTime(strEndTime);
+ sc.setStartTime(subStartTime);
+ sc.setEndTime(subEndTime);
GenericEntityStreamReader reader = new GenericEntityStreamReader(serviceName, sc);
readers.add(reader);
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericMetricEntityDecompactionStreamReader.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericMetricEntityDecompactionStreamReader.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericMetricEntityDecompactionStreamReader.java
index b1dd64c..1cf3905 100755
--- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericMetricEntityDecompactionStreamReader.java
+++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/GenericMetricEntityDecompactionStreamReader.java
@@ -47,8 +47,8 @@ public class GenericMetricEntityDecompactionStreamReader extends StreamReader im
ed = EntityDefinitionManager.getEntityByServiceName(serviceName);
checkIsMetric(ed);
reader = new GenericEntityStreamReader(serviceName, condition, metricName);
- start = DateTimeUtil.humanDateToSeconds(condition.getStartTime())*1000;
- end = DateTimeUtil.humanDateToSeconds(condition.getEndTime())*1000;
+ start = condition.getStartTime();
+ end = condition.getEndTime();
}
private void checkIsMetric(EntityDefinition ed){
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/SearchCondition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/SearchCondition.java b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/SearchCondition.java
index fada0e2..68db2c4 100755
--- a/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/SearchCondition.java
+++ b/eagle-core/eagle-query/eagle-entity-base/src/main/java/org/apache/eagle/log/entity/SearchCondition.java
@@ -34,8 +34,8 @@ import java.util.Map;
* 7. pagination: pageSize and startRowkey
*/
public class SearchCondition{
- private String startTime;
- private String endTime;
+ private long startTime;
+ private long endTime;
private Filter filter;
private List<String> outputFields;
private boolean outputAll;
@@ -113,16 +113,16 @@ public class SearchCondition{
public void setOutputFields(List<String> outputFields) {
this.outputFields = outputFields;
}
- public String getStartTime() {
+ public long getStartTime() {
return startTime;
}
- public void setStartTime(String startTime) {
+ public void setStartTime(long startTime) {
this.startTime = startTime;
}
- public String getEndTime() {
+ public long getEndTime() {
return endTime;
}
- public void setEndTime(String endTime) {
+ public void setEndTime(long endTime) {
this.endTime = endTime;
}
public List<String> getPartitionValues() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java b/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
index e3db5c0..6513452 100755
--- a/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
+++ b/eagle-core/eagle-query/eagle-query-base/src/test/java/org/apache/eagle/query/TestHBaseLogReader2.java
@@ -24,7 +24,6 @@ import org.apache.eagle.log.entity.SearchCondition;
import org.apache.eagle.log.entity.meta.EntityDefinition;
import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
-import org.apache.eagle.query.ListQueryCompiler;
import org.apache.eagle.service.hbase.EmbeddedHbase;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.Assert;
@@ -32,12 +31,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.text.ParseException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
+import java.util.*;
public class TestHBaseLogReader2 {
private final static Logger LOG = LoggerFactory.getLogger(TestHBaseLogReader2.class);
@@ -80,8 +74,8 @@ public class TestHBaseLogReader2 {
writer.write(entities);
// for timezone difference between UTC & localtime, enlarge the search range
- long queryStartTimestamp = timestamp1-24*60*60*1000;
- long queryEndTimestamp = timestamp1+24*60*60*1000;
+ long queryStartTimestamp = timestamp1 - DateTimeUtil.ONEDAY;
+ long queryEndTimestamp = timestamp1 + DateTimeUtil.ONEDAY;
LOG.info("Query start timestamp:" + queryStartTimestamp);
LOG.info("Query end timestamp:" + queryEndTimestamp);
@@ -100,8 +94,8 @@ public class TestHBaseLogReader2 {
condition.setStartRowkey(null);
condition.setPageSize(Integer.MAX_VALUE);
- condition.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(0));
- condition.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(queryEndTimestamp));
+ condition.setStartTime(0);
+ condition.setEndTime(queryEndTimestamp);
GenericEntityBatchReader reader = new GenericEntityBatchReader(serviceName, condition);
List<TestTimeSeriesAPIEntity> list = reader.read();
@@ -111,18 +105,18 @@ public class TestHBaseLogReader2 {
Assert.assertEquals("field7", list.get(0).getField7());
// for timezone difference between UTC & localtime, enlarge the search range
- queryStartTimestamp = timestamp1-24*60*60*1000;
- queryEndTimestamp = timestamp2+24*60*60*1000; // eagle timestamp is rounded to seconds
- condition.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(queryStartTimestamp));
- condition.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(queryEndTimestamp));
+ queryStartTimestamp = timestamp1 - DateTimeUtil.ONEDAY;
+ queryEndTimestamp = timestamp2 + DateTimeUtil.ONEDAY; // eagle timestamp is rounded to seconds
+ condition.setStartTime(queryStartTimestamp);
+ condition.setEndTime(queryEndTimestamp);
reader = new GenericEntityBatchReader(serviceName, condition);
list = reader.read();
Assert.assertEquals(2, list.size());
queryStartTimestamp = timestamp1;
queryEndTimestamp = timestamp1; // eagle timestamp is rounded to seconds
- condition.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(queryStartTimestamp));
- condition.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(queryEndTimestamp));
+ condition.setStartTime(queryStartTimestamp);
+ condition.setEndTime(queryEndTimestamp);
reader = new GenericEntityBatchReader(serviceName, condition);
list = reader.read();
Assert.assertEquals(0, list.size());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/common/SplitFullScanEntityReader.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/common/SplitFullScanEntityReader.java b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/common/SplitFullScanEntityReader.java
index ae9ecef..f6009b5 100755
--- a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/common/SplitFullScanEntityReader.java
+++ b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/common/SplitFullScanEntityReader.java
@@ -141,8 +141,8 @@ public class SplitFullScanEntityReader<ENTITY extends TaggedLogAPIEntity> {
final SearchCondition condition = new SearchCondition();
final String slotStartTimeString = DateTimeUtil.secondsToHumanDate(slotStartTime / 1000);
final String slotEndTimeString = DateTimeUtil.secondsToHumanDate(slotEndTime / 1000);
- condition.setStartTime(slotStartTimeString);
- condition.setEndTime(slotEndTimeString);
+ condition.setStartTime(slotStartTime);
+ condition.setEndTime(slotEndTime);
condition.setFilter(comp.filter());
condition.setQueryExpression(comp.getQueryExpression());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/generic/ListQueryResource.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/generic/ListQueryResource.java b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/generic/ListQueryResource.java
index b14dc22..eb05bf5 100755
--- a/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/generic/ListQueryResource.java
+++ b/eagle-core/eagle-query/eagle-service-base/src/main/java/org/apache/eagle/service/generic/ListQueryResource.java
@@ -117,8 +117,8 @@ public class ListQueryResource {
EntityDefinition ed = EntityDefinitionManager.getEntityByServiceName(serviceName);
if(ed.isTimeSeries()){
// TODO check timestamp exists for timeseries or topology data
- condition.setStartTime(startTime);
- condition.setEndTime(endTime);
+ condition.setStartTime(DateTimeUtil.parseTimeStrToMilliseconds(startTime));
+ condition.setEndTime(DateTimeUtil.parseTimeStrToMilliseconds(endTime));
}
// 4. Set HBase start scanning rowkey if given
@@ -240,8 +240,8 @@ public class ListQueryResource {
EntityDefinition ed = EntityDefinitionManager.getEntityByServiceName(serviceName);
if(ed.isTimeSeries()){
// TODO check timestamp exists for timeseries or topology data
- condition.setStartTime(startTime);
- condition.setEndTime(endTime);
+ condition.setStartTime(DateTimeUtil.parseTimeStrToMilliseconds(startTime));
+ condition.setEndTime(DateTimeUtil.parseTimeStrToMilliseconds(endTime));
}
condition.setOutputVerbose(verbose==null || verbose );
condition.setOutputAlias(comp.getOutputAlias());
@@ -399,7 +399,7 @@ public class ListQueryResource {
LOG.info("Output: " + StringUtils.join(condition.getOutputFields(), ", "));
}
TimeSeriesAggregator tsAgg = new TimeSeriesAggregator(groupbyFields, comp.aggregateFunctionTypes(), aggregateFields,
- DateTimeUtil.humanDateToDate(condition.getStartTime()).getTime(), DateTimeUtil.humanDateToDate(condition.getEndTime()).getTime(), intervalmin*60*1000);
+ condition.getStartTime(), condition.getEndTime(), intervalmin*60*1000);
if(parallel <= 0){
reader.register(tsAgg);
}else{
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
index e3368ab..9ec5751 100644
--- a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
+++ b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
@@ -216,24 +216,20 @@ public class CompiledQuery {
// TODO check Time exists for timeseries or topology data
long endTimeMillis = System.currentTimeMillis();
long startTimeMills = endTimeMillis - 30 * DateTimeUtil.ONEDAY;
- String endTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(endTimeMillis);
- String startTime = DateTimeUtil.millisecondsToHumanDateWithSeconds(startTimeMills);
if(this.rawQuery.getStartTime() != null && this.rawQuery.getEndTime() != null) {
- endTime = this.rawQuery.getEndTime();
- startTime = this.rawQuery.getStartTime();
- endTimeMillis = DateTimeUtil.humanDateToSeconds(endTime) * 1000;
- startTimeMills = DateTimeUtil.humanDateToSeconds(startTime) * 1000;
+ endTimeMillis = DateTimeUtil.parseTimeStrToMilliseconds(rawQuery.getEndTime());
+ startTimeMills = DateTimeUtil.parseTimeStrToMilliseconds(rawQuery.getStartTime());
} else {
LOG.warn("startTime or endTime is not given, use [currentSystemTime - 30 days, currentSystemTime]");
}
- this.searchCondition.setStartTime(startTime);
- this.searchCondition.setEndTime(endTime);
+ this.searchCondition.setStartTime(startTimeMills);
+ this.searchCondition.setEndTime(endTimeMillis);
this.setStartTime(startTimeMills);
this.setEndTime(endTimeMillis);
}else{
- this.searchCondition.setStartTime("0");
- this.searchCondition.setEndTime("1");
+ this.searchCondition.setStartTime(0);
+ this.searchCondition.setEndTime(1);
this.setStartTime(0);
this.setEndTime(1);
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericCoprocessorAggregateQuery.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericCoprocessorAggregateQuery.java b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericCoprocessorAggregateQuery.java
index 85044ec..b6242c8 100644
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericCoprocessorAggregateQuery.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericCoprocessorAggregateQuery.java
@@ -162,8 +162,8 @@ public class GenericCoprocessorAggregateQuery implements GenericQuery {
}
// Process the time range if needed
if (entityDef.isTimeSeries()) {
- start = DateTimeUtil.humanDateToDate(searchCondition.getStartTime());
- end = DateTimeUtil.humanDateToDate(searchCondition.getEndTime());
+ start = new Date(searchCondition.getStartTime());
+ end = new Date(searchCondition.getEndTime());
} else {
start = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_START_HUMANTIME);
end = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_END_HUMANTIME);
@@ -278,8 +278,8 @@ public class GenericCoprocessorAggregateQuery implements GenericQuery {
super(reader, query);
try {
if (entityDef.isTimeSeries()) {
- this.start = DateTimeUtil.humanDateToDate(searchCondition.getStartTime());
- this.end = DateTimeUtil.humanDateToDate(searchCondition.getEndTime());
+ this.start = new Date(searchCondition.getStartTime());
+ this.end = new Date(searchCondition.getEndTime());
} else {
start = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_START_HUMANTIME);
end = DateTimeUtil.humanDateToDate(EntityConstants.FIXED_READ_END_HUMANTIME);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/225d5397/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericInMemoryAggregateQuery.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericInMemoryAggregateQuery.java b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericInMemoryAggregateQuery.java
index 934929c..068fe0b 100644
--- a/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericInMemoryAggregateQuery.java
+++ b/eagle-core/eagle-query/eagle-storage-hbase/src/main/java/org/apache/eagle/storage/hbase/query/aggregate/GenericInMemoryAggregateQuery.java
@@ -174,9 +174,11 @@ public class GenericInMemoryAggregateQuery implements GenericQuery {
reader = new GenericMetricEntityDecompactionStreamReader(this.prefix, searchCondition);
}
TimeSeriesAggregator tsAgg = new TimeSeriesAggregator(groupbyFields,
- aggregateCondition.getAggregateFunctionTypes(), aggregateFields,
- DateTimeUtil.humanDateToDate(searchCondition.getStartTime()).getTime(),
- DateTimeUtil.humanDateToDate(searchCondition.getEndTime()).getTime(), aggregateCondition.getIntervalMS());
+ aggregateCondition.getAggregateFunctionTypes(),
+ aggregateFields,
+ searchCondition.getStartTime(),
+ searchCondition.getEndTime(),
+ aggregateCondition.getIntervalMS());
reader.register(tsAgg);
// for sorting
[3/6] incubator-eagle git commit: [EAGLE-787] add healthy check for
hadoop-queue/topology-health/spark-history apps
Posted by ha...@apache.org.
[EAGLE-787] add healthy check for hadoop-queue/topology-health/spark-history apps
Author: wujinhu <wu...@126.com>
Closes #671 from wujinhu/EAGLE-787.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/7471eced
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/7471eced
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/7471eced
Branch: refs/heads/master
Commit: 7471eced641b132df7e2950a93a00914be75cc97
Parents: 620959b
Author: wujinhu <wu...@126.com>
Authored: Tue Nov 22 09:53:04 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Nov 22 15:15:08 2016 +0800
----------------------------------------------------------------------
.../impl/ApplicationHealthCheckBase.java | 44 ++++++++
.../impl/ApplicationHealthCheckServiceImpl.java | 10 +-
.../eagle/app/spi/ApplicationProvider.java | 1 +
.../src/main/resources/HealthCheckTemplate.vm | 2 +-
.../queue/HadoopQueueRunningAppProvider.java | 7 ++
...adoopQueueRunningApplicationHealthCheck.java | 100 +++++++++++++++++++
.../MRHistoryJobApplicationHealthCheck.java | 41 +++++---
.../history/SparkHistoryJobAppProvider.java | 7 ++
.../SparkHistoryJobApplicationHealthCheck.java | 99 ++++++++++++++++++
.../topology/TopologyCheckAppProvider.java | 7 ++
.../TopologyCheckApplicationHealthCheck.java | 98 ++++++++++++++++++
.../eagle/topology/TopologyConstants.java | 1 +
12 files changed, 399 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
new file mode 100644
index 0000000..104e76d
--- /dev/null
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckBase.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.app.service.impl;
+
+import com.codahale.metrics.health.HealthCheck;
+import com.google.inject.Inject;
+import com.typesafe.config.Config;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
+
+public abstract class ApplicationHealthCheckBase extends HealthCheck {
+ private static final String APP_ID_PATH = "appId";
+ protected static final long DEFAULT_MAX_DELAY_TIME = 2 * 60 * 60 * 1000L;
+ protected static final String MAX_DELAY_TIME_KEY = "application.maxDelayTime";
+
+ protected Config config;
+
+ @Inject
+ private ApplicationEntityService applicationEntityService;
+
+ protected ApplicationHealthCheckBase(Config config) {
+ this.config = config;
+ }
+
+ protected ApplicationEntity.Status getApplicationStatus() {
+ ApplicationEntity applicationEntity = applicationEntityService.getByUUIDOrAppId(null, config.getString(APP_ID_PATH));
+ return applicationEntity.getStatus();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
index 9d98b2f..4aea963 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/service/impl/ApplicationHealthCheckServiceImpl.java
@@ -19,6 +19,7 @@ package org.apache.eagle.app.service.impl;
import com.codahale.metrics.health.HealthCheck;
import com.google.inject.Inject;
+import com.google.inject.Injector;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import io.dropwizard.setup.Environment;
@@ -56,6 +57,9 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
private static final String HEALTH_PUBLISHER_IMPL_PATH = "application.healthCheck.publisher.publisherImpl";
@Inject
+ private Injector currentInjector;
+
+ @Inject
public ApplicationHealthCheckServiceImpl(ApplicationProviderService applicationProviderService,
ApplicationEntityService applicationEntityService,
Config config) {
@@ -105,11 +109,12 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
}
ApplicationProvider<?> appProvider = applicationProviderService.getApplicationProviderByType(appEntity.getDescriptor().getType());
HealthCheck applicationHealthCheck = appProvider.getAppHealthCheck(
- ConfigFactory.parseMap(appEntity.getConfiguration())
+ ConfigFactory.parseMap(appEntity.getContext())
.withFallback(config)
- .withFallback(ConfigFactory.parseMap(appEntity.getContext()))
+ .withFallback(ConfigFactory.parseMap(appEntity.getConfiguration()))
);
this.environment.healthChecks().register(appEntity.getAppId(), applicationHealthCheck);
+ currentInjector.injectMembers(applicationHealthCheck);
synchronized (lock) {
if (!appHealthChecks.containsKey(appEntity.getAppId())) {
appHealthChecks.put(appEntity.getAppId(), applicationHealthCheck);
@@ -137,7 +142,6 @@ public class ApplicationHealthCheckServiceImpl extends ApplicationHealthCheckSer
registerAll();
synchronized (lock) {
for (String appId : appHealthChecks.keySet()) {
- LOG.info("check application {}", appId);
HealthCheck.Result result = appHealthChecks.get(appId).execute();
if (result.isHealthy()) {
LOG.info("application {} is healthy", appId);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
index 143e026..d9c1eff 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/java/org/apache/eagle/app/spi/ApplicationProvider.java
@@ -23,6 +23,7 @@ import org.apache.eagle.app.Application;
import org.apache.eagle.app.service.ApplicationListener;
import org.apache.eagle.common.module.ModuleRegistry;
import org.apache.eagle.metadata.model.ApplicationDesc;
+import org.apache.eagle.metadata.service.ApplicationEntityService;
import java.lang.reflect.ParameterizedType;
import java.util.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm b/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
index b368458..51c1186 100644
--- a/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
+++ b/eagle-core/eagle-app/eagle-app-base/src/main/resources/HealthCheckTemplate.vm
@@ -23,7 +23,7 @@
<body>
#set ( $elem = $unHealthyContext )
-<p><b>Message: </b>$elem["appId"] is delayed</p>
+<p><b>Message: </b>$elem["appId"] has delayed</p>
<p><b>Detail: </b>$elem["unHealthyMessage"]</p>
</body>
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppProvider.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppProvider.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppProvider.java
index 916dd5b..5d4078c 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppProvider.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningAppProvider.java
@@ -16,10 +16,17 @@
*/
package org.apache.eagle.hadoop.queue;
+import com.codahale.metrics.health.HealthCheck;
+import com.typesafe.config.Config;
import org.apache.eagle.app.spi.AbstractApplicationProvider;
public class HadoopQueueRunningAppProvider extends AbstractApplicationProvider<HadoopQueueRunningApp> {
public HadoopQueueRunningApp getApplication() {
return new HadoopQueueRunningApp();
}
+
+ @Override
+ public HealthCheck getAppHealthCheck(Config config) {
+ return new HadoopQueueRunningApplicationHealthCheck(config);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
new file mode 100644
index 0000000..c73021b
--- /dev/null
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/HadoopQueueRunningApplicationHealthCheck.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.hadoop.queue;
+
+import com.typesafe.config.Config;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.eagle.app.service.impl.ApplicationHealthCheckBase;
+import org.apache.eagle.hadoop.queue.common.HadoopClusterConstants;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class HadoopQueueRunningApplicationHealthCheck extends ApplicationHealthCheckBase {
+ private static final Logger LOG = LoggerFactory.getLogger(HadoopQueueRunningApplicationHealthCheck.class);
+
+ private HadoopQueueRunningAppConfig hadoopQueueRunningAppConfig;
+
+ public HadoopQueueRunningApplicationHealthCheck(Config config) {
+ super(config);
+ this.hadoopQueueRunningAppConfig = new HadoopQueueRunningAppConfig(config);
+ }
+
+ @Override
+ public Result check() {
+ HadoopQueueRunningAppConfig.EagleProps eagleServiceConfig = this.hadoopQueueRunningAppConfig.eagleProps;
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ eagleServiceConfig.eagleService.host,
+ eagleServiceConfig.eagleService.port,
+ eagleServiceConfig.eagleService.username,
+ eagleServiceConfig.eagleService.password);
+
+ client.getJerseyClient().setReadTimeout(60000);
+
+ try {
+ ApplicationEntity.Status status = getApplicationStatus();
+ if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
+ String message = String.format("Application is not running, status is %s", status.toString());
+ return Result.unhealthy(message);
+ }
+
+ String query = String.format("%s[@site=\"%s\"]<@site>{max(timestamp)}",
+ Constants.GENERIC_METRIC_SERVICE,
+ hadoopQueueRunningAppConfig.eagleProps.site);
+
+ GenericServiceAPIResponseEntity response = client
+ .search(query)
+ .metricName(HadoopClusterConstants.MetricName.HADOOP_CLUSTER_ALLOCATED_MEMORY)
+ .startTime(System.currentTimeMillis() - 2 * 60 * 60000L)
+ .endTime(System.currentTimeMillis())
+ .pageSize(Integer.MAX_VALUE)
+ .send();
+ List<Map<List<String>, List<Double>>> results = response.getObj();
+ long currentProcessTimeStamp = results.get(0).get("value").get(0).longValue();
+ long currentTimeStamp = System.currentTimeMillis();
+ long maxDelayTime = DEFAULT_MAX_DELAY_TIME;
+ if (hadoopQueueRunningAppConfig.getConfig().hasPath(MAX_DELAY_TIME_KEY)) {
+ maxDelayTime = hadoopQueueRunningAppConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
+ }
+
+ if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime) {
+ String message = String.format("Current process time is %sms, delay %s hours",
+ currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp) * 1.0 / 60000L / 60);
+ return Result.unhealthy(message);
+ } else {
+ return Result.healthy();
+ }
+ } catch (Exception e) {
+ return Result.unhealthy(ExceptionUtils.getStackTrace(e.getCause()));
+ } finally {
+ client.getJerseyClient().destroy();
+ try {
+ client.close();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
index ab078f5..91e16df 100644
--- a/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
+++ b/eagle-jpm/eagle-jpm-mr-history/src/main/java/org/apache/eagle/jpm/mr/history/MRHistoryJobApplicationHealthCheck.java
@@ -17,11 +17,12 @@
package org.apache.eagle.jpm.mr.history;
-import com.codahale.metrics.health.HealthCheck;
import com.typesafe.config.Config;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.eagle.app.service.impl.ApplicationHealthCheckBase;
import org.apache.eagle.jpm.util.Constants;
import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.metadata.model.ApplicationEntity;
import org.apache.eagle.service.client.IEagleServiceClient;
import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
import org.slf4j.Logger;
@@ -30,28 +31,33 @@ import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Map;
-public class MRHistoryJobApplicationHealthCheck extends HealthCheck {
+public class MRHistoryJobApplicationHealthCheck extends ApplicationHealthCheckBase {
private static final Logger LOG = LoggerFactory.getLogger(MRHistoryJobApplicationHealthCheck.class);
private MRHistoryJobConfig mrHistoryJobConfig;
- private static final long DEFAULT_MAX_DELAY_TIME = 2 * 60 * 60 * 1000L;
- private static final String MAX_DELAY_TIME_KEY = "application.maxDelayTime";
public MRHistoryJobApplicationHealthCheck(Config config) {
+ super(config);
mrHistoryJobConfig = MRHistoryJobConfig.newInstance(config);
}
@Override
public Result check() {
- try {
- MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig = mrHistoryJobConfig.getEagleServiceConfig();
- IEagleServiceClient client = new EagleServiceClientImpl(
- eagleServiceConfig.eagleServiceHost,
- eagleServiceConfig.eagleServicePort,
- eagleServiceConfig.username,
- eagleServiceConfig.password);
+ MRHistoryJobConfig.EagleServiceConfig eagleServiceConfig = mrHistoryJobConfig.getEagleServiceConfig();
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ eagleServiceConfig.eagleServiceHost,
+ eagleServiceConfig.eagleServicePort,
+ eagleServiceConfig.username,
+ eagleServiceConfig.password);
+
+ client.getJerseyClient().setReadTimeout(eagleServiceConfig.readTimeoutSeconds * 1000);
- client.getJerseyClient().setReadTimeout(eagleServiceConfig.readTimeoutSeconds * 1000);
+ try {
+ ApplicationEntity.Status status = getApplicationStatus();
+ if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
+ String message = String.format("Application is not running, status is %s", status.toString());
+ return Result.unhealthy(message);
+ }
String query = String.format("%s[@site=\"%s\"]<@site>{max(currentTimeStamp)}",
Constants.JPA_JOB_PROCESS_TIME_STAMP_NAME,
@@ -73,14 +79,21 @@ public class MRHistoryJobApplicationHealthCheck extends HealthCheck {
}
if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime) {
- String message = String.format("current process time %sms, delay %sms",
- currentProcessTimeStamp, currentTimeStamp - currentProcessTimeStamp);
+ String message = String.format("Current process time is %sms, delay %s hours",
+ currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp) * 1.0 / 60000L / 60);
return Result.unhealthy(message);
} else {
return Result.healthy();
}
} catch (Exception e) {
return Result.unhealthy(ExceptionUtils.getStackTrace(e.getCause()));
+ } finally {
+ client.getJerseyClient().destroy();
+ try {
+ client.close();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobAppProvider.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobAppProvider.java b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobAppProvider.java
index 343d9c2..2b962c9 100644
--- a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobAppProvider.java
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobAppProvider.java
@@ -17,6 +17,8 @@
package org.apache.eagle.jpm.spark.history;
+import com.codahale.metrics.health.HealthCheck;
+import com.typesafe.config.Config;
import org.apache.eagle.app.spi.AbstractApplicationProvider;
public class SparkHistoryJobAppProvider extends AbstractApplicationProvider<SparkHistoryJobApp> {
@@ -24,4 +26,9 @@ public class SparkHistoryJobAppProvider extends AbstractApplicationProvider<Spar
public SparkHistoryJobApp getApplication() {
return new SparkHistoryJobApp();
}
+
+ @Override
+ public HealthCheck getAppHealthCheck(Config config) {
+ return new SparkHistoryJobApplicationHealthCheck(config);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
new file mode 100644
index 0000000..bbca566
--- /dev/null
+++ b/eagle-jpm/eagle-jpm-spark-history/src/main/java/org/apache/eagle/jpm/spark/history/SparkHistoryJobApplicationHealthCheck.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.jpm.spark.history;
+
+import com.typesafe.config.Config;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.eagle.app.service.impl.ApplicationHealthCheckBase;
+import org.apache.eagle.jpm.util.Constants;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class SparkHistoryJobApplicationHealthCheck extends ApplicationHealthCheckBase {
+ private static final Logger LOG = LoggerFactory.getLogger(SparkHistoryJobApplicationHealthCheck.class);
+
+ private SparkHistoryJobAppConfig sparkHistoryJobAppConfig;
+
+ public SparkHistoryJobApplicationHealthCheck(Config config) {
+ super(config);
+ this.sparkHistoryJobAppConfig = SparkHistoryJobAppConfig.newInstance(config);
+ }
+
+ @Override
+ public Result check() {
+ SparkHistoryJobAppConfig.EagleInfo eagleServiceConfig = sparkHistoryJobAppConfig.eagleInfo;
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ eagleServiceConfig.host,
+ eagleServiceConfig.port,
+ eagleServiceConfig.username,
+ eagleServiceConfig.password);
+
+ client.getJerseyClient().setReadTimeout(eagleServiceConfig.timeout * 1000);
+
+ try {
+ ApplicationEntity.Status status = getApplicationStatus();
+ if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
+ String message = String.format("Application is not running, status is %s", status.toString());
+ return Result.unhealthy(message);
+ }
+
+ String query = String.format("%s[@site=\"%s\"]<@site>{max(endTime)}",
+ Constants.SPARK_APP_SERVICE_ENDPOINT_NAME,
+ sparkHistoryJobAppConfig.stormConfig.siteId);
+
+ GenericServiceAPIResponseEntity response = client
+ .search(query)
+ .startTime(0L)
+ .endTime(System.currentTimeMillis())
+ .pageSize(10)
+ .send();
+
+ List<Map<List<String>, List<Double>>> results = response.getObj();
+ long currentProcessTimeStamp = results.get(0).get("value").get(0).longValue();
+ long currentTimeStamp = System.currentTimeMillis();
+ long maxDelayTime = DEFAULT_MAX_DELAY_TIME;
+ if (sparkHistoryJobAppConfig.getConfig().hasPath(MAX_DELAY_TIME_KEY)) {
+ maxDelayTime = sparkHistoryJobAppConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
+ }
+
+ if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime * 3) {
+ String message = String.format("Current process time is %sms, delay %s hours",
+ currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp) * 1.0 / 60000L / 60);
+ return Result.unhealthy(message);
+ } else {
+ return Result.healthy();
+ }
+ } catch (Exception e) {
+ return Result.unhealthy(ExceptionUtils.getStackTrace(e.getCause()));
+ } finally {
+ client.getJerseyClient().destroy();
+ try {
+ client.close();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
index 5969a7a..5766454 100644
--- a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckAppProvider.java
@@ -18,6 +18,8 @@
package org.apache.eagle.topology;
+import com.codahale.metrics.health.HealthCheck;
+import com.typesafe.config.Config;
import org.apache.eagle.app.spi.AbstractApplicationProvider;
public class TopologyCheckAppProvider extends AbstractApplicationProvider<TopologyCheckApp> {
@@ -25,4 +27,9 @@ public class TopologyCheckAppProvider extends AbstractApplicationProvider<Topolo
public TopologyCheckApp getApplication() {
return new TopologyCheckApp();
}
+
+ @Override
+ public HealthCheck getAppHealthCheck(Config config) {
+ return new TopologyCheckApplicationHealthCheck(config);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
new file mode 100644
index 0000000..25730c8
--- /dev/null
+++ b/eagle-topology-check/eagle-topology-app/src/main/java/org/apache/eagle/topology/TopologyCheckApplicationHealthCheck.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.eagle.topology;
+
+import com.typesafe.config.Config;
+import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.eagle.app.service.impl.ApplicationHealthCheckBase;
+import org.apache.eagle.log.entity.GenericServiceAPIResponseEntity;
+import org.apache.eagle.metadata.model.ApplicationEntity;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.apache.eagle.service.client.impl.EagleServiceClientImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+public class TopologyCheckApplicationHealthCheck extends ApplicationHealthCheckBase {
+ private static final Logger LOG = LoggerFactory.getLogger(TopologyCheckApplicationHealthCheck.class);
+
+ private TopologyCheckAppConfig topologyCheckAppConfig;
+
+ public TopologyCheckApplicationHealthCheck(Config config) {
+ super(config);
+ topologyCheckAppConfig = TopologyCheckAppConfig.newInstance(config);
+ }
+
+ @Override
+ public Result check() {
+ //FIXME, this application owner please add eagle server config to Class TopologyCheckAppConfig
+ IEagleServiceClient client = new EagleServiceClientImpl(
+ topologyCheckAppConfig.getConfig().getString("service.host"),
+ topologyCheckAppConfig.getConfig().getInt("service.port"),
+ topologyCheckAppConfig.getConfig().getString("service.username"),
+ topologyCheckAppConfig.getConfig().getString("service.password"));
+
+ client.getJerseyClient().setReadTimeout(topologyCheckAppConfig.getConfig().getInt("service.readTimeOutSeconds") * 1000);
+
+ try {
+ ApplicationEntity.Status status = getApplicationStatus();
+ if (!status.toString().equals(ApplicationEntity.Status.RUNNING.toString())) {
+ String message = String.format("Application is not running, status is %s", status.toString());
+ return Result.unhealthy(message);
+ }
+
+ String query = String.format("%s[@site=\"%s\"]<@site>{max(timestamp)}",
+ TopologyConstants.GENERIC_METRIC_SERVICE,
+ topologyCheckAppConfig.dataExtractorConfig.site);
+
+ GenericServiceAPIResponseEntity response = client
+ .search(query)
+ .metricName(String.format(TopologyConstants.METRIC_LIVE_RATIO_NAME_FORMAT, TopologyConstants.REGIONSERVER_ROLE))
+ .startTime(System.currentTimeMillis() - 2 * 60 * 60000L)
+ .endTime(System.currentTimeMillis())
+ .pageSize(Integer.MAX_VALUE)
+ .send();
+ List<Map<List<String>, List<Double>>> results = response.getObj();
+ long currentProcessTimeStamp = results.get(0).get("value").get(0).longValue();
+ long currentTimeStamp = System.currentTimeMillis();
+ long maxDelayTime = DEFAULT_MAX_DELAY_TIME;
+ if (topologyCheckAppConfig.getConfig().hasPath(MAX_DELAY_TIME_KEY)) {
+ maxDelayTime = topologyCheckAppConfig.getConfig().getLong(MAX_DELAY_TIME_KEY);
+ }
+
+ if (currentTimeStamp - currentProcessTimeStamp > maxDelayTime) {
+ String message = String.format("Current process time is %sms, delay %s hours",
+ currentProcessTimeStamp, (currentTimeStamp - currentProcessTimeStamp) * 1.0 / 60000L / 60);
+ return Result.unhealthy(message);
+ } else {
+ return Result.healthy();
+ }
+ } catch (Exception e) {
+ return Result.unhealthy(ExceptionUtils.getStackTrace(e.getCause()));
+ } finally {
+ client.getJerseyClient().destroy();
+ try {
+ client.close();
+ } catch (Exception e) {
+ LOG.warn("{}", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/7471eced/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
index 74f446b..3e535a8 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/TopologyConstants.java
@@ -26,6 +26,7 @@ public class TopologyConstants {
public static final String HBASE_INSTANCE_SERVICE_NAME = "HbaseServiceInstance";
public static final String MR_INSTANCE_SERVICE_NAME = "MRServiceInstance";
public static final String JN_INSTANCE_SERVICE_NAME = "JNServiceInstance";
+ public static final String GENERIC_METRIC_SERVICE = "GenericMetricService";
public static final int DEFAULT_READ_TIMEOUT = 30 * 60 * 1000; // in milliseconds
public static final Pattern HTTP_HOST_MATCH_PATTERN = Pattern.compile("^https?://(.+?):-?(\\d+)/?");
[4/6] incubator-eagle git commit: [EAGLE-788] Add unit test for
eagle-client-base module
Posted by ha...@apache.org.
[EAGLE-788] Add unit test for eagle-client-base module
EAGLE-788 Add unit test for eagle-client-base module
- Add unit test for eagle-client-base module.
https://issues.apache.org/jira/browse/EAGLE-788
Author: chitin <ch...@gmail.com>
Closes #667 from chitin/EAGLE-788.
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/620959bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/620959bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/620959bb
Branch: refs/heads/master
Commit: 620959bbdd334aab15fd43e8312554f5e67b4a75
Parents: b821439
Author: chitin <ch...@gmail.com>
Authored: Tue Nov 22 01:35:20 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Nov 22 15:15:08 2016 +0800
----------------------------------------------------------------------
.../client/impl/TestDeleteRequestBuilder.java | 67 +++++++++++++
.../client/impl/TestEagleServiceClient.java | 98 ++++++++++++++++++++
.../TestEagleServiceGroupByQueryRequest.java | 55 +++++++++++
.../impl/TestEagleServiceQueryBuilder.java | 39 ++++++++
.../TestListQueryAPIReponseConvertHelper.java | 61 ++++++++++++
.../eagle/service/client/impl/TestQuery.java | 49 ++++++++++
...TestRowkeyQueryAPIResponseConvertHelper.java | 57 ++++++++++++
7 files changed, 426 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestDeleteRequestBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestDeleteRequestBuilder.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestDeleteRequestBuilder.java
new file mode 100644
index 0000000..993305a
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestDeleteRequestBuilder.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.eagle.service.client.EagleServiceConnector;
+import org.apache.eagle.service.client.EagleServiceSingleEntityQueryRequest;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * @Since 11/17/16.
+ */
+public class TestDeleteRequestBuilder {
+
+ @Test
+ public void testDeleteRequestByIds() throws Exception {
+ EagleServiceConnector connector = mock(EagleServiceConnector.class);
+ IEagleServiceClient client = new EagleServiceClientImpl(connector);
+ List<String> ids = new ArrayList<>();
+ ids.add("id1");
+ ids.add("id2");
+ ids.add("id3");
+ DeleteRequestBuilder deleteRequestBuilder = client.delete().startRowkey("rowkey").startTime(1479369296000L).endTime(1479369296000L + DateUtils.MILLIS_PER_DAY).pageSize(Integer.MAX_VALUE).filterIfMissing(true).
+ metricName("metric").treeAgg(false).byId(ids);
+ Field deleteRequestField = DeleteRequestBuilder.class.getDeclaredField("request");
+ Field idsField = DeleteRequestBuilder.class.getDeclaredField("deleteIds");
+ deleteRequestField.setAccessible(true);
+ idsField.setAccessible(true);
+ EagleServiceSingleEntityQueryRequest request = (EagleServiceSingleEntityQueryRequest)deleteRequestField.get(deleteRequestBuilder);
+ List<String> deleteIds = (List<String>)idsField.get(deleteRequestBuilder);
+ Assert.assertEquals("query=null&startRowkey=rowkey&pageSize=2147483647&startTime=2016-11-17%2007:54:56&endTime=2016-11-18%2007:54:56&treeAgg=false&metricName=metric&filterIfMissing=true", request.getQueryParameterString());
+ Assert.assertEquals("[id1, id2, id3]", deleteIds.toString());
+ }
+
+ @Test
+ public void testDeleteRequestByQuery() throws Exception {
+ EagleServiceConnector connector = mock(EagleServiceConnector.class);
+ IEagleServiceClient client = new EagleServiceClientImpl(connector);
+ DeleteRequestBuilder deleteRequestBuilder = client.delete().byQuery("AuditService");
+ Field deleteRequestField = DeleteRequestBuilder.class.getDeclaredField("request");
+ deleteRequestField.setAccessible(true);
+ EagleServiceSingleEntityQueryRequest request = (EagleServiceSingleEntityQueryRequest)deleteRequestField.get(deleteRequestBuilder);
+ Assert.assertEquals("query=AuditService&pageSize=0&treeAgg=false", request.getQueryParameterString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceClient.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceClient.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceClient.java
new file mode 100644
index 0000000..8dc7d3c
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceClient.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.log.entity.test.TestEntity;
+import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
+import org.apache.eagle.service.client.EagleServiceClientException;
+import org.apache.eagle.service.client.EagleServiceConnector;
+import org.junit.Test;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * @Since 11/16/16.
+ */
+public class TestEagleServiceClient {
+
+ @Test
+ public void testBuildBathPath() {
+ EagleServiceBaseClient eagleServiceBaseClient = new EagleServiceClientImpl("localhost", 9090, "admin", "secret");
+ String expectedPath = "http://localhost:9090/rest";
+ Assert.assertEquals(expectedPath, eagleServiceBaseClient.buildBathPath().toString());
+ }
+
+ @Test
+ public void testMarshall() throws IOException {
+ List<String> ids = new ArrayList<>();
+ ids.add("eagle001");
+ ids.add("eagle002");
+ ids.add("eagle003");
+ String expectedJson = "[\"eagle001\",\"eagle002\",\"eagle003\"]";
+ Assert.assertEquals(expectedJson, EagleServiceBaseClient.marshall(ids));
+ System.out.println(EagleServiceBaseClient.marshall(ids));
+ }
+
+ @Test(expected = EagleServiceClientException.class)
+ public void testGroupEntitiesByServiceException() throws EagleServiceClientException {
+ List<TaggedLogAPIEntity> entities = new ArrayList<>();
+ TaggedLogAPIEntity testEntity1 = null;
+ TaggedLogAPIEntity testEntity2 = new TestEntity();
+ entities.add(testEntity1);
+ entities.add(testEntity2);
+ EagleServiceBaseClient eagleServiceBaseClient = new EagleServiceClientImpl("localhost", 9090, "admin", "secret");
+ eagleServiceBaseClient.groupEntitiesByService(entities);
+ }
+
+ @Test
+ public void testGroupEntitiesByService() throws EagleServiceClientException {
+ List<TaggedLogAPIEntity> entities = new ArrayList<>();
+ TaggedLogAPIEntity testEntity1 = new TestTimeSeriesAPIEntity();
+ entities.add(testEntity1);
+ EagleServiceBaseClient eagleServiceBaseClient = new EagleServiceClientImpl("localhost", 9090, "admin", "secret");
+ Map<String,List<TaggedLogAPIEntity>> serviceEntityMap = eagleServiceBaseClient.groupEntitiesByService(entities);
+ System.out.println(serviceEntityMap);
+ Set<String> keySet = serviceEntityMap.keySet();
+ for(Map.Entry<String, List<TaggedLogAPIEntity>> entry : serviceEntityMap.entrySet()) {
+ Assert.assertEquals("TestTimeSeriesAPIEntity", entry.getKey());
+ Assert.assertEquals(1, entry.getValue().size());
+ }
+ }
+
+ @Test
+ public void testGetServiceNameByService() throws EagleServiceClientException {
+ EagleServiceConnector connector = mock(EagleServiceConnector.class);
+ EagleServiceBaseClient client = new EagleServiceClientImpl(connector);
+ String serviceName = client.getServiceNameByClass(TestTimeSeriesAPIEntity.class);
+ Assert.assertEquals("TestTimeSeriesAPIEntity", serviceName);
+ }
+
+ @Test(expected = EagleServiceClientException.class)
+ public void testGetServiceNameByServiceException() throws EagleServiceClientException {
+ EagleServiceConnector connector = mock(EagleServiceConnector.class);
+ EagleServiceBaseClient client = new EagleServiceClientImpl(connector);
+ String serviceName = client.getServiceNameByClass(TestEntity.class);
+ System.out.println(serviceName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceGroupByQueryRequest.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceGroupByQueryRequest.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceGroupByQueryRequest.java
new file mode 100644
index 0000000..d229fb6
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceGroupByQueryRequest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.eagle.service.client.EagleServiceClientException;
+import org.apache.eagle.service.client.EagleServiceGroupByQueryRequest;
+import org.junit.Test;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * @Since 11/18/16.
+ */
+public class TestEagleServiceGroupByQueryRequest {
+
+ @Test
+ public void testGetQueryString() throws EagleServiceClientException {
+ EagleServiceGroupByQueryRequest request = new EagleServiceGroupByQueryRequest();
+ request.setStartRowkey("rowkey");
+ request.setStartTime("2016-11-18 00:00:00 000");
+ request.setEndTime("2016-11-18 11:11:11 000");
+ request.setMetricName("metric");
+ request.setFilter("filter");
+ request.setPageSize(Integer.MAX_VALUE);
+ request.setIntervalMin(10);
+ List<String> groupBys = new ArrayList<>();
+ groupBys.add("field1");
+ groupBys.add("field2");
+ request.setGroupBys(groupBys);
+ List<String> orderBys = new ArrayList<>();
+ orderBys.add("field3");
+ orderBys.add("field3");
+ request.setOrderBys(orderBys);
+ List<String> returns = new ArrayList<>();
+ returns.add("field5");
+ returns.add("field6");
+ request.setReturns(returns);
+ Assert.assertEquals("query=AuditService%5Bfilter%5D%3C%40field1%2C%40field2%3E%7Bfield5%2Cfield6%7D.%7Bfield3%2Cfield3%7D&startRowkey=rowkey&pageSize=2147483647&startTime=2016-11-18+00%3A00%3A00+000&endTime=2016-11-18+11%3A11%3A11+000&metricName=metric&timeSeries=true&intervalmin=10", request.getQueryParameterString("AuditService"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceQueryBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceQueryBuilder.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceQueryBuilder.java
new file mode 100644
index 0000000..8da2537
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestEagleServiceQueryBuilder.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.commons.lang3.time.DateUtils;
+import org.apache.eagle.service.client.EagleServiceQueryBuilder;
+import org.apache.eagle.service.client.EagleServiceQueryRequest;
+import org.junit.Test;
+
+/**
+ * @Since 11/17/16.
+ */
+public class TestEagleServiceQueryBuilder {
+
+ @Test
+ public void testEagleQuery() throws Exception {
+ EagleServiceQueryBuilder builder = new EagleServiceQueryBuilder();
+ builder.setStartTime(0).setEndTime(DateUtils.MILLIS_PER_DAY).setPageSize(Integer.MAX_VALUE).addReturnField("field1").addReturnField("field2")
+ .addReturnTag("tag1").addSearchTag("tagKey", "tagValue");
+ EagleServiceQueryRequest request = builder.buildRequest();
+ Assert.assertEquals("pageSize=2147483647&startTime=1970-01-01%2000:00:00&endTime=1970-01-02%2000:00:00&tagNameValue=tagKey%3DtagValue&outputTag=tag1&outputField=field1&outputField=field2", request.getQueryParameterString());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestListQueryAPIReponseConvertHelper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestListQueryAPIReponseConvertHelper.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestListQueryAPIReponseConvertHelper.java
new file mode 100644
index 0000000..0fc5d2c
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestListQueryAPIReponseConvertHelper.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.eagle.log.entity.ListQueryAPIResponseEntity;
+import org.apache.eagle.log.entity.test.TestEntity;
+import org.apache.eagle.service.client.ListQueryAPIResponseConvertHelper;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * @Since 11/16/16.
+ */
+public class TestListQueryAPIReponseConvertHelper {
+
+ @Test
+ public void testConvert() throws Exception {
+ Map<String, Object> objectMap = new HashMap<>();
+ objectMap.put("prefix", "eagle");
+ objectMap.put("timestamp", 1479264382L);
+ objectMap.put("encodedRowkey", "rowkey");
+ Map<String, String> tags = new HashMap<>();
+ tags.put("field1", "value1");
+ objectMap.put("tags", tags);
+ objectMap.put("remediationID", "ID");
+ objectMap.put("remediationStatus", "status");
+ objectMap.put("count", 123456789L);
+ objectMap.put("numHosts", 9);
+ objectMap.put("numClusters", 123456789L);
+ List<Map<String, Object>> mapCollection = new ArrayList<Map<String, Object>>();
+ mapCollection.add(objectMap);
+ ListQueryAPIResponseEntity listQueryAPIResponseEntity = new ListQueryAPIResponseEntity();
+ listQueryAPIResponseEntity.setObj(mapCollection);
+ ListQueryAPIResponseConvertHelper listQueryAPIResponseConvertHelper = new ListQueryAPIResponseConvertHelper();
+ listQueryAPIResponseEntity = listQueryAPIResponseConvertHelper.convert(TestEntity.class, listQueryAPIResponseEntity);
+ List<TestEntity> objs = (List<TestEntity>) listQueryAPIResponseEntity.getObj();
+ TestEntity entity = objs.get(0);
+ Assert.assertEquals("eagle", entity.getPrefix());
+ Assert.assertEquals(1479264382L, entity.getTimestamp());
+ Assert.assertEquals("rowkey", entity.getEncodedRowkey());
+ Assert.assertEquals("ID", entity.getRemediationID());
+ Assert.assertEquals("status", entity.getRemediationStatus());
+ Assert.assertEquals(1, entity.getTags().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestQuery.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestQuery.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestQuery.java
new file mode 100644
index 0000000..6365494
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestQuery.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.eagle.service.client.EagleServiceConnector;
+import org.apache.eagle.service.client.EagleServiceSingleEntityQueryRequest;
+import org.apache.eagle.service.client.IEagleServiceClient;
+import org.junit.Test;
+import static org.mockito.Mockito.*;
+
+import java.lang.reflect.Field;
+
+/**
+ * @Since 11/15/16.
+ */
+public class TestQuery {
+
+ @Test
+ public void testQuery() throws Exception {
+ EagleServiceConnector connector = mock(EagleServiceConnector.class);
+ IEagleServiceClient client = new EagleServiceClientImpl(connector);
+ String site = "sandbox";
+ String query = "AuditService" + "[@serviceName=\"AlertDataSourceService\" AND @site=\"" + site + "\"]{*}";
+ SearchRequestBuilder searchRequestBuilder = client.search().startRowkey("rowkey").startTime(0).endTime(10 * DateUtils.MILLIS_PER_DAY).pageSize(Integer.MAX_VALUE).filterIfMissing(true).metricName("metric").query(query);
+ Field requestField = SearchRequestBuilder.class.getDeclaredField("request");
+ requestField.setAccessible(true);
+ EagleServiceSingleEntityQueryRequest request = (EagleServiceSingleEntityQueryRequest) requestField.get(searchRequestBuilder);
+ String expected = "query=AuditService%5B%40serviceName%3D%22AlertDataSourceService%22+AND+%40site%3D%22sandbox%22%5D%7B*%7D&startRowkey=rowkey&pageSize=2147483647&startTime=1970-01-01%2000:00:00&endTime=1970-01-11%2000:00:00&treeAgg=false&metricName=metric&filterIfMissing=true";
+ Assert.assertEquals(expected, request.getQueryParameterString());
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/620959bb/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestRowkeyQueryAPIResponseConvertHelper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestRowkeyQueryAPIResponseConvertHelper.java b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestRowkeyQueryAPIResponseConvertHelper.java
new file mode 100644
index 0000000..9511181
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-client-base/src/test/java/org/apache/eagle/service/client/impl/TestRowkeyQueryAPIResponseConvertHelper.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.client.impl;
+
+import junit.framework.Assert;
+import org.apache.eagle.log.entity.RowkeyQueryAPIResponseEntity;
+import org.apache.eagle.log.entity.test.TestEntity;
+import org.apache.eagle.service.client.RowkeyQueryAPIResponseConvertHelper;
+import org.junit.Test;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * @Since 11/17/16.
+ */
+public class TestRowkeyQueryAPIResponseConvertHelper {
+ @Test
+ public void testConvert() throws Exception {
+ Map<String, Object> objectMap = new HashMap<>();
+ objectMap.put("prefix", "eagle");
+ objectMap.put("timestamp", 1479264382L);
+ objectMap.put("encodedRowkey", "rowkey");
+ Map<String, String> tags = new HashMap<>();
+ tags.put("field1", "value1");
+ objectMap.put("tags", tags);
+ objectMap.put("remediationID", "ID");
+ objectMap.put("remediationStatus", "status");
+ objectMap.put("count", 123456789L);
+ objectMap.put("numHosts", 9);
+ objectMap.put("numClusters", 123456789L);
+ RowkeyQueryAPIResponseEntity rowkeyQueryAPIResponseEntity = new RowkeyQueryAPIResponseEntity();
+ rowkeyQueryAPIResponseEntity.setObj(objectMap);
+ RowkeyQueryAPIResponseConvertHelper rowkeyQueryAPIResponseConvertHelper = new RowkeyQueryAPIResponseConvertHelper();
+ rowkeyQueryAPIResponseEntity = rowkeyQueryAPIResponseConvertHelper.convert(TestEntity.class, rowkeyQueryAPIResponseEntity);
+ TestEntity entity = (TestEntity)rowkeyQueryAPIResponseEntity.getObj();
+ Assert.assertEquals("eagle", entity.getPrefix());
+ Assert.assertEquals(1479264382L, entity.getTimestamp());
+ Assert.assertEquals("rowkey", entity.getEncodedRowkey());
+ Assert.assertEquals("ID", entity.getRemediationID());
+ Assert.assertEquals("status", entity.getRemediationStatus());
+ Assert.assertEquals(1, entity.getTags().size());
+ }
+}
[6/6] incubator-eagle git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-eagle
Posted by ha...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-eagle
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/30b127b8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/30b127b8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/30b127b8
Branch: refs/heads/master
Commit: 30b127b8db310201acae1dd7801c14a8c8e62e78
Parents: 229d6b3 a267e73
Author: Hao Chen <ha...@apache.org>
Authored: Tue Nov 22 15:18:19 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Nov 22 15:18:19 2016 +0800
----------------------------------------------------------------------
.../eagle/topology/TopologyCheckAppConfig.java | 9 ++--
.../hbase/HbaseTopologyEntityParser.java | 4 +-
....eagle.topology.TopologyCheckAppProvider.xml | 52 ++++++++------------
.../src/main/resources/application.conf | 9 ++--
4 files changed, 33 insertions(+), 41 deletions(-)
----------------------------------------------------------------------