You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ha...@apache.org on 2016/03/22 08:08:40 UTC
incubator-eagle git commit: EAGLE-204 Support jdbc groupBy, timeseries,
aggregation
Repository: incubator-eagle
Updated Branches:
refs/heads/master c4bdb63f8 -> 0605b34fa
EAGLE-204 Support jdbc groupBy, timeseries, aggregation
https://issues.apache.org/jira/browse/EAGLE-204
Support timeseries metric query in front-end
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/0605b34f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/0605b34f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/0605b34f
Branch: refs/heads/master
Commit: 0605b34faaff218a847bb8b2b7ea9e2ae114b32f
Parents: c4bdb63
Author: Hao Chen <ha...@apache.org>
Authored: Tue Mar 22 14:45:54 2016 +0800
Committer: Hao Chen <ha...@apache.org>
Committed: Tue Mar 22 15:07:51 2016 +0800
----------------------------------------------------------------------
.../timeseries/AbstractAggregator.java | 8 ++
.../eagle/storage/operation/CompiledQuery.java | 14 ++
.../apache/eagle/storage/jdbc/JdbcStorage.java | 1 +
.../criteria/impl/QueryCriteriaBuilder.java | 48 ++++---
.../jdbc/entity/JdbcEntitySerDeserHelper.java | 4 +-
.../jdbc/entity/impl/JdbcEntityReaderImpl.java | 26 +++-
.../jdbc/schema/JdbcEntityDefinition.java | 42 +++++-
.../schema/JdbcEntityDefinitionManager.java | 7 +-
.../schema/serializer/MetricJdbcSerDeser.java | 56 ++++++++
.../eagle/storage/jdbc/JdbcStorageTestBase.java | 46 +++++++
.../storage/jdbc/TestGenericMetricStorage.java | 129 +++++++++++++++++++
.../eagle/storage/jdbc/TestJdbcStorage.java | 23 +---
.../src/main/resources/application.conf | 7 +-
.../app/public/feature/common/controller.js | 2 +-
.../src/main/webapp/app/public/js/app.config.js | 4 +-
15 files changed, 366 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java b/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
index 0837382..deb0838 100755
--- a/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
+++ b/eagle-core/eagle-query/eagle-query-base/src/main/java/org/apache/eagle/query/aggregate/timeseries/AbstractAggregator.java
@@ -178,6 +178,14 @@ public abstract class AbstractAggregator implements Aggregator, EntityCreationLi
return new Double(0.0);
}
}
+ if(obj instanceof double[]){
+ double[] value = (double[]) obj;
+ if(value.length > 0){
+ return new Double(value[0]);
+ }else{
+ return new Double(0.0);
+ }
+ }
throw new IllegalAggregateFieldTypeException(obj.getClass().toString() + " type is not support. The aggregated field must be numeric type, int, long or double");
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
index eacdcca..34761cb 100644
--- a/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
+++ b/eagle-core/eagle-query/eagle-storage-base/src/main/java/org/apache/eagle/storage/operation/CompiledQuery.java
@@ -41,6 +41,7 @@ import java.util.Set;
*/
public class CompiledQuery {
private final static Logger LOG = LoggerFactory.getLogger(CompiledQuery.class);
+ private boolean timeSeries;
public boolean isHasAgg() {
return hasAgg;
@@ -202,6 +203,7 @@ public class CompiledQuery {
this.sortFunctions = compiler.sortFunctions();
this.groupByFields = compiler.groupbyFields();
this.aggregateFields = compiler.aggregateFields();
+ this.timeSeries = this.getRawQuery().isTimeSeries();
final List<String[]> partitionValues = compiler.getQueryPartitionValues();
if (partitionValues != null) {
@@ -255,4 +257,16 @@ public class CompiledQuery {
}
}
}
+
+ public boolean isTimeSeries() {
+ return timeSeries;
+ }
+
+ public void setTimeSeries(boolean timeSeries) {
+ this.timeSeries = timeSeries;
+ }
+
+ public long getIntervalMin(){
+ return this.getRawQuery().getIntervalmin();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
index c490dcd..6dedbf4 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/JdbcStorage.java
@@ -18,6 +18,7 @@ package org.apache.eagle.storage.jdbc;
import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.query.aggregate.timeseries.TimeSeriesAggregator;
import org.apache.eagle.storage.DataStorageBase;
import org.apache.eagle.storage.jdbc.conn.ConnectionManagerFactory;
import org.apache.eagle.storage.jdbc.entity.JdbcEntityDeleter;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
index a82d185..9687cad 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/criteria/impl/QueryCriteriaBuilder.java
@@ -58,17 +58,32 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
Criteria root = new Criteria();
SearchCondition searchCondition = query.getSearchCondition();
- // SELECT
+
+
if(query.isHasAgg()){
- List<String> aggFields = query.getAggregateFields();
- List<AggregateFunctionType> aggFuncs = query.getAggregateFunctionTypes();
- for(int i=0;i<aggFuncs.size();i++){
- AggregateFunctionType aggFunc = aggFuncs.get(i);
- String aggField = aggFields.get(i);
- if(aggFunc.equals(AggregateFunctionType.count)){
- root.addSelectColumn(new ColumnImpl(aggFunc.name()+"(*)"));
- }else{
- root.addSelectColumn(new ColumnImpl(String.format("%s(%s.%s)",aggFunc.name(),this.tableName,aggField)));
+ if(this.jdbcEntityDefinition.getInternal().isTimeSeries() && query.isTimeSeries()) {
+ // SELECT
+ root.addSelectColumn(new ColumnImpl(this.tableName,JdbcConstants.TIMESTAMP_COLUMN_NAME));
+ List<String> aggFields = query.getAggregateFields();
+ for(String field:aggFields){
+ root.addSelectColumn(new ColumnImpl(this.tableName,field));
+ }
+
+ List<String> groupByFieldsFields = query.getGroupByFields();
+ for(String field:groupByFieldsFields){
+ root.addSelectColumn(new ColumnImpl(this.tableName,field));
+ }
+ } else {
+ List<String> aggFields = query.getAggregateFields();
+ List<AggregateFunctionType> aggFuncs = query.getAggregateFunctionTypes();
+ for (int i = 0; i < aggFuncs.size(); i++) {
+ AggregateFunctionType aggFunc = aggFuncs.get(i);
+ String aggField = aggFields.get(i);
+ if (aggFunc.equals(AggregateFunctionType.count)) {
+ root.addSelectColumn(new ColumnImpl(aggFunc.name() + "(*)"));
+ } else {
+ root.addSelectColumn(new ColumnImpl(String.format("%s(%s.%s)", aggFunc.name(), this.tableName, aggField)));
+ }
}
}
} else if(searchCondition.isOutputAll()){
@@ -88,12 +103,6 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
}
}
- // If no columns are specified, then select * by default
- if(root.getSelectColumns() == null || root.getSelectColumns().size() ==0){
- // SELECT *
- root.addSelectColumn(new ColumnImpl(this.tableName, "*"));
- }
-
// FROM $tableName
root.addFrom(this.tableName);
@@ -119,7 +128,7 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
}
// TODO: GROUP BY
- if(query.isHasAgg()){
+ if(query.isHasAgg() && !query.isTimeSeries()){
for(String groupByField:query.getGroupByFields()){
root.addGroupByColumn(new ColumnImpl(this.tableName,groupByField));
}
@@ -127,6 +136,11 @@ public class QueryCriteriaBuilder implements CriteriaBuilder {
// TODO: ORDER BY
+ // If no columns are specified, then select * by default
+ if(root.getSelectColumns() == null || root.getSelectColumns().size() ==0){
+ // SELECT *
+ root.addSelectColumn(new ColumnImpl(this.tableName, "*"));
+ }
return root;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
index 3750f71..a4816dd 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/JdbcEntitySerDeserHelper.java
@@ -22,7 +22,6 @@ import org.apache.eagle.log.entity.meta.EntityDefinition;
import org.apache.eagle.log.entity.meta.Qualifier;
import org.apache.eagle.storage.jdbc.JdbcConstants;
import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinition;
-import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinitionManager;
import org.apache.eagle.storage.jdbc.schema.serializer.JdbcSerDeser;
import org.apache.commons.beanutils.PropertyUtils;
import org.apache.torque.ColumnImpl;
@@ -211,6 +210,7 @@ public class JdbcEntitySerDeserHelper {
throw new IOException(String.format("No field %s in entity %s", columnName, entityDefinition.getInternal().getEntityClass()), e);
}
}else{
+ columnName = entityDefinition.getOriginalJavaTagName(columnName);
// treat as tag or others
value = resultSet.getObject(columnName);
}
@@ -251,7 +251,7 @@ public class JdbcEntitySerDeserHelper {
Object fieldValue = getMethod.invoke(entity);
Class<?> fieldType = qualifier.getSerDeser().type();
- JdbcSerDeser jdbcSerDeser = JdbcEntityDefinitionManager.getJdbcSerDeser(fieldType);
+ JdbcSerDeser jdbcSerDeser = jdbcEntityDefinition.getJdbcSerDeser(displayName);
JdbcTypedValue jdbcTypedValue = jdbcSerDeser.toJdbcTypedValue(fieldValue, fieldType, qualifier);
columnValues.put(new ColumnImpl(tableName,displayName),jdbcTypedValue);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
index a7b93b5..5e6ca9a 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/entity/impl/JdbcEntityReaderImpl.java
@@ -16,6 +16,9 @@
*/
package org.apache.eagle.storage.jdbc.entity.impl;
+import com.google.common.collect.Lists;
+import org.apache.eagle.log.base.taggedlog.TaggedLogAPIEntity;
+import org.apache.eagle.query.aggregate.timeseries.TimeSeriesAggregator;
import org.apache.eagle.storage.jdbc.conn.ConnectionManagerFactory;
import org.apache.eagle.storage.jdbc.conn.impl.TorqueStatementPeerImpl;
import org.apache.eagle.storage.jdbc.criteria.impl.PrimaryKeyCriteriaBuilder;
@@ -32,7 +35,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
/**
* @since 3/27/15
@@ -57,7 +62,7 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader {
if(LOG.isDebugEnabled()) LOG.debug("Querying: " + displaySql);
RecordMapper<E> recordMapper;
- if(query.isHasAgg()) {
+ if(query.isHasAgg() && !query.isTimeSeries()) {
recordMapper = (RecordMapper<E>) new AggreagteRecordMapper(query, jdbcEntityDefinition);
}else{
recordMapper = new EntityRecordMapper(jdbcEntityDefinition);
@@ -69,6 +74,9 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader {
TorqueStatementPeerImpl peer = ConnectionManagerFactory.getInstance().getStatementExecutor();
result = peer.delegate().doSelect(criteria, recordMapper);
LOG.info(String.format("Read %s records in %s ms (sql: %s)",result.size(),stopWatch.getTime(),displaySql));
+ if(result.size() > 0 && query.isTimeSeries()){
+ result = Lists.newArrayList((E) timeseriesAggregate(result, query));
+ }
}catch (Exception ex){
LOG.error("Failed to query by: "+displaySql+", due to: "+ex.getMessage(),ex);
throw new IOException("Failed to query by: "+displaySql,ex);
@@ -78,6 +86,22 @@ public class JdbcEntityReaderImpl implements JdbcEntityReader {
return result;
}
+ private <E> Map timeseriesAggregate(List<E> result, CompiledQuery query) throws Exception {
+ TimeSeriesAggregator aggregator = new TimeSeriesAggregator(query.getGroupByFields(),
+ query.getAggregateFunctionTypes(),
+ query.getAggregateFields(),
+ query.getStartTime(), query.getEndTime(),
+ query.getIntervalMin()
+ );
+ for(E entity: result)
+ aggregator.accumulate((TaggedLogAPIEntity) entity);
+ if(this.jdbcEntityDefinition.isGenericMetric()) {
+ return aggregator.getMetric();
+ } else {
+ return aggregator.result();
+ }
+ }
+
@Override
public <E> List<E> query(List<String> ids) throws Exception {
PrimaryKeyCriteriaBuilder criteriaBuilder = new PrimaryKeyCriteriaBuilder(ids,this.jdbcEntityDefinition.getJdbcTableName());
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
index eb5e874..aa95934 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinition.java
@@ -48,7 +48,7 @@ public class JdbcEntityDefinition {
* @return jdbc table name in lowercase
*/
public String getJdbcTableName(){
- if(this.internal.getService().equals(GenericMetricEntity.GENERIC_METRIC_SERVICE)){
+ if(isGenericMetric()){
return this.internal.getTable().toLowerCase();
}else {
return String.format("%s_%s", this.internal.getTable(),this.internal.getPrefix()).toLowerCase();
@@ -56,11 +56,11 @@ public class JdbcEntityDefinition {
}
public Class<?> getColumnType(String fieldName) throws NoSuchFieldException {
- if (fieldName.equals(JdbcConstants.TIMESTAMP_COLUMN_NAME)){
+ if (fieldName.equalsIgnoreCase(JdbcConstants.TIMESTAMP_COLUMN_NAME)){
return Long.class;
- }else if(fieldName.equals(JdbcConstants.ROW_KEY_COLUMN_NAME)) {
+ }else if(fieldName.equalsIgnoreCase(JdbcConstants.ROW_KEY_COLUMN_NAME)) {
return String.class;
- }else if(fieldName.equals(JdbcConstants.METRIC_NAME_COLUMN_NAME)){
+ }else if(fieldName.equalsIgnoreCase(JdbcConstants.METRIC_NAME_COLUMN_NAME)){
return String.class;
}
for(String realField:internal.getDisplayNameMap().keySet()){
@@ -80,6 +80,9 @@ public class JdbcEntityDefinition {
}
public Integer getJdbcColumnTypeCodeOrNull(String fieldName){
+ if(this.isGenericMetric() && GenericMetricEntity.VALUE_FIELD.equalsIgnoreCase(fieldName)){
+ return JdbcEntityDefinitionManager.getJdbcType(double.class);
+ }
Class<?> columnType;
try {
columnType = getColumnType(fieldName);
@@ -91,6 +94,10 @@ public class JdbcEntityDefinition {
@SuppressWarnings("unchecked")
public JdbcSerDeser getJdbcSerDeser(String columnName) {
+ if(this.isGenericMetric() && GenericMetricEntity.VALUE_FIELD.equalsIgnoreCase(columnName)){
+ return JdbcEntityDefinitionManager.METRIC_JDBC_SERDESER;
+ }
+
Qualifier qualifier = this.getColumnQualifier(columnName);
if(qualifier == null){
return JdbcEntityDefinitionManager.DEFAULT_JDBC_SERDESER;
@@ -112,6 +119,33 @@ public class JdbcEntityDefinition {
return false;
}
+ /**
+ * TODO: Optimize with hashmap
+ */
+ public String getJavaEntityFieldName(String jdbcColumnName){
+ for(String javaEntityFieldName:this.internal.getDisplayNameMap().keySet()){
+ if(javaEntityFieldName.equalsIgnoreCase(jdbcColumnName)){
+ return javaEntityFieldName;
+ }
+ }
+ throw new IllegalArgumentException("Can't map jdbc column '"+jdbcColumnName+"' with entity: "+this.getInternal().getEntityClass());
+ }
+
+ /**
+ * TODO: Optimize with hashmap
+ *
+ * @param jdbcTagName
+ * @return
+ */
+ public String getOriginalJavaTagName(String jdbcTagName){
+ for(String tag:this.getInternal().getTags()){
+ if(tag.equalsIgnoreCase(jdbcTagName)){
+ return tag;
+ }
+ }
+ return jdbcTagName.toLowerCase();
+ }
+
public Qualifier getColumnQualifier(String columnName) {
for(Map.Entry<String,Qualifier> entry:this.internal.getDisplayNameMap().entrySet()){
if(entry.getKey().equalsIgnoreCase(columnName)){
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
index cf163fb..418f552 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/JdbcEntityDefinitionManager.java
@@ -22,6 +22,7 @@ import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
import org.apache.eagle.log.entity.meta.EntitySerDeser;
import org.apache.eagle.storage.jdbc.schema.serializer.JdbcSerDeser;
import org.apache.eagle.storage.jdbc.schema.serializer.DefaultJdbcSerDeser;
+import org.apache.eagle.storage.jdbc.schema.serializer.MetricJdbcSerDeser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -85,7 +86,8 @@ public class JdbcEntityDefinitionManager {
checkInit();
}
- public static DefaultJdbcSerDeser DEFAULT_JDBC_SERDESER = new DefaultJdbcSerDeser();
+ public static JdbcSerDeser DEFAULT_JDBC_SERDESER = new DefaultJdbcSerDeser();
+ public static JdbcSerDeser METRIC_JDBC_SERDESER = new MetricJdbcSerDeser();
private final static Map<Class<?>,JdbcSerDeser> _columnTypeSerDeserMapping = new HashMap<Class<?>, JdbcSerDeser>();
/**
@@ -124,7 +126,8 @@ public class JdbcEntityDefinitionManager {
if(fieldType == null){
return Types.NULL;
} else if(!_classJdbcType.containsKey(fieldType)){
- LOG.debug("Unable to locate simple jdbc type for: {}, return type as JAVA_OBJECT",fieldType);
+ if(LOG.isDebugEnabled())
+ LOG.debug("Unable to locate simple jdbc type for: {}, return type as JAVA_OBJECT",fieldType);
return Types.JAVA_OBJECT;
}
return _classJdbcType.get(fieldType);
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
new file mode 100644
index 0000000..8162955
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/schema/serializer/MetricJdbcSerDeser.java
@@ -0,0 +1,56 @@
+package org.apache.eagle.storage.jdbc.schema.serializer;
+
+import org.apache.eagle.log.entity.meta.Qualifier;
+import org.apache.eagle.storage.jdbc.JdbcConstants;
+import org.apache.eagle.storage.jdbc.schema.JdbcEntityDefinitionManager;
+import org.apache.torque.util.JdbcTypedValue;
+
+import java.io.IOException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Types;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class MetricJdbcSerDeser implements JdbcSerDeser<double[]> {
+ @Override
+ public double[] toJavaTypedValue(ResultSet result, Class<?> fieldType, String fieldName, Qualifier qualifier) throws IOException {
+ try {
+ return new double[]{result.getDouble(fieldName)};
+ } catch (SQLException e) {
+ throw new IOException("Generic Metric Field: "+fieldName+", java type:"+fieldType,e);
+ }
+ }
+
+ /**
+ *
+ * @param fieldValue
+ * @param fieldType
+ * @return
+ */
+ @Override
+ public JdbcTypedValue toJdbcTypedValue(Object fieldValue, Class<?> fieldType, Qualifier qualifier) {
+ double[] metricFieldValues = (double[]) fieldValue;
+ if(metricFieldValues == null || metricFieldValues.length == 0){
+ return null;
+ } else if(metricFieldValues.length > 1){
+ throw new IllegalArgumentException("Not support metric value length > 1: "+fieldType);
+ }else{
+ return new JdbcTypedValue(metricFieldValues[0],Types.DOUBLE);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
new file mode 100644
index 0000000..4cee46f
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/JdbcStorageTestBase.java
@@ -0,0 +1,46 @@
+package org.apache.eagle.storage.jdbc;
+
+import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
+import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
+import org.apache.eagle.storage.DataStorageManager;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.GregorianCalendar;
+import java.util.TimeZone;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class JdbcStorageTestBase {
+ JdbcStorage storage;
+ long baseTimestamp;
+ final static Logger LOG = LoggerFactory.getLogger(TestJdbcStorage.class);
+
+ @Before
+ public void setUp() throws Exception {
+ storage = (JdbcStorage) DataStorageManager.getDataStorageByEagleConfig();
+ storage.init();
+ GregorianCalendar gc = new GregorianCalendar();
+ gc.clear();
+ gc.set(2014, 1, 6, 1, 40, 12);
+ gc.setTimeZone(TimeZone.getTimeZone("UTC"));
+ baseTimestamp = gc.getTime().getTime();
+ System.out.println("timestamp:" + baseTimestamp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
new file mode 100644
index 0000000..524b294
--- /dev/null
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestGenericMetricStorage.java
@@ -0,0 +1,129 @@
+package org.apache.eagle.storage.jdbc;
+
+import junit.framework.Assert;
+import org.apache.eagle.common.DateTimeUtil;
+import org.apache.eagle.log.entity.GenericMetricEntity;
+import org.apache.eagle.log.entity.meta.EntityDefinition;
+import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
+import org.apache.eagle.storage.exception.QueryCompileException;
+import org.apache.eagle.storage.operation.CompiledQuery;
+import org.apache.eagle.storage.operation.RawQuery;
+import org.apache.eagle.storage.result.ModifyResult;
+import org.apache.eagle.storage.result.QueryResult;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class TestGenericMetricStorage extends JdbcStorageTestBase {
+
+ EntityDefinition entityDefinition;
+ Random random = new Random();
+ String metricName = "unittest.metric.name";
+
+ @Override
+ public void setUp() throws Exception {
+ entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(GenericMetricEntity.class);
+ entityDefinition.setTags(new String[]{"site","application"});
+ super.setUp();
+ }
+
+ private GenericMetricEntity newMetric(){
+ GenericMetricEntity instance = new GenericMetricEntity();
+ instance.setPrefix(metricName);
+ instance.setTags(new HashMap<String, String>() {{
+ put("site", "unittest_site");
+ put("application", "unittest_application");
+ }});
+ instance.setValue(new double[]{random.nextDouble()});
+ instance.setTimestamp(System.currentTimeMillis());
+ return instance;
+ }
+
+ @Test
+ public void testWrite1000Metrics() throws InterruptedException, IOException {
+ // Write 1000 entities
+ List<GenericMetricEntity> entityList = new ArrayList<>();
+ int i= 0;
+ while( i++ < 1000){
+ entityList.add(newMetric());
+ Thread.sleep(1);
+ }
+ ModifyResult<String> result = storage.create(entityList, entityDefinition);
+ Assert.assertTrue(result.getSize() >= 1000);
+ }
+
+ @Test
+ public void testSimpleRead() throws IOException, QueryCompileException, InterruptedException {
+ // record insert init time
+ long startTime = System.currentTimeMillis();
+ // Write 1000 entities
+ testWrite1000Metrics();
+ // record insertion finish time
+ long endTime = System.currentTimeMillis();
+ // init read in time range [startTime, endTime)
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[]{*}");
+ rawQuery.setMetricName(metricName);
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime+1000));
+ rawQuery.setPageSize(10000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult queryResult = storage.query(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 1000);
+ }
+
+ @Test
+ public void testSimpleGroupAggregateRead() throws IOException, InterruptedException, QueryCompileException {
+ long startTime = System.currentTimeMillis();
+ testWrite1000Metrics();
+ long endTime = System.currentTimeMillis();
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[@site=\"unittest_site\" AND @application=\"unittest_application\"]<@site>{sum(value)}");
+ rawQuery.setMetricName(metricName);
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime+1000));
+ rawQuery.setPageSize(10000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult queryResult = storage.query(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 1);
+ }
+
+
+ @Test
+ public void testTimeSeriesGroupAggregateRead() throws IOException, InterruptedException, QueryCompileException {
+ long startTime = System.currentTimeMillis();
+ testWrite1000Metrics();
+ long endTime = System.currentTimeMillis();
+ RawQuery rawQuery = new RawQuery();
+ rawQuery.setQuery(GenericMetricEntity.GENERIC_METRIC_SERVICE+"[@site=\"unittest_site\" AND @application=\"unittest_application\"]<@site>{sum(value)}");
+ rawQuery.setMetricName(metricName);
+ rawQuery.setTimeSeries(true);
+ rawQuery.setIntervalmin(10);
+ rawQuery.setStartTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(startTime));
+ rawQuery.setEndTime(DateTimeUtil.millisecondsToHumanDateWithSeconds(endTime + 10*60*1000));
+ rawQuery.setPageSize(10000);
+ CompiledQuery query = new CompiledQuery(rawQuery);
+ QueryResult queryResult = storage.query(query, entityDefinition);
+ Assert.assertTrue(queryResult.getSize() >= 1);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
index 0b4178e..cb0f0a7 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/test/java/org/apache/eagle/storage/jdbc/TestJdbcStorage.java
@@ -22,38 +22,23 @@ import org.apache.eagle.common.DateTimeUtil;
import org.apache.eagle.log.entity.meta.EntityDefinition;
import org.apache.eagle.log.entity.meta.EntityDefinitionManager;
import org.apache.eagle.log.entity.test.TestTimeSeriesAPIEntity;
-import org.apache.eagle.storage.DataStorageManager;
import org.apache.eagle.storage.exception.QueryCompileException;
import org.apache.eagle.storage.operation.CompiledQuery;
import org.apache.eagle.storage.operation.RawQuery;
import org.apache.eagle.storage.result.ModifyResult;
import org.apache.eagle.storage.result.QueryResult;
-import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.*;
-public class TestJdbcStorage {
- JdbcStorage storage;
+public class TestJdbcStorage extends JdbcStorageTestBase {
EntityDefinition entityDefinition;
- long baseTimestamp;
- final static Logger LOG = LoggerFactory.getLogger(TestJdbcStorage.class);
-
- @Before
+ @Override
public void setUp() throws Exception {
entityDefinition = EntityDefinitionManager.getEntityDefinitionByEntityClass(TestTimeSeriesAPIEntity.class);
entityDefinition.setTags(new String[]{"cluster","datacenter","random"});
- storage = (JdbcStorage) DataStorageManager.getDataStorageByEagleConfig();
- storage.init();
- GregorianCalendar gc = new GregorianCalendar();
- gc.clear();
- gc.set(2014, 1, 6, 1, 40, 12);
- gc.setTimeZone(TimeZone.getTimeZone("UTC"));
- baseTimestamp = gc.getTime().getTime();
- System.out.println("timestamp:" + baseTimestamp);
+ super.setUp();
}
@Test
@@ -243,8 +228,10 @@ public class TestJdbcStorage {
return instance;
}
+
@Test
public void testInitSuccessfully() {
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/resources/application.conf b/eagle-webservice/src/main/resources/application.conf
index 23a7067..61a899f 100644
--- a/eagle-webservice/src/main/resources/application.conf
+++ b/eagle-webservice/src/main/resources/application.conf
@@ -16,14 +16,13 @@
eagle {
service {
storage-type="jdbc"
- storage-adapter="derby"
+ storage-adapter="mysql"
storage-username="eagle"
storage-password=eagle
storage-database=eagle
- # Derby database location: $TOMCAT_HOME/data/eagle
- storage-connection-url="jdbc:derby:../data/eagle;create=true"
+ storage-connection-url="jdbc:mysql://localhost:3306/eagle"
storage-connection-props="encoding=UTF-8"
- storage-driver-class="org.apache.derby.jdbc.EmbeddedDriver"
+ storage-driver-class="com.mysql.jdbc.Driver"
storage-connection-max=8
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js b/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
index d87ec6d..b758431 100644
--- a/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
+++ b/eagle-webservice/src/main/webapp/app/public/feature/common/controller.js
@@ -181,7 +181,7 @@
];
function _loadSeries(seriesName, metricName, condition) {
- var list = Entities.querySeries("GenericMetricService", $.extend({_metricName: metricName}, condition), "@cluster", "sum(value)", _intervalList[_intervalType][1]);
+ var list = Entities.querySeries("GenericMetricService", $.extend({_metricName: metricName}, condition), "@site", "sum(value)", _intervalList[_intervalType][1]);
var seriesList = nvd3.convert.eagle([list]);
if(!$scope[seriesName]) $scope[seriesName] = seriesList;
list._promise.then(function() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/0605b34f/eagle-webservice/src/main/webapp/app/public/js/app.config.js
----------------------------------------------------------------------
diff --git a/eagle-webservice/src/main/webapp/app/public/js/app.config.js b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
index 876114f..4392f71 100644
--- a/eagle-webservice/src/main/webapp/app/public/js/app.config.js
+++ b/eagle-webservice/src/main/webapp/app/public/js/app.config.js
@@ -32,8 +32,8 @@
deleteEntity: 'rest/entities/delete?serviceName=${serviceName}&byId=true',
deleteEntities: 'rest/entities?query=${serviceName}[${condition}]{*}&pageSize=100000',
- queryGroup: 'rest/list?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000',
- querySeries: 'rest/list?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000&timeSeries=true&intervalmin=${intervalmin}',
+ queryGroup: 'rest/entities?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000',
+ querySeries: 'rest/entities?query=${serviceName}[${condition}]<${groupBy}>{${values}}&pageSize=100000&timeSeries=true&intervalmin=${intervalmin}',
query: 'rest/',