You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by qi...@apache.org on 2017/05/19 05:55:35 UTC
eagle git commit: [EAGLE-1016] fix keyword "group" conflict in JDBC
storage
Repository: eagle
Updated Branches:
refs/heads/master 7d39863f4 -> af9b056dc
[EAGLE-1016] fix keyword "group" conflict in JDBC storage
https://issues.apache.org/jira/browse/EAGLE-1016
If you meet the same issue after applying this patch, please drop the old table first.
Author: Zhao, Qingwen <qi...@apache.org>
Closes #933 from qingwen220/EAGLE-1016.
Project: http://git-wip-us.apache.org/repos/asf/eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/eagle/commit/af9b056d
Tree: http://git-wip-us.apache.org/repos/asf/eagle/tree/af9b056d
Diff: http://git-wip-us.apache.org/repos/asf/eagle/diff/af9b056d
Branch: refs/heads/master
Commit: af9b056dc453f46c6a2aeb92d8dd6fe503f1b577
Parents: 7d39863
Author: Zhao, Qingwen <qi...@apache.org>
Authored: Fri May 19 13:55:27 2017 +0800
Committer: Zhao, Qingwen <qi...@apache.org>
Committed: Fri May 19 13:55:27 2017 +0800
----------------------------------------------------------------------
eagle-assembly/src/main/conf/eagle.conf | 26 +++++++++++---------
.../metadata/model/MetricSchemaEntity.java | 6 ++---
.../jdbc/conn/ConnectionConfigFactory.java | 22 +++++++++++------
.../system_metric_collector.py | 2 +-
.../scheduler/QueueStructureAPIEntity.java | 1 +
.../model/scheduler/RunningQueueAPIEntity.java | 1 +
.../mr/historyentity/JobExecutionAPIEntity.java | 1 +
.../eagle/metric/HadoopMetricMonitorApp.java | 5 ++--
.../app/dev/public/js/ctrls/metricCtrl.js | 2 +-
.../entity/HBaseServiceTopologyAPIEntity.java | 1 +
.../entity/HdfsServiceTopologyAPIEntity.java | 2 +-
.../entity/MRServiceTopologyAPIEntity.java | 1 +
.../entity/SystemServiceTopologyAPIEntity.java | 1 +
13 files changed, 43 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-assembly/src/main/conf/eagle.conf
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/conf/eagle.conf b/eagle-assembly/src/main/conf/eagle.conf
index 496e3d0..3441fe6 100644
--- a/eagle-assembly/src/main/conf/eagle.conf
+++ b/eagle-assembly/src/main/conf/eagle.conf
@@ -40,32 +40,34 @@ zookeeper {
storage {
# storage type: ["hbase","jdbc"]
- # default is "hbase"
+ # "hbase" is preferred
type = "hbase"
+ ## for prod env
hbase {
autoCreateTable = true
-
# hbase configuration: hbase.zookeeper.quorum
- # default is "localhost"
zookeeperQuorum = "localhost"
-
# hbase configuration: hbase.zookeeper.property.clientPort
- # default is 2181
zookeeperPropertyClientPort = 2181
-
# hbase configuration: zookeeper.znode.parent
- # default is "/hbase"
zookeeperZnodeParent = "/hbase-unsecure"
-
- # eagle web login profile: [sandbox, default]
- # default is sandbox
tableNamePrefixedWithEnvironment = false
-
# eagle coprocessor enabled or not: [true, false]
- # default is false
coprocessorEnabled = false
}
+
+ ## for dev env, in which JPM is NOT supported
+ jdbc {
+ adapter = "mysql"
+ username = "root"
+ password = ""
+ database = "eagle"
+ driverClass = "com.mysql.jdbc.Driver"
+ connectionUrl = "jdbc:mysql://localhost:3306/eagle"
+ connectionProps = "encoding=UTF-8"
+ connectionMax = 8
+ }
}
# ---------------------------------------------
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
index 3bd5825..36eef9d 100644
--- a/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
+++ b/eagle-core/eagle-metadata/eagle-metadata-base/src/main/java/org/apache/eagle/metadata/model/MetricSchemaEntity.java
@@ -30,12 +30,12 @@ import java.util.List;
@Service(MetricSchemaEntity.METRIC_SCHEMA_SERVICE)
@JsonIgnoreProperties(ignoreUnknown = true)
@TimeSeries(false)
-@Tags({"site","site","group"})
+@Tags({"site","metricName","metricGroup"})
public class MetricSchemaEntity extends TaggedLogAPIEntity {
static final String METRIC_SCHEMA_SERVICE = "MetricSchemaService";
- public static final String METRIC_NAME_TAG = "name";
public static final String METRIC_SITE_TAG = "site";
- public static final String METRIC_GROUP_TAG = "group";
+ public static final String METRIC_NAME_TAG = "metricName";
+ public static final String METRIC_GROUP_TAG = "metricGroup";
@Column("a")
private List<String> dimensionFields;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
index 71f19aa..bb9e5f1 100644
--- a/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
+++ b/eagle-core/eagle-query/eagle-storage-jdbc/src/main/java/org/apache/eagle/storage/jdbc/conn/ConnectionConfigFactory.java
@@ -32,21 +32,29 @@ public class ConnectionConfigFactory {
String username = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DB_USERNAME);
String password = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DB_PASSWORD);
String connUrl = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_URL);
- String connProps = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_PROPS);
String adapter = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_ADAPTER);
String databaseName = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DATABASE);
String driverClass = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_DRIVER_CLASS);
- String connMaxSize = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_MAX_SIZE);
+
+
+ String connProps = null;
+ if (EagleConfigFactory.load().getConfig().hasPath(JdbcConstants.EAGLE_CONN_PROPS)) {
+ connProps = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_PROPS);
+ }
+
+ String connMaxSize = null;
+ if (EagleConfigFactory.load().getConfig().hasPath(JdbcConstants.EAGLE_CONN_MAX_SIZE)) {
+ connMaxSize = EagleConfigFactory.load().getConfig().getString(JdbcConstants.EAGLE_CONN_MAX_SIZE);
+ }
ConnectionConfig config = new ConnectionConfig();
if(username != null) config.setUserName(username);
if(password != null)config.setPassword(password);
if(connUrl != null) config.setConnectionUrl(connUrl);
- if(connProps!=null) config.setConnectionProperties(connProps);
- if(adapter!=null) config.setAdapter(adapter);
- if(databaseName!=null) config.setDatabaseName(databaseName);
- if(driverClass!=null) config.setDriverClassName(driverClass);
-
+ if(connProps != null) config.setConnectionProperties(connProps);
+ if(adapter != null) config.setAdapter(adapter);
+ if(databaseName != null) config.setDatabaseName(databaseName);
+ if(driverClass != null) config.setDriverClassName(driverClass);
if(connMaxSize!=null){
config.setConnectionMaxActive(Integer.parseInt(connMaxSize));
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-external/hadoop_jmx_collector/system_metric_collector.py
----------------------------------------------------------------------
diff --git a/eagle-external/hadoop_jmx_collector/system_metric_collector.py b/eagle-external/hadoop_jmx_collector/system_metric_collector.py
index fba06f2..501798b 100644
--- a/eagle-external/hadoop_jmx_collector/system_metric_collector.py
+++ b/eagle-external/hadoop_jmx_collector/system_metric_collector.py
@@ -224,7 +224,7 @@ class SystemMetricCollector(MetricCollector):
output = os.popen('sudo ipmitool sdr | grep Temp | grep CPU').readlines()
for item in output:
items = re.split("^(CPU\d+)\sTemp\.\s+\|\s+(\d+|\d+\.\d+)\s", item.rstrip())
- event = self.new_metric("System.CPU")
+ event = self.new_metric("system.cpu")
event["timestamp"] = int(round(time.time() * 1000))
event["metric"] = DATA_TYPE + "." + 'cpu.temp'
event["value"] = items[2]
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
index 72f67bc..fde709d 100644
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/QueueStructureAPIEntity.java
@@ -31,6 +31,7 @@ import java.util.List;
@Service(HadoopClusterConstants.QUEUE_MAPPING_SERVICE_NAME)
@TimeSeries(false)
@Partition( {"site"})
+@Tags({HadoopClusterConstants.TAG_SITE, HadoopClusterConstants.TAG_QUEUE, HadoopClusterConstants.TAG_PARENT_QUEUE})
public class QueueStructureAPIEntity extends TaggedLogAPIEntity {
@Column("a")
private List<String> subQueues;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
index be4ef31..a93bad0 100755
--- a/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
+++ b/eagle-jpm/eagle-hadoop-queue/src/main/java/org/apache/eagle/hadoop/queue/model/scheduler/RunningQueueAPIEntity.java
@@ -32,6 +32,7 @@ import java.util.List;
@Service(HadoopClusterConstants.RUNNING_QUEUE_SERVICE_NAME)
@TimeSeries(true)
@Partition( {"site"})
+@Tags({HadoopClusterConstants.TAG_SITE, HadoopClusterConstants.TAG_QUEUE, HadoopClusterConstants.TAG_PARENT_QUEUE})
public class RunningQueueAPIEntity extends TaggedLogAPIEntity {
@Column("a")
private String state;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
index 746843d..7a649b7 100644
--- a/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
+++ b/eagle-jpm/eagle-jpm-entity/src/main/java/org/apache/eagle/jpm/mr/historyentity/JobExecutionAPIEntity.java
@@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@Index(name = "Index_1_jobId", columns = { "jobId" }, unique = true),
@Index(name = "Index_2_jobDefId", columns = { "jobDefId" }, unique = false)
})
+@Tags({"site", "jobId", "jobName", "jobDefId", "jobType", "user", "queue"})
public class JobExecutionAPIEntity extends JobBaseAPIEntity {
@Column("a")
private String currentState;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
----------------------------------------------------------------------
diff --git a/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java b/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
index aa1753f..1761aba 100644
--- a/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
+++ b/eagle-metric/eagle-hadoop-metric/src/main/java/org/apache/eagle/metric/HadoopMetricMonitorApp.java
@@ -51,15 +51,14 @@ public class HadoopMetricMonitorApp extends StormApplication {
.siteAs(AppConfigUtils.getSiteId(config))
.namedByField("metric")
.eventTimeByField("timestamp")
- .dimensionFields("host", "group", "site", "device")
+ .dimensionFields("host", "site", "device")
.granularity(Calendar.SECOND)
.valueField("value");
return environment.newApp(config)
.fromStream("HADOOP_JMX_METRIC_STREAM").transformBy(new CounterToRateFunction(hadoopMetricDescriptor,3, TimeUnit.SECONDS, ClockWithOffset.INSTANCE))
.saveAsMetric(hadoopMetricDescriptor)
.fromStream("SYSTEM_METRIC_STREAM").transformBy(new CounterToRateFunction(systemMetricDescriptor,3, TimeUnit.SECONDS, ClockWithOffset.INSTANCE))
- .saveAsMetric(systemMetricDescriptor
- )
+ .saveAsMetric(systemMetricDescriptor)
.toTopology();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
----------------------------------------------------------------------
diff --git a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
index b310738..e0f0f15 100644
--- a/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
+++ b/eagle-server/src/main/webapp/app/dev/public/js/ctrls/metricCtrl.js
@@ -40,7 +40,7 @@
$scope.metricList = [$scope.metricName];
CompatibleEntity.groups({
query: 'MetricSchemaService',
- groups: 'name',
+ groups: 'metricName',
fields: 'count',
limit: 9999,
})._promise.then(function (res) {
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
index 9d7233e..ce829ed 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HBaseServiceTopologyAPIEntity.java
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@Prefix("hbaseservicestatus")
@Service(TopologyConstants.HBASE_INSTANCE_SERVICE_NAME)
@TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG})
public class HBaseServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
@Column("a")
private String status;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
index 4054cba..1362736 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/HdfsServiceTopologyAPIEntity.java
@@ -22,13 +22,13 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.eagle.log.entity.meta.*;
import org.apache.eagle.topology.TopologyConstants;
-
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@Table("hadoop_topology")
@ColumnFamily("f")
@Prefix("hdfsservicestatus")
@Service(TopologyConstants.HDFS_INSTANCE_SERVICE_NAME)
@TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG})
public class HdfsServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
@Column("a")
private String status;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
index fa7e1a4..4c3040e 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/MRServiceTopologyAPIEntity.java
@@ -28,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize;
@Prefix("mrservicestatus")
@Service(TopologyConstants.MR_INSTANCE_SERVICE_NAME)
@TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG})
public class MRServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
@Column("a")
private String status;
http://git-wip-us.apache.org/repos/asf/eagle/blob/af9b056d/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
----------------------------------------------------------------------
diff --git a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
index 119cb46..e441741 100644
--- a/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
+++ b/eagle-topology-check/eagle-topology-entity/src/main/java/org/apache/eagle/topology/entity/SystemServiceTopologyAPIEntity.java
@@ -29,6 +29,7 @@ import org.apache.eagle.topology.TopologyConstants;
@Prefix("systemservicestatus")
@Service(TopologyConstants.SYSTEM_INSTANCE_SERVICE_NAME)
@TimeSeries(false)
+@Tags( {TopologyConstants.SITE_TAG, TopologyConstants.HOSTNAME_TAG, TopologyConstants.RACK_TAG, TopologyConstants.ROLE_TAG})
public class SystemServiceTopologyAPIEntity extends TopologyBaseAPIEntity {
@Column("a")
private String status;