You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by sc...@apache.org on 2018/10/26 00:12:16 UTC
[beam] branch master updated: [BEAM-5837] Add dashboard for
monitoring input data staleness (#6820)
This is an automated email from the ASF dual-hosted git repository.
scott pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0f9ef7c [BEAM-5837] Add dashboard for monitoring input data staleness (#6820)
0f9ef7c is described below
commit 0f9ef7c021cef5dc0e99c008b6e2e9e76ce82a03
Author: Scott Wegner <sc...@apache.org>
AuthorDate: Thu Oct 25 17:12:09 2018 -0700
[BEAM-5837] Add dashboard for monitoring input data staleness (#6820)
---
.test-infra/metrics/build.gradle | 22 +-
.../metrics/dashboards/source_data_freshness.json | 258 +++++++++++++++++++++
.../metrics/src/test/groovy/ProberTests.groovy | 14 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 57 +++--
4 files changed, 317 insertions(+), 34 deletions(-)
diff --git a/.test-infra/metrics/build.gradle b/.test-infra/metrics/build.gradle
index b91f302..4ae1a9e 100644
--- a/.test-infra/metrics/build.gradle
+++ b/.test-infra/metrics/build.gradle
@@ -16,12 +16,10 @@
* limitations under the License.
*/
-apply plugin: "base"
-apply plugin: "groovy"
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+applyGroovyNature()
-repositories {
- mavenCentral()
-}
+repositories { mavenCentral() }
// https://github.com/avast/gradle-docker-compose-plugin
apply plugin: "com.avast.gradle.docker-compose"
@@ -32,17 +30,11 @@ dependencies {
}
task testMetricsStack {
- doLast {
- // TODO(BEAM-5837): Add some actual validation of the metrics stack
- println "Hello world!"
- }
+ doLast { // TODO(BEAM-5837): Add some actual validation of the metrics stack
+ println "Hello world!" }
}
dockerCompose.isRequiredBy(testMetricsStack)
-task preCommit {
- dependsOn testMetricsStack
-}
+task preCommit { dependsOn testMetricsStack }
-task checkProber {
- dependsOn test
-}
+task checkProber { dependsOn test }
diff --git a/.test-infra/metrics/dashboards/source_data_freshness.json b/.test-infra/metrics/dashboards/source_data_freshness.json
new file mode 100644
index 0000000..655dd50
--- /dev/null
+++ b/.test-infra/metrics/dashboards/source_data_freshness.json
@@ -0,0 +1,258 @@
+{
+ "__inputs": [
+ {
+ "name": "DS_BEAMPSQL",
+ "label": "BeamPSQL",
+ "description": "",
+ "type": "datasource",
+ "pluginId": "postgres",
+ "pluginName": "PostgreSQL"
+ }
+ ],
+ "__requires": [
+ {
+ "type": "grafana",
+ "id": "grafana",
+ "name": "Grafana",
+ "version": "5.3.1"
+ },
+ {
+ "type": "panel",
+ "id": "graph",
+ "name": "Graph",
+ "version": "5.0.0"
+ },
+ {
+ "type": "datasource",
+ "id": "postgres",
+ "name": "PostgreSQL",
+ "version": "5.0.0"
+ }
+ ],
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": "-- Grafana --",
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "editable": true,
+ "gnetId": null,
+ "graphTooltip": 0,
+ "id": null,
+ "links": [],
+ "panels": [
+ {
+ "alert": {
+ "conditions": [
+ {
+ "evaluator": {
+ "params": [
+ 480
+ ],
+ "type": "gt"
+ },
+ "operator": {
+ "type": "and"
+ },
+ "query": {
+ "params": [
+ "A",
+ "5m",
+ "now"
+ ]
+ },
+ "reducer": {
+ "params": [],
+ "type": "max"
+ },
+ "type": "query"
+ }
+ ],
+ "executionErrorState": "alerting",
+ "frequency": "5m",
+ "handler": 1,
+ "name": "Source Data Freshness alert",
+ "noDataState": "alerting",
+ "notifications": []
+ },
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": "${DS_BEAMPSQL}",
+ "description": "Data freshness for each metrics input source. Used for health monitoring for other dashboards.",
+ "fill": 1,
+ "gridPos": {
+ "h": 12,
+ "w": 6,
+ "x": 0,
+ "y": 0
+ },
+ "id": 2,
+ "legend": {
+ "alignAsTable": false,
+ "avg": false,
+ "current": true,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "percentage": false,
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "format": "time_series",
+ "group": [
+ {
+ "params": [
+ "$__interval",
+ "none"
+ ],
+ "type": "time"
+ }
+ ],
+ "hide": false,
+ "metricColumn": "build_result",
+ "rawQuery": true,
+ "rawSql": "WITH sources AS (\n SELECT\n 'Jenkins' as source,\n MAX(build_timestamp + make_interval(secs:= timing_executingtimemillis / 1000.0)) AS last_sync\n FROM jenkins_builds\n UNION SELECT\n 'JIRA' as source,\n lastsynctime AS last_sync\n FROM jira_issues_metadata\n)\nSELECT\n current_timestamp AS time,\n source,\n EXTRACT(EPOCH FROM age(current_timestamp, last_sync)) / (60) AS value\nFROM sources",
+ "refId": "A",
+ "select": [
+ [
+ {
+ "params": [
+ "build_duration"
+ ],
+ "type": "column"
+ },
+ {
+ "params": [
+ "max"
+ ],
+ "type": "aggregate"
+ },
+ {
+ "params": [
+ "build_duration"
+ ],
+ "type": "alias"
+ }
+ ]
+ ],
+ "table": "jenkins_builds",
+ "timeColumn": "build_timestamp",
+ "timeColumnType": "timestamp",
+ "where": []
+ }
+ ],
+ "thresholds": [
+ {
+ "colorMode": "critical",
+ "fill": true,
+ "line": true,
+ "op": "gt",
+ "value": 480
+ }
+ ],
+ "timeFrom": null,
+ "timeShift": null,
+ "title": "Source Data Freshness",
+ "tooltip": {
+ "shared": false,
+ "sort": 1,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "buckets": null,
+ "mode": "series",
+ "name": null,
+ "show": false,
+ "values": [
+ "current"
+ ]
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "m",
+ "label": "Staleness",
+ "logBase": 1,
+ "max": "60",
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ }
+ }
+ ],
+ "schemaVersion": 16,
+ "style": "dark",
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-30d",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Source Data Freshness",
+ "uid": "data-freshness",
+ "version": 7
+}
diff --git a/.test-infra/metrics/src/test/groovy/ProberTests.groovy b/.test-infra/metrics/src/test/groovy/ProberTests.groovy
index f365327..e514972 100644
--- a/.test-infra/metrics/src/test/groovy/ProberTests.groovy
+++ b/.test-infra/metrics/src/test/groovy/ProberTests.groovy
@@ -17,6 +17,8 @@
* limitations under the License.
*/
+package org.apache.beam.testinfra.metrics
+
import org.junit.Test
import groovy.json.JsonSlurper
import static groovy.test.GroovyAssert.shouldFail
@@ -36,7 +38,17 @@ class ProberTests {
def dashboardNames = allDashboards.title
// Validate at least one expected dashboard exists
assert dashboardNames.contains('Post-commit Tests') : 'Expected dashboard does not exist'
- assert dashboardNames.size >= 3 : "Number of dashboards is less than expected ${dashboardNames}"
+ assert dashboardNames.size > 0 : "No dashboards found. Check Grafana dashboard initialization script."
+ }
+
+ @Test
+ void CheckGrafanaStalenessAlerts() {
+ def alertsJson = "${grafanaEndpoint}/api/alerts?dashboardId=data-freshness".toURL().text
+ def alerts = new JsonSlurper().parseText(alertsJson)
+ assert alerts.size > 0
+ alerts.each { alert ->
+ assert alert.state == 'ok' : "Input data is stale! ${grafanaEndpoint}/d/data-freshness"
+ }
}
}
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 7f2d822..18543ce 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -53,6 +53,26 @@ import org.gradle.testing.jacoco.tasks.JacocoReport
*/
class BeamModulePlugin implements Plugin<Project> {
+ /** Licence header enforced by spotless */
+ static final String javaLicenseHeader = """/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+"""
+
/** A class defining the set of configurable properties accepted by applyJavaNature. */
class JavaNatureConfiguration {
/** Controls the JDK source language and target compatibility. */
@@ -682,24 +702,7 @@ class BeamModulePlugin implements Plugin<Project> {
project.apply plugin: "com.diffplug.gradle.spotless"
project.spotless {
java {
- licenseHeader """/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-"""
+ licenseHeader javaLicenseHeader
googleJavaFormat()
}
}
@@ -1269,6 +1272,24 @@ artifactId=${project.name}
project.docker { noCache true }
}
+ /** ***********************************************************************************************/
+
+ project.ext.applyGroovyNature = {
+ println "Applying groovy nature"
+ project.apply plugin: "groovy"
+
+ project.apply plugin: "com.diffplug.gradle.spotless"
+ project.spotless {
+ def grEclipseConfig = project.project(":").file("buildSrc/greclipse.properties")
+ groovy {
+ licenseHeader javaLicenseHeader
+ paddedCell() // Recommended to avoid cyclic ambiguity issues
+ greclipse().configFile(grEclipseConfig)
+ }
+ groovyGradle { greclipse().configFile(grEclipseConfig) }
+ }
+ }
+
// containerImageName returns a configurable container image name, by default a
// development image at bintray.io (see sdks/CONTAINERS.md):
//