You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2018/02/02 10:21:54 UTC
incubator-griffin git commit: modify the format of Metric DTO
Repository: incubator-griffin
Updated Branches:
refs/heads/master a723f4753 -> a92697225
modify the format of Metric DTO
Add 'type' field to Metric DTO, which corresponds to the DQ type of the related measure.
Author: He Wang <wa...@qq.com>
Closes #206 from whhe/master.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/a9269722
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/a9269722
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/a9269722
Branch: refs/heads/master
Commit: a92697225660ba1675f41d923672c5fbb14803d7
Parents: a723f47
Author: He Wang <wa...@qq.com>
Authored: Fri Feb 2 18:21:48 2018 +0800
Committer: Lionel Liu <bh...@163.com>
Committed: Fri Feb 2 18:21:48 2018 +0800
----------------------------------------------------------------------
.../apache/griffin/core/job/JobController.java | 7 +++-
.../org/apache/griffin/core/job/JobService.java | 7 +++-
.../apache/griffin/core/job/JobServiceImpl.java | 5 +--
.../core/measure/entity/GriffinMeasure.java | 12 ------
.../griffin/core/measure/entity/Measure.java | 20 +++++++--
.../griffin/core/metric/MetricServiceImpl.java | 2 +-
.../griffin/core/metric/MetricStoreImpl.java | 43 ++++++++++++--------
.../griffin/core/metric/model/Metric.java | 12 +++++-
.../griffin/core/job/JobControllerTest.java | 5 +--
.../griffin/core/job/JobServiceImplTest.java | 4 +-
.../core/metric/MetricControllerTest.java | 2 +-
.../core/metric/MetricStoreImplTest.java | 19 +++++++++
12 files changed, 89 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/job/JobController.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/JobController.java b/service/src/main/java/org/apache/griffin/core/job/JobController.java
index 7705e43..3f03f8c 100644
--- a/service/src/main/java/org/apache/griffin/core/job/JobController.java
+++ b/service/src/main/java/org/apache/griffin/core/job/JobController.java
@@ -19,7 +19,10 @@ under the License.
package org.apache.griffin.core.job;
-import org.apache.griffin.core.job.entity.*;
+import org.apache.griffin.core.job.entity.JobDataBean;
+import org.apache.griffin.core.job.entity.JobHealth;
+import org.apache.griffin.core.job.entity.JobInstanceBean;
+import org.apache.griffin.core.job.entity.JobSchedule;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.web.bind.annotation.*;
@@ -40,7 +43,7 @@ public class JobController {
@RequestMapping(value = "/jobs", method = RequestMethod.POST)
@ResponseStatus(HttpStatus.CREATED)
- public GriffinJob addJob(@RequestBody JobSchedule jobSchedule) throws Exception {
+ public JobSchedule addJob(@RequestBody JobSchedule jobSchedule) throws Exception {
return jobService.addJob(jobSchedule);
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/job/JobService.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/JobService.java b/service/src/main/java/org/apache/griffin/core/job/JobService.java
index 8c2c197..3c15f8f 100644
--- a/service/src/main/java/org/apache/griffin/core/job/JobService.java
+++ b/service/src/main/java/org/apache/griffin/core/job/JobService.java
@@ -19,7 +19,10 @@ under the License.
package org.apache.griffin.core.job;
-import org.apache.griffin.core.job.entity.*;
+import org.apache.griffin.core.job.entity.JobDataBean;
+import org.apache.griffin.core.job.entity.JobHealth;
+import org.apache.griffin.core.job.entity.JobInstanceBean;
+import org.apache.griffin.core.job.entity.JobSchedule;
import org.quartz.SchedulerException;
import java.util.List;
@@ -28,7 +31,7 @@ public interface JobService {
List<JobDataBean> getAliveJobs();
- GriffinJob addJob(JobSchedule jobSchedule) throws Exception;
+ JobSchedule addJob(JobSchedule jobSchedule) throws Exception;
void pauseJob(String group, String name) throws SchedulerException;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
index 226ad64..aa84f11 100644
--- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
@@ -137,7 +137,7 @@ public class JobServiceImpl implements JobService {
@Override
@Transactional(rollbackFor = Exception.class)
- public GriffinJob addJob(JobSchedule js) throws Exception {
+ public JobSchedule addJob(JobSchedule js) throws Exception {
Long measureId = js.getMeasureId();
GriffinMeasure measure = getMeasureIfValid(measureId);
checkJobScheduleParams(js, measure);
@@ -151,8 +151,7 @@ public class JobServiceImpl implements JobService {
job = jobRepo.save(job);
js = jobScheduleRepo.save(js);
addJob(triggerKey, js, job);
- return job;
-
+ return js;
}
private void addJob(TriggerKey triggerKey, JobSchedule js, GriffinJob job) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
index 6b060b5..8576b86 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
@@ -34,8 +34,6 @@ import java.util.List;
@Entity
public class GriffinMeasure extends Measure {
- private String dqType;
-
private String processType;
@Transient
@@ -52,16 +50,6 @@ public class GriffinMeasure extends Measure {
@JoinColumn(name = "evaluate_rule_id")
private EvaluateRule evaluateRule;
- @JsonProperty("dq.type")
- public String getDqType() {
- return dqType;
- }
-
- @JsonProperty("dq.type")
- public void setDqType(String dqType) {
- this.dqType = dqType;
- }
-
@JsonProperty("process.type")
public String getProcessType() {
return processType;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
index a5c97a1..4ccc75f 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
@@ -38,13 +38,15 @@ public abstract class Measure extends AbstractAuditableEntity {
@NotNull
protected String name;
- protected String description;
+ protected String owner;
- protected String organization;
+ private String dqType;
- protected String owner;
+ private String description;
- protected Boolean deleted = false;
+ private String organization;
+
+ private Boolean deleted = false;
public String getName() {
return name;
@@ -54,6 +56,16 @@ public abstract class Measure extends AbstractAuditableEntity {
this.name = name;
}
+ @JsonProperty("dq.type")
+ public String getDqType() {
+ return dqType;
+ }
+
+ @JsonProperty("dq.type")
+ public void setDqType(String dqType) {
+ this.dqType = dqType;
+ }
+
public String getDescription() {
return description;
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java b/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java
index b09c050..77cc168 100644
--- a/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/metric/MetricServiceImpl.java
@@ -71,7 +71,7 @@ public class MetricServiceImpl implements MetricService {
List<Metric> metrics = new ArrayList<>();
for (AbstractJob job : jobList) {
List<MetricValue> metricValues = getMetricValues(job.getMetricName(), 0, 300);
- metrics.add(new Metric(job.getMetricName(), measure.getOwner(), metricValues));
+ metrics.add(new Metric(job.getMetricName(), measure.getDqType(), measure.getOwner(), metricValues));
}
metricMap.put(measure.getName(), metrics);
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java b/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java
index b641cc7..360d4ec 100644
--- a/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/metric/MetricStoreImpl.java
@@ -33,6 +33,7 @@ import org.apache.http.message.BasicHeader;
import org.apache.http.nio.entity.NStringEntity;
import org.apache.http.util.EntityUtils;
import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseException;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.springframework.beans.factory.annotation.Value;
@@ -43,7 +44,6 @@ import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import java.io.IOException;
-import java.nio.charset.Charset;
import java.util.*;
@Component
@@ -51,14 +51,14 @@ public class MetricStoreImpl implements MetricStore {
private static final String INDEX = "griffin";
private static final String TYPE = "accuracy";
- private static final String URL_BASE = "/griffin/accuracy";
private RestClient client;
private HttpHeaders responseHeaders;
- private String url_get;
- private String url_delete;
- private String url_post;
+ private String urlGet;
+ private String urlDelete;
+ private String urlPost;
private ObjectMapper mapper;
+ private String indexMetaData;
public MetricStoreImpl(@Value("${elasticsearch.host}") String host,
@Value("${elasticsearch.port}") int port,
@@ -73,21 +73,30 @@ public class MetricStoreImpl implements MetricStore {
new BasicHeader(org.apache.http.HttpHeaders.AUTHORIZATION, encodedAuth)};
builder.setDefaultHeaders(requestHeaders);
}
- client = builder.build();
+ this.client = builder.build();
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentType(MediaType.APPLICATION_JSON);
this.responseHeaders = responseHeaders;
- this.url_get = URL_BASE + "/_search?filter_path=hits.hits._source";
- this.url_post = URL_BASE + "/_bulk";
- this.url_delete = URL_BASE + "/_delete_by_query";
+ String urlBase = String.format("/%s/%s", INDEX, TYPE);
+ this.urlGet = urlBase.concat("/_search?filter_path=hits.hits._source");
+ this.urlPost = urlBase.concat("/_bulk");
+ this.urlDelete = urlBase.concat("/_delete_by_query");
+ this.indexMetaData = String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }\n", INDEX, TYPE);
this.mapper = new ObjectMapper();
}
@Override
public List<MetricValue> getMetricValues(String metricName, int from, int size) throws IOException {
HttpEntity entity = getHttpEntityForSearch(metricName, from, size);
- Response response = client.performRequest("GET", url_get, Collections.emptyMap(), entity);
- return getMetricValuesFromResponse(response);
+ try {
+ Response response = client.performRequest("GET", urlGet, Collections.emptyMap(), entity);
+ return getMetricValuesFromResponse(response);
+ } catch (ResponseException e) {
+ if (e.getResponse().getStatusLine().getStatusCode() == 404) {
+ return Collections.emptyList();
+ }
+ throw e;
+ }
}
private HttpEntity getHttpEntityForSearch(String metricName, int from, int size) throws JsonProcessingException {
@@ -119,16 +128,15 @@ public class MetricStoreImpl implements MetricStore {
public ResponseEntity addMetricValues(List<MetricValue> metricValues) throws IOException {
String bulkRequestBody = getBulkRequestBody(metricValues);
HttpEntity entity = new NStringEntity(bulkRequestBody, ContentType.APPLICATION_JSON);
- Response response = client.performRequest("POST", url_post, Collections.emptyMap(), entity);
+ Response response = client.performRequest("POST", urlPost, Collections.emptyMap(), entity);
return getResponseEntityFromResponse(response);
}
private String getBulkRequestBody(List<MetricValue> metricValues) throws JsonProcessingException {
- String actionMetaData = String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }\n", INDEX, TYPE);
StringBuilder bulkRequestBody = new StringBuilder();
for (MetricValue metricValue : metricValues) {
- bulkRequestBody.append(actionMetaData);
+ bulkRequestBody.append(indexMetaData);
bulkRequestBody.append(JsonUtil.toJson(metricValue));
bulkRequestBody.append("\n");
}
@@ -141,7 +149,7 @@ public class MetricStoreImpl implements MetricStore {
Map<String, Object> param = Collections.singletonMap("query",
Collections.singletonMap("term", Collections.singletonMap("name.keyword", metricName)));
HttpEntity entity = new NStringEntity(JsonUtil.toJson(param), ContentType.APPLICATION_JSON);
- Response response = client.performRequest("POST", url_delete, Collections.emptyMap(), entity);
+ Response response = client.performRequest("POST", urlDelete, Collections.emptyMap(), entity);
return getResponseEntityFromResponse(response);
}
@@ -151,9 +159,8 @@ public class MetricStoreImpl implements MetricStore {
return new ResponseEntity<>(body, responseHeaders, status);
}
- private static String buildBasicAuthString (String user, String password) {
+ private static String buildBasicAuthString(String user, String password) {
String auth = user + ":" + password;
- String encodedAuth = "Basic " + Base64.getEncoder().encodeToString(auth.getBytes());
- return encodedAuth;
+ return "Basic " + Base64.getEncoder().encodeToString(auth.getBytes());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java b/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java
index 6de7a33..ed33658 100644
--- a/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java
+++ b/service/src/main/java/org/apache/griffin/core/metric/model/Metric.java
@@ -24,14 +24,16 @@ import java.util.List;
public class Metric {
private String name;
+ private String type;
private String owner;
private List<MetricValue> metricValues;
public Metric() {
}
- public Metric(String name, String owner, List<MetricValue> metricValues) {
+ public Metric(String name, String type, String owner, List<MetricValue> metricValues) {
this.name = name;
+ this.type = type;
this.owner = owner;
this.metricValues = metricValues;
}
@@ -44,6 +46,14 @@ public class Metric {
this.name = name;
}
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
public String getOwner() {
return owner;
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java b/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java
index 0290306..90a522b 100644
--- a/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java
+++ b/service/src/test/java/org/apache/griffin/core/job/JobControllerTest.java
@@ -40,7 +40,6 @@ import java.util.Collections;
import static org.apache.griffin.core.exception.GriffinExceptionMessage.JOB_ID_DOES_NOT_EXIST;
import static org.apache.griffin.core.exception.GriffinExceptionMessage.JOB_NAME_DOES_NOT_EXIST;
-import static org.apache.griffin.core.util.EntityHelper.createGriffinJob;
import static org.apache.griffin.core.util.EntityHelper.createJobSchedule;
import static org.hamcrest.CoreMatchers.is;
import static org.mockito.BDDMockito.given;
@@ -84,8 +83,8 @@ public class JobControllerTest {
@Test
public void testAddJobForSuccess() throws Exception {
JobSchedule jobSchedule = createJobSchedule();
- GriffinJob job = createGriffinJob();
- given(service.addJob(jobSchedule)).willReturn(job);
+ jobSchedule.setId(1L);
+ given(service.addJob(jobSchedule)).willReturn(jobSchedule);
mvc.perform(post(URLHelper.API_VERSION_PATH + "/jobs")
.contentType(MediaType.APPLICATION_JSON)
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java b/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java
index a5beb51..ba2afaa 100644
--- a/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java
+++ b/service/src/test/java/org/apache/griffin/core/job/JobServiceImplTest.java
@@ -146,8 +146,8 @@ public class JobServiceImplTest {
given(jobScheduleRepo.save(js)).willReturn(js);
given(jobRepo.save(Matchers.any(GriffinJob.class))).willReturn(job);
- GriffinJob griffinJob = service.addJob(js);
- assertEquals(griffinJob.getJobName(), job.getJobName());
+ JobSchedule createdJs = service.addJob(js);
+ assertEquals(js.getJobName(), createdJs.getJobName());
}
@Test(expected = GriffinException.BadRequestException.class)
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java b/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java
index 97ca3b6..84c8fb3 100644
--- a/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java
+++ b/service/src/test/java/org/apache/griffin/core/metric/MetricControllerTest.java
@@ -73,7 +73,7 @@ public class MetricControllerTest {
@Test
public void testGetAllMetricsSuccess() throws Exception {
- Metric metric = new Metric("metricName", "owner", Collections.emptyList());
+ Metric metric = new Metric("metricName", "accuracy", "owner", Collections.emptyList());
given(service.getAllMetrics()).willReturn(
Collections.singletonMap("measureName", Collections.singletonList(metric)));
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a9269722/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java b/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java
index 2ff5a3f..79f873e 100644
--- a/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java
+++ b/service/src/test/java/org/apache/griffin/core/metric/MetricStoreImplTest.java
@@ -1,3 +1,22 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+*/
+
package org.apache.griffin.core.metric;
import org.junit.Test;