You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/05/04 03:04:57 UTC
[49/51] [partial] incubator-griffin git commit: refactor arch
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java
deleted file mode 100644
index 6a50e46..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/repo/UserSubscriptionRepoImpl.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.repo;
-
-import org.springframework.stereotype.Repository;
-
-import com.ebay.oss.griffin.domain.UserSubscription;
-import com.google.gson.Gson;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DBObject;
-import com.mongodb.util.JSON;
-
-@Repository
-public class UserSubscriptionRepoImpl extends BaseRepo<UserSubscription>
- implements UserSubscriptionRepo {
-
- private UserSubscriptionRepoImpl() {
- super("user_subscribe", UserSubscription.class);
- }
-
- @Override
- public void upsertUserSubscribe(UserSubscription item) {
- if (item.getNtaccount() == null)
- return;
-
- item.setId(item.getNtaccount());// user_subscribe
-
- DBObject find = dbCollection.findOne(new BasicDBObject("_id", item.getId()));
- if (find != null)
- dbCollection.remove(find);
-
- Gson gson = new Gson();
- DBObject t1 = (DBObject) JSON.parse(gson.toJson(item));
- dbCollection.save(t1);
- }
-
- @Override
- public UserSubscription getUserSubscribeItem(String user) {
- Gson gson = new Gson();
- DBObject find = dbCollection.findOne(new BasicDBObject("_id", user));
-
- if (find == null)
- return null;
- else
- return gson.fromJson(find.toString(), UserSubscription.class);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java
deleted file mode 100644
index 44c2acb..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQMetricsController.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import javax.validation.Valid;
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.ResponseBuilder;
-import javax.ws.rs.core.StreamingOutput;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import com.ebay.oss.griffin.domain.DqMetricsValue;
-import com.ebay.oss.griffin.domain.SampleFilePathLKP;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.error.BarkWebException;
-import com.ebay.oss.griffin.error.ErrorMessage;
-import com.ebay.oss.griffin.service.DQMetricsService;
-import com.ebay.oss.griffin.service.DqModelService;
-import com.ebay.oss.griffin.vo.AssetLevelMetrics;
-import com.ebay.oss.griffin.vo.DqModelVo;
-import com.ebay.oss.griffin.vo.OverViewStatistics;
-import com.ebay.oss.griffin.vo.SampleOut;
-import com.ebay.oss.griffin.vo.SystemLevelMetrics;
-
-@Component
-// @Scope("request")
-@Path("/metrics")
-public class DQMetricsController {
-
- //
-
- @Autowired
- private DQMetricsService dqMetricsService;
-
- @Autowired
- private DqModelService dqModelService;
-
- @POST
- @Path("/")
- @Consumes({ "application/json" })
- public void insertMetadata(@Valid DqMetricsValue dq) {
- if (dq != null) {
- ErrorMessage err = dq.validate();
- if (err != null) {
-
- throw new BarkWebException(Response.Status.BAD_REQUEST.getStatusCode(),
- err.getMessage());
- }
- }
-
- try{
- dqMetricsService.insertMetadata(dq);
- }catch(BarkDbOperationException e){
- throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- e.getMessage());
- }
-
-
- }
-
- @GET
- @Path("/{asset_id}/latest")
- @Produces(MediaType.APPLICATION_JSON)
- public DqMetricsValue getLatestMetricsValueById(
- @PathParam("asset_id") String assetId) {
- return dqMetricsService.getLatestlMetricsbyId(assetId);
- }
-
- @GET
- @Path("/heatmap")
- @Produces(MediaType.APPLICATION_JSON)
- public List<SystemLevelMetrics> getHeatMap() {
- return dqMetricsService.heatMap();
- }
-
- @GET
- @Path("/briefmetrics")
- @Produces(MediaType.APPLICATION_JSON)
- public List<SystemLevelMetrics> getAllBriefMetrics() {
- return dqMetricsService.briefMetrics("all");
- }
-
- @GET
- @Path("/briefmetrics/{system}")
- @Produces(MediaType.APPLICATION_JSON)
- public List<SystemLevelMetrics> getBriefMetrics(
- @PathParam("system") String system) {
- return dqMetricsService.briefMetrics(system);
- }
-
- @GET
- @Path("/dashboard")
- @Produces(MediaType.APPLICATION_JSON)
- public List<SystemLevelMetrics> getAllDashboard() {
- List<DqModelVo> models = dqModelService.getAllModles();
- Map<String, String> modelMap = new HashMap<String, String>();
-
- for (DqModelVo model : models) {
- modelMap.put(
- model.getName(),
- model.getAssetName() == null ? "unknow" : model
- .getAssetName());
- }
-
- List<SystemLevelMetrics> sysMetricsList = dqMetricsService
- .dashboard("all");
- for (SystemLevelMetrics sys : sysMetricsList) {
- List<AssetLevelMetrics> assetList = sys.getMetrics();
- if (assetList != null && assetList.size() > 0) {
- for (AssetLevelMetrics metrics : assetList) {
- metrics.setAssetName(modelMap.get(metrics.getName()));
- }
- }
- }
-
- return sysMetricsList;
- }
-
- @GET
- @Path("/dashboard/{system}")
- @Produces(MediaType.APPLICATION_JSON)
- public List<SystemLevelMetrics> getDashboard(
- @PathParam("system") String system) {
- return dqMetricsService.dashboard(system);
- }
-
- @GET
- @Path("/mydashboard/{user}")
- public List<SystemLevelMetrics> getAllDashboard(
- @PathParam("user") String user) {
- return dqMetricsService.mydashboard(user);
- }
-
- @GET
- @Path("/complete/{name}")
- @Produces(MediaType.APPLICATION_JSON)
- public AssetLevelMetrics getCompelete(@PathParam("name") String name) {
- return dqMetricsService.oneDataCompleteDashboard(name);
- }
-
- @GET
- @Path("/brief/{name}")
- @Produces(MediaType.APPLICATION_JSON)
- public AssetLevelMetrics getBrief(@PathParam("name") String name) {
- return dqMetricsService.oneDataBriefDashboard(name);
- }
-
- @GET
- @Path("/refresh")
- public void refreshDQ() {
- dqMetricsService.updateLatestDQList();
- }
-
- @GET
- @Path("/statics")
- @Produces(MediaType.APPLICATION_JSON)
- public OverViewStatistics getOverViewStats() {
- return dqMetricsService.getOverViewStats();
- }
-
- @POST
- @Path("/sample")
- @Consumes({ "application/json" })
- public Response insertSamplePath(SampleFilePathLKP sample) {
- dqMetricsService.insertSampleFilePath(sample);
- return Response.status(Response.Status.CREATED).build();
- }
-
- @GET
- @Path("/sample/{name}")
- @Produces(MediaType.APPLICATION_JSON)
- public List<SampleOut> getSampleList(@PathParam("name") String name) {
- return dqMetricsService.listSampleFile(name);
- }
-
- @GET
- @Path("/download/{path:.+}")
- @Produces(MediaType.APPLICATION_OCTET_STREAM)
- public Response downloadFile(@PathParam("path") final String path,
- @DefaultValue("100") @QueryParam("count") final Long count) {
-
- StreamingOutput fileStream = new StreamingOutput() {
- @Override
- public void write(OutputStream output) throws IOException,
- WebApplicationException {
- try {
- Process pro = Runtime.getRuntime().exec(
- "hadoop fs -cat /" + path);
-
- BufferedReader buff = new BufferedReader(
- new InputStreamReader(pro.getInputStream()));
-
- int cnt = 0;
- String line = null;
- while ((line = buff.readLine()) != null) {
- if (cnt < count) {
- output.write((line.getBytes()));
- output.write("\n".getBytes());
- output.flush();
- } else {
- break;
- }
- cnt++;
- }
- buff.close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- };
-
- String[] pathTokens = path.split("/");
- String fileName = pathTokens[pathTokens.length-2] + "_" + pathTokens[pathTokens.length-1];
-
- ResponseBuilder response = Response.ok(fileStream);
- response.header("Content-Disposition",
- "attachment; filename=" + fileName);
- return response.build();
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java
deleted file mode 100644
index 5e8c479..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DQModelController.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import java.util.Date;
-import java.util.List;
-
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.error.BarkWebException;
-import com.ebay.oss.griffin.error.ErrorMessage;
-import com.ebay.oss.griffin.service.DqModelService;
-import com.ebay.oss.griffin.service.NotificationService;
-import com.ebay.oss.griffin.vo.DqModelVo;
-import com.ebay.oss.griffin.vo.ModelInput;
-import com.ebay.oss.griffin.vo.NotificationRecord;
-import com.sun.jersey.api.Responses;
-
-@Component
-// @Scope("request")
-@Path("/models")
-public class DQModelController {
-
- @Autowired
- private DqModelService dqModelService;
-
- @Autowired
- private NotificationService notificationService;
-
- @GET
- @Path("/")
- @Produces(MediaType.APPLICATION_JSON)
- public List<DqModelVo> allModels() {
- return dqModelService.getAllModles();
- }
-
- @GET
- @Path("/{name}")
- @Produces(MediaType.APPLICATION_JSON)
- public ModelInput getModel(@PathParam("name") String name) {
-
- ModelInput model = dqModelService.getModelByName(name);
- if (model == null) {
- throw new BarkWebException(Responses.NOT_FOUND,
- "The model you are looking for doesn't exist!");
- }
- return model;
- }
-
- @GET
- @Path("/enableModel/{name}")
- public void enableModel(@PathParam("name") String name) {
- dqModelService.enableSchedule4Model(dqModelService.getGeneralModel(name));
- }
-
-
- @DELETE
- @Path("/{name}")
- public void deleteModel(@PathParam("name") String name) throws BarkDbOperationException {
-
- try {
- dqModelService.deleteModel(name);
- } catch (BarkDbOperationException e) {
- throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), e.getMessage());
- }
- }
-
- @POST
- @Path("/")
- public void newModel(@RequestBody ModelInput input) {
- if (input != null) {
-
- ErrorMessage err = input.validate();
- if (err != null) {
- throw new BarkWebException(Response.Status.BAD_REQUEST.getStatusCode(), err.getMessage());
- }
-
- try {
- dqModelService.newModel(input);
- notificationService.insert(new NotificationRecord(new Date().getTime(), input.getBasic().getOwner(),
- "create", "model", input.getBasic().getName()));
-
- } catch (BarkDbOperationException e) {
- throw new BarkWebException(Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(), e.getMessage());
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java
deleted file mode 100644
index 2386dc0..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/DataAssetController.java
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import java.util.Date;
-import java.util.List;
-
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import com.ebay.oss.griffin.domain.DataAsset;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.error.BarkWebException;
-import com.ebay.oss.griffin.service.DataAssetService;
-import com.ebay.oss.griffin.service.NotificationService;
-import com.ebay.oss.griffin.vo.DataAssetInput;
-import com.ebay.oss.griffin.vo.NotificationRecord;
-import com.ebay.oss.griffin.vo.PlatformMetadata;
-import com.sun.jersey.api.Responses;
-
-@Component
-//@Scope("request")
-@Path("/dataassets")
-public class DataAssetController {
-
- @Autowired
- private DataAssetService dataAssetService;
- @Autowired
- private NotificationService notificationService;
-
- /**
- * Get data asset metadata
- *
- * @return
- */
- @GET
- @Path("/metadata")
- @Produces(MediaType.APPLICATION_JSON)
- public List<PlatformMetadata> getSrcTree() {
- return dataAssetService.getSourceTree();
- }
-
- /**
- * Register a new data asset
- *
- * @param input
- * @return
- * @throws BarkWebException
- */
- @POST
- @Path("/")
- public void registerNewDataAsset(DataAssetInput input)
- throws BarkWebException {
- try {
- dataAssetService.createDataAsset(input);
- notificationService.insert(new NotificationRecord(new Date()
- .getTime(), input.getOwner(), "create", "dataasset", input
- .getAssetName()));
- // return Response.status(Response.Status.CREATED).build();
- } catch (BarkDbOperationException e) {
-
- throw new BarkWebException(
- Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- e.getMessage());
- }
-
- }
-
- /**
- * Get all data asset list
- *
- * @return
- */
- @GET
- @Path("/")
- @Produces(MediaType.APPLICATION_JSON)
- public List<DataAsset> getAssetList() {
- return dataAssetService.getAllDataAssets();
- }
-
- /**
- * Get one data asset by id
- *
- * @return
- * @throws BarkWebException
- */
- @GET
- @Path("/{id}")
- @Produces(MediaType.APPLICATION_JSON)
- public DataAsset getOneAsset(@PathParam("id") Long id)
- throws BarkWebException {
- DataAsset da = dataAssetService.getDataAssetById(id);
- if (da == null) {
- throw new BarkWebException(Responses.NOT_FOUND,
- "The asset you are looking for doesn't exist");
- } else {
- return da;
- }
- }
-
- /**
- * Remove a data asset by id
- *
- * @param id
- * @return
- */
- @DELETE
- @Path("/{id}")
- public void removeAssetById(@PathParam("id") Long id) throws BarkWebException {
-
- try {
- dataAssetService.removeAssetById(id);
-
- } catch (BarkDbOperationException e) {
- throw new BarkWebException(
- Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- e.getMessage());
- }
-
- }
-
- /**
- * Update a new data asset
- *
- * @param input
- * @return
- */
- @PUT
- @Path("/")
- public void updateDataAsset(DataAssetInput input) throws BarkWebException{
-
- try {
- dataAssetService.updateDataAsset(input);
-
- } catch (BarkDbOperationException e) {
- throw new BarkWebException(
- Response.Status.INTERNAL_SERVER_ERROR.getStatusCode(),
- e.getMessage());
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java
deleted file mode 100644
index 7f4d4df..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/LoginController.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import com.ebay.oss.griffin.service.LoginService;
-import com.ebay.oss.griffin.vo.LoginUser;
-
-@Component
-//@Scope("request")
-@Path("/login")
-public class LoginController {
-
- @Autowired
- private LoginService loginService;
-
- @POST
- @Path("/authenticate")
- @Produces(MediaType.APPLICATION_JSON)
- public String authenticate(@RequestBody LoginUser input) {
- if("test".equals(input.getUsername())){
- return "{\"status\": 0, \"ntAccount\": \"test\", \"fullName\": \"test\"}";
- }
-
- String result = loginService.login(input.getUsername(),
- input.getPassword());
- if (result != null)
- return "{\"status\": 0, \"ntAccount\": \"" + input.getUsername()
- + "\", \"fullName\": \"" + result + "\"}";
- else
- return "{\"status\": -1, \"ntAccount\": \"" + input.getUsername()
- + "\", \"fullName\": \"\",\"errMsg\":\"logon failed.\"}";
- // return "{\"status\": -1, \"errMsg\": \"Invalid username\"}";
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java
deleted file mode 100644
index e2ffb59..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/NotificationController.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import java.util.List;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import com.ebay.oss.griffin.service.NotificationService;
-import com.ebay.oss.griffin.vo.NotificationRecord;
-
-@Component
-//@Scope("request")
-@Path("/notifications")
-public class NotificationController {
- @Autowired
- private NotificationService notificationService;
-
- @GET
- @Path("/")
- @Produces(MediaType.APPLICATION_JSON)
- public List<NotificationRecord> getAll(){
- return notificationService.getAll();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java
deleted file mode 100644
index d3dfc1a..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/ScheduleController.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import com.ebay.oss.griffin.service.DqScheduleService;
-
-@Component
-//@Scope("request")
-@Path("/schedule")
-public class ScheduleController {
-
- @Autowired
- private DqScheduleService dqJobSchedulingService;
-
- @GET
- @Path("/")
- public void schedule() {
- dqJobSchedulingService.schedulingJobs();
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java b/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java
deleted file mode 100644
index 91e3f81..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/resources/SubscribeController.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.resources;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.core.MediaType;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.springframework.web.bind.annotation.RequestBody;
-
-import com.ebay.oss.griffin.domain.UserSubscription;
-import com.ebay.oss.griffin.service.SubscribeService;
-
-
-@Component
-//@Scope("request")
-@Path("/subscribe")
-public class SubscribeController {
-
- @Autowired
- private SubscribeService subscribeService;
-
- @GET
- @Path("/{user}")
- @Produces(MediaType.APPLICATION_JSON)
- public UserSubscription get(@PathParam("user") String user) {
- return subscribeService.getSubscribe(user);
- }
-
- @POST
- @Path("/")
- @Consumes({"application/json"})
- public void set(@RequestBody UserSubscription input) {
- subscribeService.subscribe(input);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java
deleted file mode 100644
index 14a827c..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/Converter.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-public interface Converter<E, V> {
-
- V voOf(E e);
-
- E entityOf(V vo);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java
deleted file mode 100644
index aa19cbc..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsService.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import java.util.List;
-
-import com.ebay.oss.griffin.domain.DqMetricsValue;
-import com.ebay.oss.griffin.domain.SampleFilePathLKP;
-import com.ebay.oss.griffin.vo.AssetLevelMetrics;
-import com.ebay.oss.griffin.vo.OverViewStatistics;
-import com.ebay.oss.griffin.vo.SampleOut;
-import com.ebay.oss.griffin.vo.SystemLevelMetrics;
-
-public interface DQMetricsService {
-
- public void insertMetadata(DqMetricsValue dq);
-
- public DqMetricsValue getLatestlMetricsbyId(String assetId);
-
- public List<SystemLevelMetrics> briefMetrics(String system);
-
- public void updateLatestDQList();
-
- public OverViewStatistics getOverViewStats();
-
- ////////////// sample-file
- public List<SampleOut> listSampleFile(String modelName);
-
-// public void downloadSample(String filePath);
-
- public void insertSampleFilePath(SampleFilePathLKP samplePath);
-
- ////////////// HeatMap
- public List<SystemLevelMetrics> heatMap();
-
- ///////////// dashboard
- public List<SystemLevelMetrics> dashboard(String system);
-
- public AssetLevelMetrics oneDataCompleteDashboard(String name);
-
- public AssetLevelMetrics oneDataBriefDashboard(String name);
-
- public AssetLevelMetrics metricsForReport(String name);
-
- public List<SystemLevelMetrics> mydashboard(String user);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java
deleted file mode 100644
index e7ee1d8..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DQMetricsServiceImpl.java
+++ /dev/null
@@ -1,321 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.ebay.oss.griffin.common.Pair;
-import com.ebay.oss.griffin.domain.DqMetricsValue;
-import com.ebay.oss.griffin.domain.DqModel;
-import com.ebay.oss.griffin.domain.SampleFilePathLKP;
-import com.ebay.oss.griffin.domain.SystemType;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.repo.DataAssetRepo;
-import com.ebay.oss.griffin.repo.DqMetricsRepo;
-import com.ebay.oss.griffin.repo.DqModelRepo;
-import com.ebay.oss.griffin.repo.SampleFilePathRepo;
-import com.ebay.oss.griffin.vo.AssetLevelMetrics;
-import com.ebay.oss.griffin.vo.DQHealthStats;
-import com.ebay.oss.griffin.vo.DqModelVo;
-import com.ebay.oss.griffin.vo.OverViewStatistics;
-import com.ebay.oss.griffin.vo.SampleOut;
-import com.ebay.oss.griffin.vo.SystemLevelMetrics;
-import com.ebay.oss.griffin.vo.SystemLevelMetricsList;
-import com.mongodb.DBObject;
-
-@Service("dqmetrics")
-public class DQMetricsServiceImpl implements DQMetricsService {
-
- private static Logger logger = LoggerFactory.getLogger(DQMetricsServiceImpl.class);
-
- @Autowired
- private DqModelService dqModelService;
-
- @Autowired
- private SubscribeService subscribeService;
-
- @Autowired
- private DqMetricsRepo metricsRepo;
-
- @Autowired
- private DqModelRepo modelRepo;
-
- @Autowired
- private DataAssetRepo dataAssetRepo;
-
- @Autowired
- private SampleFilePathRepo missedFileRepo;
-
- @Autowired
- private RefMetrcsCalc refMetricCalc;
-
- public static List<DqMetricsValue> cacheValues = new ArrayList<DqMetricsValue>();;
-
- public static SystemLevelMetricsList totalSystemLevelMetricsList;
-
- public void insertMetadata(DqMetricsValue metrics) {
-
- List<Pair> queryList = new ArrayList<Pair>();
- queryList.add(new Pair("metricName", metrics.getMetricName()));
- // queryList.add(new KeyValue("metricType", dq.getMetricType()));
- // queryList.add(new KeyValue("assetId", dq.getAssetId()));
- queryList.add(new Pair("timestamp", metrics.getTimestamp()));
-
- List<Pair> updateValues = new ArrayList<Pair>();
- updateValues.add(new Pair("value", metrics.getValue()));
-
- DBObject item = metricsRepo.getByCondition(queryList);
-
- try {
- if (item == null) {
- long seq = metricsRepo.getNextId();
- logger.info("log: new record inserted" + seq);
- metrics.set_id(seq);
- metricsRepo.save(metrics);
- } else {
- logger.info("log: updated record");
- metricsRepo.update(metrics, item);
- }
- }catch(Exception e) {
- throw new BarkDbOperationException("Failed to save metrics value!", e);
- }
-
- }
-
- public DqMetricsValue getLatestlMetricsbyId(String assetId) {
- return metricsRepo.getLatestByAssetId(assetId);
- }
-
- void refreshAllDQMetricsValuesinCache() {
- refreshModelSystemCache();
-
- cacheValues.clear();
- for (DqMetricsValue each : metricsRepo.getAll()) {
- cacheValues.add(each);
- }
- }
- Map<String, String> modelSystem = new HashMap<String, String>();
- void refreshModelSystemCache() {
- for (DqModel model : modelRepo.getAll()) {
- modelSystem.put(model.getModelName(), SystemType.val(model.getSystem()));
- }
- }
-
- public synchronized void updateLatestDQList() {
- try {
- logger.info("==============updating all latest dq metrics==================");
- refreshAllDQMetricsValuesinCache();
-
- totalSystemLevelMetricsList = new SystemLevelMetricsList();
- for (DqMetricsValue temp : cacheValues) {
- // totalSystemLevelMetricsList.upsertNewAsset(temp, assetSystem,
- // 1);
- totalSystemLevelMetricsList.upsertNewAssetExecute(
- temp.getMetricName(), "", temp.getTimestamp(),
- temp.getValue(), modelSystem.get(temp.getMetricName()),
- 0, true, null);
- }
-
- totalSystemLevelMetricsList.updateDQFail(getThresholds());
- refMetricCalc.calc(totalSystemLevelMetricsList);
-
- logger.info("==============update all latest dq metrics done==================");
- } catch (Exception e) {
- logger.error("{}", e);
- }
- }
-
- Map<String, String> getThresholds() {
- Map<String, String> thresHolds = new HashMap<>();
- for(DqModel each : modelRepo.getAll()) {
- thresHolds.put(each.getModelName(), "" + each.getThreshold());
- }
- return thresHolds;
- }
-
- List<SystemLevelMetrics> addAssetNames(
- List<SystemLevelMetrics> result) {
- List<DqModelVo> models = dqModelService.getAllModles();
- Map<String, String> modelMap = new HashMap<String, String>();
-
- for (DqModelVo model : models) {
- modelMap.put(
- model.getName(),
- model.getAssetName() == null ? "unknow" : model
- .getAssetName());
- }
-
- for (SystemLevelMetrics sys : result) {
- List<AssetLevelMetrics> assetList = sys.getMetrics();
- if (assetList != null && assetList.size() > 0) {
- for (AssetLevelMetrics metrics : assetList) {
- metrics.setAssetName(modelMap.get(metrics.getName()));
- }
- }
- }
-
- return result;
- }
-
- Map<String, String> getAssetMap() {
- Map<String, String> modelMap = new HashMap<String, String>();
-
- for (DqModelVo model : dqModelService.getAllModles()) {
- modelMap.put( model.getName(),
- model.getAssetName() == null ? "unknow" : model.getAssetName());
- }
-
- return modelMap;
- }
-
- @Override
- public List<SystemLevelMetrics> briefMetrics(String system) {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return totalSystemLevelMetricsList.getListWithLatestNAssets(24, system, null, null);
- }
-
- @Override
- public List<SystemLevelMetrics> heatMap() {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return totalSystemLevelMetricsList.getHeatMap(getThresholds());
- }
-
- @Override
- public List<SystemLevelMetrics> dashboard(String system) {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return addAssetNames(totalSystemLevelMetricsList
- .getListWithLatestNAssets(30, system, null, null));
- }
-
- @Override
- public List<SystemLevelMetrics> mydashboard(String user) {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return addAssetNames(totalSystemLevelMetricsList
- .getListWithLatestNAssets(30, "all",
- subscribeService.getSubscribe(user), getAssetMap()));
- }
-
- @Override
- public AssetLevelMetrics oneDataCompleteDashboard(String name) {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return totalSystemLevelMetricsList.getListWithSpecificAssetName(name);
- }
-
- @Override
- public AssetLevelMetrics oneDataBriefDashboard(String name) {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return totalSystemLevelMetricsList.getListWithSpecificAssetName(name, 30);
- }
-
- public OverViewStatistics getOverViewStats() {
-
- OverViewStatistics os = new OverViewStatistics();
-
- os.setAssets(dataAssetRepo.getAll().size());
- os.setMetrics(modelRepo.getAll().size());
-
- DQHealthStats health = new DQHealthStats();
-
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
-
- List<SystemLevelMetrics> allMetrics = totalSystemLevelMetricsList
- .getLatestDQList();
-
- int healthCnt = 0;
- int invalidCnt = 0;
-
- for (SystemLevelMetrics metricS : allMetrics) {
-
- List<AssetLevelMetrics> metricsA = metricS.getMetrics();
-
- for (AssetLevelMetrics m : metricsA) {
- if (m.getDqfail() == 0) {
- healthCnt++;
- } else {
- invalidCnt++;
- }
- }
- }
-
- health.setHealth(healthCnt);
- health.setInvalid(invalidCnt);
-
- health.setWarn(0);
- os.setStatus(health);
-
- return os;
-
- }
-
- /**
- * Get the metrics for 24 hours
- */
- @Override
- public AssetLevelMetrics metricsForReport(String name) {
- if (totalSystemLevelMetricsList == null)
- updateLatestDQList();
- return totalSystemLevelMetricsList.getListWithSpecificAssetName(name, 24);
- }
-
- @Override
- public List<SampleOut> listSampleFile(String modelName) {
-
- List<SampleOut> samples = new ArrayList<SampleOut>();
-
- List<DBObject> dbos = missedFileRepo.findByModelName(modelName);
-
- for (DBObject dbo : dbos) {
-
- SampleOut so = new SampleOut();
-
- so.setDate(Long.parseLong(dbo.get("timestamp").toString()));
- so.setPath(dbo.get("hdfsPath").toString());
-
- samples.add(so);
- }
-
- return samples;
-
- }
-
- @Override
- public void insertSampleFilePath(SampleFilePathLKP samplePath) {
- SampleFilePathLKP entity = new SampleFilePathLKP();
-
- entity.set_id(missedFileRepo.getNextId());
- entity.setModelName(samplePath.getModelName());
- entity.setTimestamp(samplePath.getTimestamp());
- entity.setHdfsPath(samplePath.getHdfsPath());
-
- missedFileRepo.save(entity);
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java
deleted file mode 100644
index 118061a..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetService.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-*/
-package com.ebay.oss.griffin.service;
-
-import java.util.List;
-
-import com.ebay.oss.griffin.domain.DataAsset;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.vo.DataAssetInput;
-import com.ebay.oss.griffin.vo.PlatformMetadata;
-
-public interface DataAssetService {
- public List<DataAsset> getAllDataAssets();
-
- public int createDataAsset(DataAssetInput input) throws BarkDbOperationException;
-
- public void updateDataAsset(DataAssetInput input) throws BarkDbOperationException;
-
- public DataAsset getDataAssetById(Long id) throws BarkDbOperationException;
-
- public List<PlatformMetadata> getSourceTree();
-
- public void removeAssetById(Long id) throws BarkDbOperationException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java
deleted file mode 100644
index b6a9f28..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DataAssetServiceImpl.java
+++ /dev/null
@@ -1,363 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.core.env.Environment;
-import org.springframework.stereotype.Service;
-//import org.springframework.validation.annotation.Validated;
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-import com.ebay.oss.griffin.common.HDFSUtils;
-import com.ebay.oss.griffin.common.Pair;
-import com.ebay.oss.griffin.domain.DataAsset;
-import com.ebay.oss.griffin.domain.DqModel;
-import com.ebay.oss.griffin.domain.ModelStatus;
-import com.ebay.oss.griffin.domain.ModelType;
-import com.ebay.oss.griffin.domain.ScheduleType;
-import com.ebay.oss.griffin.domain.SystemType;
-import com.ebay.oss.griffin.domain.ValidityType;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.repo.DataAssetRepo;
-import com.ebay.oss.griffin.repo.DqModelRepo;
-import com.ebay.oss.griffin.vo.DataAssetIndex;
-import com.ebay.oss.griffin.vo.DataAssetInput;
-import com.ebay.oss.griffin.vo.DqModelVo;
-import com.ebay.oss.griffin.vo.ModelBasicInputNew;
-import com.ebay.oss.griffin.vo.ModelExtraInputNew;
-import com.ebay.oss.griffin.vo.ModelInput;
-import com.ebay.oss.griffin.vo.PlatformMetadata;
-import com.ebay.oss.griffin.vo.SystemMetadata;
-import com.mongodb.DBObject;
-
-@Service
-public class DataAssetServiceImpl implements DataAssetService {
-
- private static Logger logger = LoggerFactory.getLogger(DataAssetServiceImpl.class);
-
- @Autowired
- private DataAssetRepo dataAssetRepo;
-
- @Autowired
- private DqModelRepo dqModelRepo;
-
- @Autowired
- private DqModelService dqModelService;
-
- @Autowired
- private Environment env;
-
- // FIXME ???
- public static List<DqModelVo> allModels;
-
- // FIXME ???
- public static HashMap<String, String> thresholds;
-
- @Override
- public List<DataAsset> getAllDataAssets() {
- return dataAssetRepo.getAll();
- }
-
- protected DataAsset ofEntity(DBObject o) {
- return new DataAsset(o);
- }
- @Override
- public int createDataAsset(DataAssetInput input)
- throws BarkDbOperationException {
-
- DataAsset da = new DataAsset();
-
- List<Pair> queryList = new ArrayList<Pair>();
- queryList.add(new Pair("assetName", input.getAssetName()));
- queryList.add(new Pair("assetType", input.getAssetType()));
- queryList.add(new Pair("system", input.getSystem()));
-
- List<Pair> updateValues = new ArrayList<Pair>();
- updateValues.add(new Pair("schema", input.getSchema()));
- updateValues.add(new Pair("platform", input.getPlatform()));
- updateValues
- .add(new Pair("assetHDFSPath", input.getAssetHDFSPath()));
- updateValues.add(new Pair("owner", input.getOwner()));
-
- DBObject item = dataAssetRepo.getByCondition(queryList);
-
- if (item != null) {
- throw new BarkDbOperationException("Record already existing");
- }
-
- String hdfsPath = input.getAssetHDFSPath();
-
- String[] subs = hdfsPath.split("/");
-
- StringBuilder sb = new StringBuilder();
-
- for (String s : subs) {
- if(!s.contains("[")){
- sb.append("/");
- sb.append(s);
- }
- }
-
- hdfsPath = sb.toString().replaceFirst("/", "");
-
-
- logger.info("HDFS Path: " + hdfsPath);
-
- if("prod".equals(env.getProperty("env"))){//in prod environment, need to validate the hdfs path
-
- if (!HDFSUtils.checkHDFSFolder(hdfsPath)) {
-
-
- throw new BarkDbOperationException(
- "Hdfs Path is invalid, please input valid hdfs path!");
- }
- }
-
- try {
-
- long seq = dataAssetRepo.getNextId();
- da.set_id(seq);
- da.setAssetName(input.getAssetName());
- da.setAssetType(input.getAssetType());
- da.setPlatform(input.getPlatform());
- da.setSystem(input.getSystem());
- da.setAssetHDFSPath(input.getAssetHDFSPath());
- da.setSchema(input.getSchema());
- da.setOwner(input.getOwner());
- da.setPartitions(input.getPartitions());
- da.setTimestamp(new Date());
-
- logger.debug("log: new record inserted" + seq);
- dataAssetRepo.save(da);
-
- //create new total count dqmodel
- {
- ModelInput tempCountModel = new ModelInput();
- ModelBasicInputNew basic = new ModelBasicInputNew();
- ModelExtraInputNew extra = new ModelExtraInputNew();
- basic.setDataaset(input.getAssetName());
- basic.setDataasetId(seq);
- basic.setDesc("Count for " + input.getAssetName());
- basic.setEmail(input.getOwner() + "@ebay.com");
- basic.setName("TotalCount_" + input.getAssetName());
- basic.setOwner(input.getOwner());
- basic.setScheduleType(ScheduleType.DAILY);
- basic.setStatus(ModelStatus.DEPLOYED);
- basic.setSystem(SystemType.indexOf(input.getSystem()));
- basic.setType(ModelType.VALIDITY);
- extra.setVaType(ValidityType.TOTAL_COUNT);
- extra.setColumn("wholeDataSet");
- extra.setSrcDataSet(input.getAssetName());
- extra.setSrcDb(input.getPlatform());
- tempCountModel.setBasic(basic);
- tempCountModel.setExtra(extra);
- dqModelService.newModel(tempCountModel);
- }
-
- return 0;
-
- } catch (Exception e) {
- throw new BarkDbOperationException(
- "Failed to create a new data asset", e);
- }
-
- }
-
- @Override
- public void updateDataAsset(DataAssetInput input)
- throws BarkDbOperationException {
- try {
-
- DataAsset da = new DataAsset();
-
- List<Pair> queryList = new ArrayList<Pair>();
- queryList.add(new Pair("assetName", input.getAssetName()));
- queryList.add(new Pair("assetType", input.getAssetType()));
- queryList.add(new Pair("system", input.getSystem()));
-
- List<Pair> updateValues = new ArrayList<Pair>();
- updateValues.add(new Pair("schema", input.getSchema()));
- updateValues.add(new Pair("platform", input.getPlatform()));
- updateValues.add(new Pair("assetHDFSPath", input
- .getAssetHDFSPath()));
- updateValues.add(new Pair("owner", input.getOwner()));
-
- DBObject item = dataAssetRepo.getByCondition(queryList);
- if (item == null) {
- throw new BarkDbOperationException( "The data asset doesn't exist");
- }
-
- da.setAssetName(input.getAssetName());
- da.setAssetType(input.getAssetType());
- da.setPlatform(input.getPlatform());
- da.setSystem(input.getSystem());
- da.setAssetHDFSPath(input.getAssetHDFSPath());
- da.setSchema(input.getSchema());
- da.setOwner(input.getOwner());
- da.setPartitions(input.getPartitions());
- da.setTimestamp(new Date());
-
- logger.warn( "log: updated record, id is: "
- + (long) Double.parseDouble(item.get("_id").toString()));
- da.set_id(new Long((long) Double.parseDouble(item.get("_id")
- .toString())));
- dataAssetRepo.update(da, item);
- } catch (Exception e) {
- throw new BarkDbOperationException("Failed to update data asset", e);
- }
-
- }
-
- @Override
- public DataAsset getDataAssetById(Long id) throws BarkDbOperationException {
- return dataAssetRepo.getById(id);
- }
-
- @Override
- public List<PlatformMetadata> getSourceTree() {
-
- List<PlatformMetadata> alltree = new ArrayList<PlatformMetadata>();
-
- List<DataAsset> allAssets = dataAssetRepo.getAll();
-
-
- for (String platform : getAllPlatforms(allAssets)) {
-
- PlatformMetadata plat = new PlatformMetadata();
- plat.setPlatform(platform);
-
-
- List<SystemMetadata> d = new ArrayList<SystemMetadata>();
-
- for (String dataset : getSystemsByPlatform( platform, allAssets)) {
-
- SystemMetadata db = new SystemMetadata();
- db.setName(dataset);
-
- List<DataAssetIndex> assets = new ArrayList<DataAssetIndex>();
-
- Map<Long, String> tableB = getAssetsBySystem(platform, dataset, allAssets);
- for (Entry<Long, String> entry : tableB.entrySet()) {
- DataAssetIndex dai = new DataAssetIndex();
- dai.setId(entry.getKey());
- dai.setName(entry.getValue());
-
- assets.add(dai);
- }
-
- db.setAssets(assets);
- d.add(db);
-
- }
-
- plat.setSystems(d);
- alltree.add(plat);
-
- }
-
- return alltree;
-
- }
-
- private Set<String> getAllPlatforms(List<DataAsset> records) {
- Set<String> p = new LinkedHashSet<>();
- for (DataAsset o : records) {
- p.add(o.getPlatform());
- }
- return p;
- }
-
- public Set<String> getSystemsByPlatform(String platform, List<DataAsset> records) {
- Set<String> p = new HashSet<String>();
- for (DataAsset o : records) {
- if (o.getPlatform().equals(platform)) {
- p.add(o.getSystem());
- }
- }
- return p;
-
- }
-
- public Map<Long, String> getAssetsBySystem(String platform, String system, List<DataAsset> records) {
- Map<Long, String> p = new HashMap<Long, String>();
- for (DataAsset o : records) {
- if (o.getPlatform().equals(platform) && o.getSystem().equals(system)) {
- p.put(o.get_id(), o.getAssetName());
- }
- }
- return p;
-
- }
-
- @Override
- public void removeAssetById(Long id) throws BarkDbOperationException {
- try {
- removeRelatedModels(id);
-
- dataAssetRepo.delete(id);
- } catch (Exception e) {
- throw new BarkDbOperationException("Failed to delete data asset with id of '" + id + "'", e);
- }
-
- }
-
- //remove all the models related with the given data asset
- private int removeRelatedModels(Long dataAssetId) {
- try {
- DataAsset da = dataAssetRepo.getById(dataAssetId);
- if (da != null) {
- //delete all the accuracy models with this given source asset
- List<DqModel> models = dqModelRepo.getByDataAsset(da);
- for(DqModel each : models) {
- dqModelService.deleteModel(each.getModelName());
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- return 0;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java
deleted file mode 100644
index 82b3fd6..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelConverter.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import java.util.Date;
-
-import org.springframework.stereotype.Component;
-
-import com.ebay.oss.griffin.domain.DqModel;
-import com.ebay.oss.griffin.vo.DqModelVo;
-
-@Component("modelVoConverter")
-public class DqModelConverter implements Converter<DqModel, DqModelVo>{
-
- @Override
- public DqModelVo voOf(DqModel o) {
- if(o == null) {
- return null;
- }
-
- DqModelVo vo = new DqModelVo();
- vo.setName(o.getModelName());
- vo.setSystem(o.getSystem());
- vo.setDescription(o.getModelDesc());
- vo.setType(o.getModelType());
- vo.setCreateDate(new Date(o.getTimestamp()));
- vo.setStatus("" + o.getStatus());
- vo.setAssetName(o.getAssetName());
- vo.setOwner(o.getOwner());
- return vo;
- }
-
- @Override
- public DqModel entityOf(DqModelVo vo) {
- // TODO Auto-generated method stub
- throw new RuntimeException("not implemented yet...");
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java
deleted file mode 100644
index 6404d39..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelCreator.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import com.ebay.oss.griffin.domain.DqModel;
-import com.ebay.oss.griffin.vo.ModelInput;
-
-public interface DqModelCreator {
-
- /** the minimum number of Jobs executed in the ModelStatus.TESTING, and then it could be shifted
- * to ModelStatus.VERIFIED.
- */
- int MIN_TESTING_JOB_NUMBER = 1; // here set only 1 time to execute a job, for test
-
- boolean isSupport(ModelInput input);
-
- // FIXME the return value here doesn't mean anything
- DqModel newModel(ModelInput input);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java
deleted file mode 100644
index c5d19a8..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelService.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import java.util.List;
-
-import com.ebay.oss.griffin.domain.DqModel;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.vo.DqModelVo;
-import com.ebay.oss.griffin.vo.ModelInput;
-
-public interface DqModelService {
-
- public List<DqModelVo> getAllModles();
-
- // FIXME what the return value means
- // HACK why need to delete a model
- public int deleteModel(String name) throws BarkDbOperationException;
-
- public DqModel getGeneralModel(String name);
-
-
- public ModelInput getModelByName(String name) throws BarkDbOperationException;
-
- public void enableSchedule4Model(DqModel input);
-
- /**
- * Create a new Model
- * @param input
- * @return 0 if successful, -1 if already existing, other negative values for other reasons
- */
- public DqModel newModel(ModelInput input) throws BarkDbOperationException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java
deleted file mode 100644
index 4a369da..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqModelServiceImpl.java
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import javax.annotation.Resource;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Service;
-
-import com.ebay.oss.griffin.common.ScheduleModelSeperator;
-import com.ebay.oss.griffin.domain.DqModel;
-import com.ebay.oss.griffin.domain.DqSchedule;
-import com.ebay.oss.griffin.domain.ModelStatus;
-import com.ebay.oss.griffin.domain.ModelType;
-import com.ebay.oss.griffin.error.BarkDbOperationException;
-import com.ebay.oss.griffin.repo.DqMetricsRepo;
-import com.ebay.oss.griffin.repo.DqModelRepo;
-import com.ebay.oss.griffin.repo.DqScheduleRepo;
-import com.ebay.oss.griffin.vo.DqModelVo;
-import com.ebay.oss.griffin.vo.ModelInput;
-import com.mongodb.DBObject;
-
-@Service
-public class DqModelServiceImpl implements DqModelService {
-
- private static Logger logger = LoggerFactory.getLogger(DqModelServiceImpl.class);
-
- @Autowired
- private DqModelRepo dqModelRepo;
-
- @Autowired
- private DqScheduleRepo scheduleRepo;
-
- @Resource(name = "modelVoConverter")
- private Converter<DqModel, DqModelVo> converter;
-
- @Resource(name = "modelInputConverter")
- Converter<DqModel, ModelInput> modelInputConverter;
-
- @Autowired
- private DqMetricsRepo metricsRepo;
-
- @Override
- public List<DqModelVo> getAllModles() {
- List<DqModelVo> allModels = new ArrayList<>();
- for(DqModel each : dqModelRepo.getAll()) {
- allModels.add( converter.voOf(each));
- }
- return allModels;
- }
-
-
- @Override
- public int deleteModel(String name) throws BarkDbOperationException {
- try {
- DqModel dqModel = dqModelRepo.findByName(name);
-
- // TODO need to mark related metrics as deleted, instead of real deletion
- // markMetricsDeleted(dqModel);
-
- dqModelRepo.delete(dqModel.get_id());
-
- if (dqModel.getModelType() == ModelType.ACCURACY) {
- scheduleRepo.deleteByModelList(name);
- } else if (dqModel.getModelType() == ModelType.VALIDITY) {
- DBObject currentSchedule = scheduleRepo.getValiditySchedule(dqModel.getAssetId());
- if (currentSchedule == null || currentSchedule.get("modelList") == null) {
- return -1;
- }
-
- String rawModelList = currentSchedule.get("modelList") .toString();
- String newModelList = "";
- if (rawModelList.contains(ScheduleModelSeperator.SEPERATOR)) {
- String[] rawModelArray = rawModelList.split(ScheduleModelSeperator.SPLIT_SEPERATOR);
- for (int i = 0; i < rawModelArray.length; i++) {
- if (!rawModelArray[i].equals(name))
- newModelList = newModelList + ScheduleModelSeperator.SEPERATOR + rawModelArray[i];
- }
- newModelList = newModelList.substring(ScheduleModelSeperator.SEPERATOR.length());
- currentSchedule.put("modelList", newModelList);
- scheduleRepo.updateModelType(currentSchedule, dqModel.getModelType());
-
- } else if (rawModelList.equals(name)) {
- scheduleRepo.deleteByModelList(name);
- }
- }
-
- return 0;
- } catch (Exception e) {
- logger.warn(e.toString());
- throw new BarkDbOperationException("Failed to delete model of '"
- + name + "'", e);
- }
- }
-
- // FIXME to be removed
- @Override
- public DqModel getGeneralModel(String name) {
- return dqModelRepo.findByName(name);
- }
-
- @Override
- public ModelInput getModelByName(String name) throws BarkDbOperationException {
- try {
- DqModel dqModel = dqModelRepo.findByName(name);
- return modelInputConverter.voOf(dqModel);
- } catch (Exception e) {
- logger.error(e.toString());
- throw new BarkDbOperationException("Failed to find model with name of '" + name + "'", e);
- }
- }
-
- @Override
- public void enableSchedule4Model(DqModel dqModel) {
- if(dqModel==null) return;
-
- dqModel.setStatus(ModelStatus.DEPLOYED);
- dqModelRepo.update(dqModel);
-
- if (dqModel.getModelType() == ModelType.ACCURACY
- || dqModel.getModelType() == ModelType.VALIDITY) {
- DqSchedule schedule = new DqSchedule();
- schedule.set_id(scheduleRepo.getNextId());
- schedule.setScheduleType(dqModel.getSchedule());
- Date d = new Date(dqModel.getStarttime());
- d.setMinutes(0);
- d.setSeconds(0);
- schedule.setStarttime(d.getTime() / 1000 * 1000); // FIXME ????
- schedule.setStatus(0);
- schedule.setAssetId(dqModel.getAssetId());
- schedule.setJobType(dqModel.getModelType());
-
- String modellist = "";
- if (dqModel.getModelType() == ModelType.VALIDITY) {
- DBObject currentSchedule = scheduleRepo.getValiditySchedule(
- dqModel.getAssetId());
- if (currentSchedule != null) {
- if (currentSchedule.get("modelList") != null
- && !currentSchedule.get("modelList").toString().equals("")) {
- modellist = currentSchedule.get("modelList")
- .toString();
- modellist = modellist
- + ScheduleModelSeperator.SEPERATOR
- + dqModel.getModelName();
- }
- }
- if (modellist.equals(""))
- modellist = dqModel.getModelName();
- } else
- modellist = dqModel.getModelName();
- schedule.setModelList(modellist);
-
- scheduleRepo.updateByModelType(schedule, dqModel.getModelType());
- }
- }
-
- @Resource(name = "modelCreatorChain")
- DqModelCreator modelCreator;
-
- @Override
- public DqModel newModel(ModelInput input) throws BarkDbOperationException {
- return modelCreator.newModel(input);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/f629d0f4/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java b/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java
deleted file mode 100644
index 1cca030..0000000
--- a/griffin-core/src/main/java/com/ebay/oss/griffin/service/DqScheduleService.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- Copyright (c) 2016 eBay Software Foundation.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-package com.ebay.oss.griffin.service;
-
-
-
-/**
- * TODO: to decouple the schedule/job from model/metrics. schedule/job depends on model/metric, but
- * not vice versa.
- */
-public interface DqScheduleService {
-
- void schedulingJobs();
-
-}