You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/03/03 12:16:50 UTC
[44/51] [partial] incubator-griffin git commit: Init move
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/DataAssetServiceTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/DataAssetServiceTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DataAssetServiceTest.java
new file mode 100644
index 0000000..6a428b9
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DataAssetServiceTest.java
@@ -0,0 +1,168 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.*;
+
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.ebay.oss.griffin.domain.DataAsset;
+import com.ebay.oss.griffin.service.DataAssetService;
+import com.ebay.oss.griffin.vo.DataAssetInput;
+import com.ebay.oss.griffin.vo.PlatformMetadata;
+
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = {"classpath:context.xml"})
+public class DataAssetServiceTest {
+ @Autowired
+ private DataAssetService dataAssetService;
+
+ private String assetName = "testAsset_";
+ private String system = "Bullseye";
+ private String owner = "lliu13";
+ private String newOwner = "wenzhao";
+
+ @Test
+ public void testGetSourceTree() {
+ List<PlatformMetadata> pmdList = dataAssetService.getSourceTree();
+ assertNotNull(pmdList);
+ }
+
+ private List<DataAsset> testGetAllDataAssets() {
+ return dataAssetService.getAllDataAssets();
+ }
+
+ private DataAssetInput testCreateDataAsset() {
+ try {
+ DataAssetInput newData = new DataAssetInput();
+ newData.setAssetName(assetName + new Date().getTime());
+ newData.setAssetType("hdfsfile");
+ newData.setSystem(system);
+ newData.setAssetHDFSPath("/user/b_des/bark/");
+ newData.setPlatform("Apollo");
+ newData.setOwner(owner);
+ int success = dataAssetService.createDataAsset(newData);
+ assertEquals(success, 0);
+ System.out.println("add new data asset success");
+ return newData;
+ } catch (Exception e) {
+ System.out.println("fail to add new data asset");
+ e.printStackTrace();
+ return null;
+ }
+ }
+
+ private long getDataAssetByName(List<DataAsset> daList, String name) {
+ long newId = -1L;
+ Iterator<DataAsset> itr = daList.iterator();
+ while (itr.hasNext()) {
+ DataAsset da = itr.next();
+ if (da.getAssetName().equals(name)) {
+ newId = da.get_id();
+ break ;
+ }
+ }
+ assertTrue(newId > 0);
+ System.out.println("get new add data asset id: " + newId);
+ return newId;
+ }
+
+ private void testGetDataAsset(long id, String sys, String onr) {
+ try {
+ //get that data asset
+ DataAsset tda = dataAssetService.getDataAssetById(id);
+ assertNotNull(tda);
+ assertEquals(tda.getSystem(), sys);
+ assertEquals(tda.getOwner(), onr);
+ System.out.println("find id " + id + " success");
+ } catch (Exception e) {
+ System.out.println("fail to find data asset");
+ e.printStackTrace();
+ }
+ }
+
+ private void testGetNonExistDataAsset(long id) {
+ try {
+ DataAsset tda = dataAssetService.getDataAssetById(id);
+ assertNull(tda);
+ System.out.println("can not find non-exist data asset");
+ } catch (Exception e) {
+ System.out.println("fail to find non-exist data asset");
+ e.printStackTrace();
+ }
+ }
+
+ private void updateDataAsset(DataAssetInput da, String newOwner) {
+ try {
+ da.setOwner(newOwner);
+ dataAssetService.updateDataAsset(da);
+ System.out.println("update data asset success");
+ } catch (Exception e) {
+ System.out.println("fail to update data asset");
+ e.printStackTrace();
+ }
+ }
+
+ private void testRemoveDataAsset(long id) {
+ try {
+ dataAssetService.removeAssetById(id);
+
+ DataAsset tda2 = dataAssetService.getDataAssetById(id);
+ assertNull(tda2);
+ System.out.println("remove the data asset " + id + " success");
+ } catch (Exception e) {
+ System.out.println("fail to remove data asset");
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testDataAssetService(){
+
+ //get all data
+ List<DataAsset> daList = testGetAllDataAssets();
+ System.out.println("current data assets count: " + daList.size());
+
+ //add new dataAsset
+ DataAssetInput newData = testCreateDataAsset();
+ assertNotNull(newData);
+
+ //get all data again
+ List<DataAsset> daList1 = testGetAllDataAssets();
+ assertTrue(daList1.size() - daList.size() == 1);
+ System.out.println("current data assets count: " + daList1.size());
+
+ //find new data asset id
+ long id = getDataAssetByName(daList1, assetName);
+ testGetDataAsset(id, system, owner);
+
+ //find non-exist data asset
+ testGetNonExistDataAsset(863510L);
+
+ //update exist data asset
+ updateDataAsset(newData, newOwner);
+ testGetDataAsset(id, system, newOwner);
+
+ //update non-exist data asset
+ DataAssetInput newData1 = new DataAssetInput();
+ newData1.setAssetName("test_newDataAsset");
+ newData1.setAssetType("hdfsfile");
+ newData1.setSystem(system);
+ newData1.setAssetHDFSPath("/user/b_des/bark/");
+ newData1.setPlatform("Apollo");
+ newData1.setOwner(owner);
+ updateDataAsset(newData1, newOwner);
+
+ //remove the data asset
+ testRemoveDataAsset(id);
+
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelConverterTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelConverterTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelConverterTest.java
new file mode 100644
index 0000000..703ab7e
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelConverterTest.java
@@ -0,0 +1,19 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+import com.ebay.oss.griffin.service.DqModelConverter;
+
+
+public class DqModelConverterTest {
+
+ private DqModelConverter converter;
+
+ @Test
+ public void test_voOf_null() {
+ converter = new DqModelConverter();
+ assertNull(converter.voOf(null));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelServiceTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelServiceTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelServiceTest.java
new file mode 100644
index 0000000..8976776
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqModelServiceTest.java
@@ -0,0 +1,213 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.*;
+
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.ebay.oss.griffin.domain.AnomalyType;
+import com.ebay.oss.griffin.domain.DqModel;
+import com.ebay.oss.griffin.domain.ModelStatus;
+import com.ebay.oss.griffin.domain.ModelType;
+import com.ebay.oss.griffin.domain.ScheduleType;
+import com.ebay.oss.griffin.domain.SystemType;
+import com.ebay.oss.griffin.domain.ValidityType;
+import com.ebay.oss.griffin.service.DqModelService;
+import com.ebay.oss.griffin.vo.DqModelVo;
+import com.ebay.oss.griffin.vo.ModelBasicInputNew;
+import com.ebay.oss.griffin.vo.ModelExtraInputNew;
+import com.ebay.oss.griffin.vo.ModelInput;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = {"classpath:context.xml"})
+public class DqModelServiceTest {
+
+ @Autowired
+ private DqModelService dqModelService;
+
+ private String[] newModelName = {"testAccu", "testVali", "testAnom", "testPub"};
+ private String testOwner = "lliu13";
+ private String testPlatform = "Apollo";
+
+ public void testNewModel(int type) {
+ DqModel ret = null;
+
+ if (type == ModelType.ACCURACY) {
+ ModelInput tempModel = new ModelInput();
+ ModelBasicInputNew basic = new ModelBasicInputNew();
+ ModelExtraInputNew extra = new ModelExtraInputNew();
+ basic.setDataaset("rheos_view_event");
+ basic.setDataasetId(16);
+ basic.setDesc("accu for rheos_view_event");
+ basic.setEmail(testOwner + "@ebay.com");
+ basic.setName(newModelName[0]);
+ basic.setOwner(testOwner);
+ basic.setScheduleType(ScheduleType.DAILY);
+ basic.setStatus(ModelStatus.DEPLOYED);
+ basic.setSystem(SystemType.KAFKA);
+ basic.setType(type);
+ extra.setVaType(ValidityType.TOTAL_COUNT);
+ extra.setColumn("u_id");
+ extra.setSrcDataSet("Sojourner.soj_view_event");
+ extra.setSrcDb(testPlatform);
+ extra.setTargetDataSet("Kafka.rheos_view_event");
+ extra.setTargetDb(testPlatform);
+ tempModel.setBasic(basic);
+ tempModel.setExtra(extra);
+ ret = dqModelService.newModel(tempModel);
+ } else if (type == ModelType.VALIDITY) {
+ ModelInput tempModel = new ModelInput();
+ ModelBasicInputNew basic = new ModelBasicInputNew();
+ ModelExtraInputNew extra = new ModelExtraInputNew();
+ basic.setDataaset("dmg");
+ basic.setDataasetId(6);
+ basic.setDesc("total count for dmg");
+ basic.setEmail(testOwner + "@ebay.com");
+ basic.setName(newModelName[1]);
+ basic.setOwner(testOwner);
+ basic.setScheduleType(ScheduleType.DAILY);
+ basic.setStatus(ModelStatus.DEPLOYED);
+ basic.setSystem(SystemType.BULLSEYE);
+ basic.setType(type);
+ extra.setVaType(ValidityType.TOTAL_COUNT);
+ extra.setColumn("age");
+ extra.setSrcDataSet("Bullseye.dmg");
+ extra.setSrcDb(testPlatform);
+ tempModel.setBasic(basic);
+ tempModel.setExtra(extra);
+ ret = dqModelService.newModel(tempModel);
+ } else if (type == ModelType.ANOMALY) {
+ ModelInput tempModel = new ModelInput();
+ ModelBasicInputNew basic = new ModelBasicInputNew();
+ ModelExtraInputNew extra = new ModelExtraInputNew();
+ basic.setDataaset("dmg");
+ basic.setDataasetId(6);
+ basic.setDesc("anomaly for dmg");
+ basic.setEmail(testOwner + "@ebay.com");
+ basic.setName(newModelName[2]);
+ basic.setOwner(testOwner);
+ basic.setScheduleType(ScheduleType.DAILY);
+ basic.setStatus(ModelStatus.DEPLOYED);
+ basic.setSystem(SystemType.BULLSEYE);
+ basic.setType(type);
+ extra.setAnType(AnomalyType.ANOMALY_DEFAULT);
+ extra.setColumn("age");
+ extra.setSrcDataSet("Bullseye.dmg");
+ extra.setSrcDb(testPlatform);
+ tempModel.setBasic(basic);
+ tempModel.setExtra(extra);
+ ret = dqModelService.newModel(tempModel);
+ } else if (type == ModelType.PUBLISH) {
+ ModelInput tempModel = new ModelInput();
+ ModelBasicInputNew basic = new ModelBasicInputNew();
+ ModelExtraInputNew extra = new ModelExtraInputNew();
+ basic.setDataaset("dmg");
+ basic.setDataasetId(6);
+ basic.setDesc("publish metrics for dmg");
+ basic.setEmail(testOwner + "@ebay.com");
+ basic.setName(newModelName[3]);
+ basic.setOwner(testOwner);
+ basic.setScheduleType(ScheduleType.DAILY);
+ basic.setStatus(ModelStatus.DEPLOYED);
+ basic.setSystem(SystemType.BULLSEYE);
+ basic.setType(type);
+ extra.setPublishUrl("http://www.test.com");
+ extra.setColumn("age");
+ extra.setSrcDataSet("Bullseye.dmg");
+ extra.setSrcDb(testPlatform);
+ tempModel.setBasic(basic);
+ tempModel.setExtra(extra);
+ ret = dqModelService.newModel(tempModel);
+ }
+ assertNotNull(ret);
+ }
+
+ private List<DqModelVo> testGetAllModels() {
+ return dqModelService.getAllModles();
+ }
+
+ private void testGetModels() {
+ try {
+ //find success
+ for (int i = 1; i < 4; i++) {
+ ModelInput mi = dqModelService.getModelByName(newModelName[i]);
+ if (i == 1 || i == 2) {
+ assertEquals(mi.getBasic().getOwner(), testOwner);
+ assertEquals(mi.getExtra().getSrcDb(), testPlatform);
+ System.out.println("get model by name: " + mi.getBasic().getName()
+ + " with owner: " + mi.getBasic().getOwner()
+ + " and platform: " + mi.getExtra().getSrcDb());
+ } else {
+ assertEquals(mi.getBasic().getOwner(), testOwner);
+ assertEquals(mi.getExtra().getPublishUrl(), "http://www.test.com");
+ System.out.println("get model by name: " + mi.getBasic().getName()
+ + " with owner: " + mi.getBasic().getOwner()
+ + " and publish url: " + mi.getExtra().getPublishUrl());
+ }
+ }
+ //find fail
+ ModelInput mi = dqModelService.getModelByName("findfailmodel");
+ assertNull(mi);
+ } catch (Exception e) {
+ System.out.println("fail to get model");
+ e.printStackTrace();
+ }
+ }
+
+ private void testDeleteModel(String name) {
+ try {
+ int del = dqModelService.deleteModel(name);
+ assertTrue(del <= 0);
+ System.out.println("delete model success");
+ } catch (Exception e) {
+ System.out.println("fail to delete model");
+ System.out.println(e.getMessage());
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void testDQModelService(){
+
+ //get all models
+ List<DqModelVo> modelsList = testGetAllModels();
+ System.out.println("current models count: " + modelsList.size());
+
+ //add new models success
+ testNewModel(ModelType.ACCURACY);
+ testNewModel(ModelType.VALIDITY);
+ testNewModel(ModelType.ANOMALY);
+ testNewModel(ModelType.PUBLISH);
+
+ //get all models
+ List<DqModelVo> modelsList1 = testGetAllModels();
+ assertTrue(modelsList1.size() - modelsList.size() == 4);
+ System.out.println("current models count: " + modelsList1.size());
+
+ //add the same new model fail
+ testNewModel(ModelType.VALIDITY);
+
+ //find new add model
+ testGetModels();
+
+ //delete models
+ //TODO:: move to tearDown
+ for (int i = 0; i < newModelName.length; i++){
+ testDeleteModel(newModelName[i]);
+ }
+ }
+
+ @Test
+ public void testGtGeneralModeltAndEnableSchedule4Model() {
+ String name = "test_accuracy_1";
+ DqModel me = dqModelService.getGeneralModel(name);
+ assertNotNull(me);
+
+ dqModelService.enableSchedule4Model(me);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqScheduleServiceTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqScheduleServiceTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqScheduleServiceTest.java
new file mode 100644
index 0000000..2249c66
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/DqScheduleServiceTest.java
@@ -0,0 +1,24 @@
+package com.ebay.oss.griffin.service;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.ebay.oss.griffin.service.DqScheduleService;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = {"classpath:context.xml"})
+public class DqScheduleServiceTest {
+ @Autowired
+ private DqScheduleService dqJobSchedulingService;
+
+ @Test
+ public void testSchedulingJobs(){
+
+ dqJobSchedulingService.schedulingJobs();
+ System.out.println("scheduling jobs success");
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/LoginServiceTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/LoginServiceTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/LoginServiceTest.java
new file mode 100644
index 0000000..b3d29bf
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/LoginServiceTest.java
@@ -0,0 +1,34 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.ebay.oss.griffin.service.LoginService;
+
+
+
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = {"classpath:context.xml"})
+public class LoginServiceTest {
+ @Autowired
+ private LoginService loginService;
+
+ @Test
+ public void testLogin(){
+
+ //login success
+ String fullname = loginService.login("alex", "alex");
+ assertEquals(fullname, "alex");
+
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/NotificationServiceImplTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/NotificationServiceImplTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/NotificationServiceImplTest.java
new file mode 100644
index 0000000..f965388
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/NotificationServiceImplTest.java
@@ -0,0 +1,110 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.*;
+
+import java.util.Date;
+import java.util.List;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import com.ebay.oss.griffin.service.NotificationService;
+import com.ebay.oss.griffin.vo.NotificationRecord;
+
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = {"classpath:context.xml"})
+public class NotificationServiceImplTest {
+
+ @Autowired
+ private NotificationService notificationService;
+
+ private List<NotificationRecord> testGetAll() {
+ return notificationService.getAll();
+ }
+
+ private void testInsert(int n) {
+ System.out.println("===== Insert " + n + " Notification Records =====");
+ for (int i = 1; i <= n; i++) {
+ NotificationRecord rcd = new NotificationRecord(new Date().getTime(), "lliu13", "Operation" + i, "Target" + i, "NotifyRecord" + i);
+ notificationService.insert(rcd);
+ }
+ System.out.println();
+ }
+
+ private NotificationRecord testGet(int id, boolean expectNull) {
+ NotificationRecord fnr = notificationService.get(id);
+ if (expectNull) {
+ assertNull(fnr);
+ } else {
+ assertNotNull(fnr);
+ System.out.println("get notification record: " + fnr.getName() + ": " + fnr.getTarget() + " " + fnr.getOperation());
+ }
+ return fnr;
+ }
+
+ private void testGetTop(int n) {
+ System.out.println("===== Get Top " + n + " Notification Record =====");
+ List<NotificationRecord> fnrList = notificationService.getTop(n);
+ assertEquals(fnrList.size(), n);
+ for (NotificationRecord nr : fnrList) {
+ System.out.println(nr.getName() + ": " + nr.getTarget() + " " + nr.getOperation());
+ }
+ System.out.println();
+ }
+
+ private void testDelete(NotificationRecord nfr) {
+ notificationService.delete(nfr);
+ }
+ private void testDelete(int id) {
+ notificationService.delete(id);
+ }
+
+ @Test
+ public void TestNotificationService() {
+
+ //getAll
+ System.out.println("===== All Notification Records 0 =====");
+ List<NotificationRecord> nrList = testGetAll();
+ int sz = nrList.size();
+ System.out.println("current notification records list size: " + sz);
+ System.out.println();
+
+ //insert
+ int nInsert = 7;
+ testInsert(nInsert);
+
+ //getAll
+ System.out.println("===== All Notification Records 1 =====");
+ int sz1 = nrList.size();
+ assertEquals(sz1 - sz, nInsert);
+ System.out.println("current notification records list size: " + sz1);
+ System.out.println();
+
+ //get
+ System.out.println("===== Get Notification Record =====");
+ NotificationRecord fnr = testGet(3, false);
+ testGet(9009, true);
+
+ //getTop
+ testGetTop(5);
+
+ //delete
+ System.out.println("===== delete 2 Notification Records =====");
+ testDelete(fnr);
+ testDelete(6);
+ NotificationRecord nrcd = new NotificationRecord(new Date().getTime(), "lliu13", "Operation", "Target", "NotifyRecord");
+ testDelete(nrcd);
+ System.out.println();
+
+ //getAll
+ System.out.println("===== All Notification Records 2 =====");
+ int sz2 = nrList.size();
+ assertEquals(sz1 - sz2, 2);
+ System.out.println("current notification records list size: " + sz2);
+ System.out.println();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/RefMetricsCalcImplTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/RefMetricsCalcImplTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/RefMetricsCalcImplTest.java
new file mode 100644
index 0000000..feda65f
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/RefMetricsCalcImplTest.java
@@ -0,0 +1,65 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.ebay.oss.griffin.domain.DqModel;
+import com.ebay.oss.griffin.repo.DqModelRepo;
+import com.ebay.oss.griffin.service.RefMetricsCalcImpl;
+
+
+public class RefMetricsCalcImplTest {
+
+ private RefMetricsCalcImpl calc;
+
+ @Before
+ public void setUp() {
+ calc = new RefMetricsCalcImpl();
+ }
+
+ @SuppressWarnings("serial")
+ @Test
+ public void test_getReferences() {
+ calc.modelRepo = mock(DqModelRepo.class);
+
+ List<DqModel> models = new ArrayList<>();
+ models.add(newModelWithRef("model1", "ref1"));
+ models.add(newModelWithRef("model2", "ref2_0, ref2_1, ref2_2"));
+ models.add(newModelWithRef("model2", null));
+ models.add(newModelWithRef("model2", ""));
+ when(calc.modelRepo.getAll()).thenReturn(models);
+
+ Map<String, List<String>> expect = new HashMap<String, List<String>>() {{
+ this.put("model1", new ArrayList<String>() {{
+ this.add("ref1");
+ }});
+ this.put("model2", new ArrayList<String>() {{
+ this.add("ref2_0");
+ this.add("ref2_1");
+ this.add("ref2_2");
+ }});
+ }};
+
+ Map<String, List<String>> actual = calc.getReferences();
+
+ assertEquals(expect, actual);
+
+ verify(calc.modelRepo).getAll();
+ }
+
+ private DqModel newModelWithRef(String name, String ref) {
+ DqModel model = new DqModel();
+ model.setModelName(name);
+ model.setReferenceModel(ref);
+ return model;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/service/SubscribeServiceImplTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/service/SubscribeServiceImplTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/service/SubscribeServiceImplTest.java
new file mode 100644
index 0000000..11ecb4a
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/service/SubscribeServiceImplTest.java
@@ -0,0 +1,57 @@
+package com.ebay.oss.griffin.service;
+
+import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
+
+import java.util.ArrayList;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import com.ebay.oss.griffin.domain.UserSubscription;
+import com.ebay.oss.griffin.repo.UserSubscriptionRepo;
+import com.ebay.oss.griffin.service.SubscribeServiceImpl;
+import com.ebay.oss.griffin.vo.PlatformSubscription;
+
+
+public class SubscribeServiceImplTest {
+
+ private SubscribeServiceImpl svc;
+
+ String testUser = "lliu13", testId = "testSubscribe";
+
+ @Before
+ public void setUp() {
+ svc = new SubscribeServiceImpl();
+ svc.subscriptionRepo = mock(UserSubscriptionRepo.class);
+ }
+
+ @Test
+ public void test_subscribe() {
+
+
+ UserSubscription usi = new UserSubscription();
+ usi.setNtaccount(testUser);
+ usi.setId(testId);
+ usi.setSubscribes(new ArrayList<PlatformSubscription>());
+
+ doNothing().when(svc.subscriptionRepo).upsertUserSubscribe(usi);
+
+ svc.subscribe(usi);
+
+ verify(svc.subscriptionRepo).upsertUserSubscribe(usi);
+ }
+
+ @Test
+ public void test_getSubscribe() {
+ UserSubscription expect = mock(UserSubscription.class);
+ when(svc.subscriptionRepo.getUserSubscribeItem(testUser)).thenReturn(expect);
+
+ UserSubscription actual = svc.getSubscribe(testUser);
+
+ assertNotNull(actual);
+ assertSame(expect, actual);
+ verify(svc.subscriptionRepo).getUserSubscribeItem(testUser);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/java/com/ebay/oss/griffin/vo/VoResourceTest.java
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/java/com/ebay/oss/griffin/vo/VoResourceTest.java b/griffin-core/src/test/java/com/ebay/oss/griffin/vo/VoResourceTest.java
new file mode 100644
index 0000000..45e1603
--- /dev/null
+++ b/griffin-core/src/test/java/com/ebay/oss/griffin/vo/VoResourceTest.java
@@ -0,0 +1,651 @@
+package com.ebay.oss.griffin.vo;
+
+import com.ebay.oss.griffin.domain.*;
+import com.ebay.oss.griffin.vo.AccuracyHiveJobConfig;
+import com.ebay.oss.griffin.vo.AccuracyHiveJobConfigDetail;
+import com.ebay.oss.griffin.vo.AssetLevelMetrics;
+import com.ebay.oss.griffin.vo.AssetLevelMetricsDetail;
+import com.ebay.oss.griffin.vo.BaseObj;
+import com.ebay.oss.griffin.vo.BollingerBandsEntity;
+import com.ebay.oss.griffin.vo.DQHealthStats;
+import com.ebay.oss.griffin.vo.DataAssetIndex;
+import com.ebay.oss.griffin.vo.DataAssetInput;
+import com.ebay.oss.griffin.vo.DqModelVo;
+import com.ebay.oss.griffin.vo.LoginUser;
+import com.ebay.oss.griffin.vo.MADEntity;
+import com.ebay.oss.griffin.vo.MappingItemInput;
+import com.ebay.oss.griffin.vo.ModelBasicInputNew;
+import com.ebay.oss.griffin.vo.ModelExtraInputNew;
+import com.ebay.oss.griffin.vo.ModelInput;
+import com.ebay.oss.griffin.vo.NotificationRecord;
+import com.ebay.oss.griffin.vo.OverViewStatistics;
+import com.ebay.oss.griffin.vo.PartitionConfig;
+import com.ebay.oss.griffin.vo.PlatformMetadata;
+import com.ebay.oss.griffin.vo.PlatformSubscription;
+import com.ebay.oss.griffin.vo.SampleOut;
+import com.ebay.oss.griffin.vo.SystemLevelMetrics;
+import com.ebay.oss.griffin.vo.SystemLevelMetricsList;
+import com.ebay.oss.griffin.vo.SystemMetadata;
+import com.ebay.oss.griffin.vo.SystemSubscription;
+import com.ebay.oss.griffin.vo.ValidateHiveJobConfig;
+import com.ebay.oss.griffin.vo.ValidateHiveJobConfigLv1Detail;
+import com.ebay.oss.griffin.vo.ValidateHiveJobConfigLv2Detail;
+
+import org.junit.Test;
+
+import java.util.*;
+
+import static org.junit.Assert.*;
+
+public class VoResourceTest {
+
+ @Test
+ public void testAccuracyHiveJobConfigDetail() {
+ AccuracyHiveJobConfigDetail detail = new AccuracyHiveJobConfigDetail();
+
+ detail.setSourceColId(1);
+ assertEquals(1, detail.getSourceColId());
+ detail.setSourceColName("source");
+ assertEquals("source", detail.getSourceColName());
+ detail.setTargetColId(2);
+ assertEquals(2, detail.getTargetColId());
+ detail.setTargetColName("target");
+ assertEquals("target", detail.getTargetColName());
+ detail.setMatchFunction("function");
+ assertEquals("function", detail.getMatchFunction());
+ detail.setPK(true);
+ assertEquals(true, detail.isPK());
+ }
+
+ @Test
+ public void testPartitionConfig() {
+ PartitionConfig detail = new PartitionConfig();
+
+ detail.setColName("name");
+ assertEquals("name", detail.getColName());
+ detail.setColValue("value");
+ assertEquals("value", detail.getColValue());
+ }
+
+ @Test
+ public void testAccuracyHiveJobConfig() {
+ AccuracyHiveJobConfig config = new AccuracyHiveJobConfig();
+ AccuracyHiveJobConfig config1 = new AccuracyHiveJobConfig("source", "target");
+
+ config.setSource("source");
+ assertEquals("source", config.getSource());
+ config.setTarget("target");
+ assertEquals("target", config.getTarget());
+
+ List<AccuracyHiveJobConfigDetail> detailList = new ArrayList<AccuracyHiveJobConfigDetail>();
+ detailList.add(new AccuracyHiveJobConfigDetail(1, "src", 2, "tgt", "func", false));
+ config.setAccuracyMapping(detailList);
+ assertEquals(detailList, config.getAccuracyMapping());
+
+ List<PartitionConfig> srcPtConfigList = new ArrayList<PartitionConfig>();
+ srcPtConfigList.add(new PartitionConfig("name", "value"));
+ config.setSrcPartitions(srcPtConfigList);
+ assertEquals(srcPtConfigList, config.getSrcPartitions());
+
+ List<PartitionConfig> l1 = new ArrayList<PartitionConfig>();
+ List<List<PartitionConfig>> tgtPtConfigList = new ArrayList<List<PartitionConfig>>();
+ tgtPtConfigList.add(l1);
+ config.setTgtPartitions(tgtPtConfigList);
+ assertEquals(tgtPtConfigList, config.getTgtPartitions());
+ }
+
+ @Test
+ public void testAssetLevelMetrics() {
+ AssetLevelMetrics metrics = new AssetLevelMetrics();
+ AssetLevelMetrics metrics1 = new AssetLevelMetrics("name");
+ AssetLevelMetrics metrics2 = new AssetLevelMetrics("name", 12.3f, 12345L);
+ AssetLevelMetrics metrics3 = new AssetLevelMetrics("name", "type", 12.3f, 12345L, 30);
+
+
+ metrics.setName("name");
+ assertEquals("name", metrics.getName());
+ metrics.setDq(12.3f);
+ assertTrue(12.3f == metrics.getDq());
+ metrics.setTimestamp(12345L);
+ assertEquals(12345L, metrics.getTimestamp());
+ metrics.setDqfail(30);
+ assertEquals(30, metrics.getDqfail());
+
+ List<AssetLevelMetricsDetail> detailList = new ArrayList<AssetLevelMetricsDetail>();
+ metrics.setDetails(detailList);
+ metrics.addAssetLevelMetricsDetail(new AssetLevelMetricsDetail());
+ assertEquals(detailList, metrics.getDetails());
+
+ metrics.setMetricType("type");
+ assertEquals("type", metrics.getMetricType());
+ metrics.setAssetName("assetName");
+ assertEquals("assetName", metrics.getAssetName());
+
+ AssetLevelMetrics metrics4 = new AssetLevelMetrics(metrics, 2);
+ }
+
+ @Test
+ public void testAssetLevelMetricsDetail() {
+ AssetLevelMetricsDetail detail = new AssetLevelMetricsDetail();
+
+ detail.setTimestamp(12345L);
+ assertEquals(12345L, detail.getTimestamp());
+ detail.setValue(12.3f);
+ assertTrue(12.3f == detail.getValue());
+
+ BollingerBandsEntity bbe = new BollingerBandsEntity(200L, 100L, 150L);
+ detail.setBolling(bbe);
+ assertEquals(bbe, detail.getBolling());
+
+ detail.setComparisionValue(12.3f);
+ assertTrue(12.3f == detail.getComparisionValue());
+
+ MADEntity mad = new MADEntity(200L, 100L);
+ detail.setMAD(mad);
+ assertEquals(mad, detail.getMAD());
+
+ AssetLevelMetricsDetail detail1 = new AssetLevelMetricsDetail(12345L, 25.3f);
+ assertEquals(0, detail1.compareTo(detail));
+
+ detail1.setTimestamp(12346L);
+ assertEquals(-1, detail1.compareTo(detail));
+ detail1.setTimestamp(12344L);
+ assertEquals(1, detail1.compareTo(detail));
+
+ AssetLevelMetricsDetail detail2 = new AssetLevelMetricsDetail(12345L, 25.3f, bbe);
+ AssetLevelMetricsDetail detail3 = new AssetLevelMetricsDetail(12345L, 25.3f, 12.3f);
+ AssetLevelMetricsDetail detail4 = new AssetLevelMetricsDetail(12345L, 25.3f, mad);
+ AssetLevelMetricsDetail detail5 = new AssetLevelMetricsDetail(bbe);
+ AssetLevelMetricsDetail detail6 = new AssetLevelMetricsDetail(mad);
+ AssetLevelMetricsDetail detail7 = new AssetLevelMetricsDetail(detail);
+ AssetLevelMetricsDetail detail8 = new AssetLevelMetricsDetail(12.3f);
+ }
+
+ @Test
+ public void testBaseObj() {
+ BaseObj base = new BaseObj();
+ assertNull(base.validate());
+ }
+
+ @Test
+ public void testBollingerBandsEntity() {
+ BollingerBandsEntity entity = new BollingerBandsEntity();
+
+ entity.setUpper(300L);
+ assertEquals(300L, entity.getUpper());
+ entity.setLower(100L);
+ assertEquals(100L, entity.getLower());
+ entity.setMean(200L);
+ assertEquals(200L, entity.getMean());
+ }
+
+ @Test
+ public void testDataAssetIndex() {
+ DataAssetIndex dai = new DataAssetIndex();
+
+ dai.setId(12345L);
+ assertEquals(new Long(12345L), dai.getId());
+ dai.setName("name");
+ assertEquals("name", dai.getName());
+ }
+
+ @Test
+ public void testDataAssetInput() {
+ DataAssetInput input = new DataAssetInput();
+
+ input.setAssetName("aname");
+ assertEquals("aname", input.getAssetName());
+ input.setAssetType("atype");
+ assertEquals("atype", input.getAssetType());
+ input.setAssetHDFSPath("apath");
+ assertEquals("apath", input.getAssetHDFSPath());
+ input.setSystem("system");
+ assertEquals("system", input.getSystem());
+ input.setPlatform("platform");
+ assertEquals("platform", input.getPlatform());
+
+ List<DataSchema> dsList = new ArrayList<DataSchema>();
+ dsList.add(new DataSchema("sname", "stype", "sdesc", "ssample"));
+ input.setSchema(dsList);
+ assertEquals(dsList, input.getSchema());
+
+ input.setPartition("patition");
+ assertEquals("patition", input.getPartition());
+
+ List<PartitionFormat> pfList = new ArrayList<PartitionFormat>();
+ pfList.add(new PartitionFormat("pfname", "pfformat"));
+ input.setPartitions(pfList);
+ assertEquals(pfList, input.getPartitions());
+
+ input.setOwner("owner");
+ assertEquals("owner", input.getOwner());
+ }
+
+ @Test
+ public void testDQHealthStats() {
+ DQHealthStats stats = new DQHealthStats();
+
+ stats.setHealth(20);
+ assertEquals(20, stats.getHealth());
+ stats.setWarn(30);
+ assertEquals(30, stats.getWarn());
+ stats.setInvalid(10);
+ assertEquals(10, stats.getInvalid());
+ }
+
+ @Test
+ public void testDqModelVo() {
+ DqModelVo vo = new DqModelVo();
+
+ vo.setAssetName("assetname");
+ assertEquals("assetname", vo.getAssetName());
+ vo.setName("name");
+ assertEquals("name", vo.getName());
+ vo.setSystem(3);
+ assertEquals(3, vo.getSystem());
+ vo.setDescription("desc");
+ assertEquals("desc", vo.getDescription());
+ vo.setType(2);
+ assertEquals(2, vo.getType());
+ vo.setOwner("owner");
+ assertEquals("owner", vo.getOwner());
+ Date date = new Date();
+ vo.setCreateDate(date);
+ assertEquals(date, vo.getCreateDate());
+ vo.setStatus("status");
+ assertEquals("status", vo.getStatus());
+ }
+
+ @Test
+ public void testLoginUser() {
+ LoginUser user = new LoginUser();
+
+ user.setUsername("name");
+ assertEquals("name", user.getUsername());
+ user.setPassword("pwd");
+ assertEquals("pwd", user.getPassword());
+
+ LoginUser user1 = new LoginUser("name1", "pwd1");
+ }
+
+ @Test
+ public void testMADEntity() {
+ MADEntity mad = new MADEntity();
+
+ mad.setUpper(300L);
+ assertEquals(300L, mad.getUpper());
+ mad.setLower(100L);
+ assertEquals(100L, mad.getLower());
+
+ MADEntity mad1 = new MADEntity(300L, 100L);
+
+ MADEntity mad2 = mad1.clone();
+ assertEquals(300L, mad2.getUpper());
+ assertEquals(100L, mad2.getLower());
+ }
+
+ @Test
+ public void testMappingItemInput() {
+ MappingItemInput input = new MappingItemInput();
+
+ input.setTarget("target");
+ assertEquals("target", input.getTarget());
+ input.setSrc("source");
+ assertEquals("source", input.getSrc());
+ input.setIsPk(true);
+ assertEquals(true, input.isIsPk());
+ input.setMatchMethod("matchMethod");
+ assertEquals("matchMethod", input.getMatchMethod());
+
+ MappingItemInput input1 = new MappingItemInput("target", "source", true, "matchMethod");
+ }
+
+ @Test
+ public void testModelBasicInputNew() {
+ ModelBasicInputNew inputNew = new ModelBasicInputNew();
+
+ inputNew.setType(1);
+ assertEquals(1, inputNew.getType());
+ inputNew.setSystem(2);
+ assertEquals(2, inputNew.getSystem());
+ inputNew.setScheduleType(3);
+ assertEquals(3, inputNew.getScheduleType());
+ inputNew.setOwner("owner");
+ assertEquals("owner", inputNew.getOwner());
+ inputNew.setName("name");
+ assertEquals("name", inputNew.getName());
+ inputNew.setDesc("desc");
+ assertEquals("desc", inputNew.getDesc());
+ inputNew.setEmail("email");
+ assertEquals("email", inputNew.getEmail());
+ inputNew.setDataaset("dataAsset");
+ assertEquals("dataAsset", inputNew.getDataaset());
+ inputNew.setDataasetId(12345L);
+ assertEquals(12345L, inputNew.getDataasetId());
+ inputNew.setThreshold(20.2f);
+ assertTrue(20.2f == inputNew.getThreshold());
+ inputNew.setStatus(1);
+ assertEquals(1, inputNew.getStatus());
+ inputNew.setStarttime(12345L);
+ assertEquals(12345L, inputNew.getStarttime());
+ }
+
+ @Test
+ public void testModelExtraInputNew() {
+ ModelExtraInputNew inputNew = new ModelExtraInputNew();
+
+ inputNew.setSrcDb("db");
+ assertEquals("db", inputNew.getSrcDb());
+ inputNew.setSrcDataSet("dataSet");
+ assertEquals("dataSet", inputNew.getSrcDataSet());
+ inputNew.setTargetDb("db");
+ assertEquals("db", inputNew.getTargetDb());
+ inputNew.setTargetDataSet("dataSet");
+ assertEquals("dataSet", inputNew.getTargetDataSet());
+ inputNew.setVaType(1);
+ assertEquals(1, inputNew.getVaType());
+ inputNew.setColumn("col");
+ assertEquals("col", inputNew.getColumn());
+ inputNew.setPublishUrl("url");
+ assertEquals("url", inputNew.getPublishUrl());
+ inputNew.setAnType(2);
+ assertEquals(2, inputNew.getAnType());
+ }
+
+ @Test
+ public void testModelInput() {
+ ModelInput input = new ModelInput();
+
+ ModelBasicInputNew basic = new ModelBasicInputNew();
+ input.setBasic(basic);
+ assertEquals(basic, input.getBasic());
+
+ ModelExtraInputNew extra = new ModelExtraInputNew();
+ input.setExtra(extra);
+ assertEquals(extra, input.getExtra());
+
+ List<MappingItemInput> mappings = new ArrayList<MappingItemInput>();
+ MappingItemInput mi = new MappingItemInput("target", "source", true, "matchMethod");
+ mappings.add(mi);
+ input.setMappings(mappings);
+ assertEquals(mappings, input.getMappings());
+
+ ModelInput input1 = new ModelInput(basic, extra, mappings);
+
+ ModelInput input2 = new ModelInput();
+ String content = "srcDb|srcDataSet|targetDb|targetDataSet|id,uid,true,null;name,uname,false,null;age,age,false,null";
+ input2.parseFromString(content);
+ assertEquals("srcDb", input2.getExtra().getSrcDb());
+ assertEquals("srcDataSet", input2.getExtra().getSrcDataSet());
+ assertEquals("targetDb", input2.getExtra().getTargetDb());
+ assertEquals("targetDataSet", input2.getExtra().getTargetDataSet());
+ assertEquals(3, input2.getMappings().size());
+
+ //field basic has notNull field name, so the validate result can not be null
+ assertNotNull(input2.validate());
+ }
+
+ @Test
+ public void testNotificationRecord() {
+ NotificationRecord rcd = new NotificationRecord();
+
+ rcd.setName("name");
+ assertEquals("name", rcd.getName());
+ rcd.setId(12);
+ assertEquals(12, rcd.getId());
+ rcd.setTimestamp(12345L);
+ assertEquals(12345L, rcd.getTimestamp());
+ rcd.setOwner("owner");
+ assertEquals("owner", rcd.getOwner());
+ rcd.setOperation("opr");
+ assertEquals("opr", rcd.getOperation());
+ rcd.setTarget("target");
+ assertEquals("target", rcd.getTarget());
+ rcd.setLink("link");
+ assertEquals("link", rcd.getLink());
+
+ NotificationRecord rcd1 = new NotificationRecord(12345L, "owner", "opr", "target", "name");
+ }
+
+ @Test
+ public void testOverViewStatistics() {
+ OverViewStatistics stat = new OverViewStatistics();
+
+ stat.setAssets(2);
+ assertEquals(2, stat.getAssets());
+ stat.setMetrics(4);
+ assertEquals(4, stat.getMetrics());
+ DQHealthStats dqs = new DQHealthStats();
+ stat.setStatus(dqs);
+ assertEquals(dqs, stat.getStatus());
+ }
+
+ @Test
+ public void testPlatformMetadata() {
+ PlatformMetadata metaData = new PlatformMetadata();
+
+ metaData.setId("id");
+ assertEquals("id", metaData.getId());
+ metaData.setPlatform("platform");
+ assertEquals("platform", metaData.getPlatform());
+
+ List<SystemMetadata> list = new ArrayList<SystemMetadata>();
+ metaData.setSystems(list);
+ assertEquals(list, metaData.getSystems());
+ }
+
+ @Test
+ public void testPlatformSubscription() {
+ PlatformSubscription sub = new PlatformSubscription();
+
+ sub.setPlatform("platform");
+ assertEquals("platform", sub.getPlatform());
+ sub.setSelectAll(true);
+ assertEquals(true, sub.isSelectAll());
+
+ List<SystemSubscription> list = new ArrayList<SystemSubscription>();
+ sub.setSystems(list);
+ assertEquals(list, sub.getSystems());
+
+ PlatformSubscription sub1 = new PlatformSubscription("pltfm");
+ }
+
+ @Test
+ public void testSampleOut() {
+ SampleOut so = new SampleOut();
+
+ so.setDate(12345L);
+ assertEquals(12345L, so.getDate());
+ so.setPath("path");
+ assertEquals("path", so.getPath());
+ }
+
+ @Test
+ public void testSystemLevelMetrics() {
+ SystemLevelMetrics slm = new SystemLevelMetrics();
+
+ slm.setName("name");
+ assertEquals("name", slm.getName());
+ slm.setDq(12.3f);
+ assertTrue(12.3f == slm.getDq());
+
+ List<AssetLevelMetrics> list = new ArrayList<AssetLevelMetrics>();
+ slm.setMetrics(list);
+ assertEquals(list, slm.getMetrics());
+
+ AssetLevelMetrics alm = new AssetLevelMetrics();
+ slm.addAssetLevelMetrics(alm);
+ assertEquals(1, slm.getMetrics().size());
+
+ SystemLevelMetrics slm1 = new SystemLevelMetrics("name");
+ }
+
+ @Test
+ public void testSystemLevelMetricsList() {
+ SystemLevelMetricsList ml = new SystemLevelMetricsList();
+
+ List<SystemLevelMetrics> list = new ArrayList<SystemLevelMetrics>();
+ SystemLevelMetrics metric = new SystemLevelMetrics("system");
+ ml.setLatestDQList(list);
+ assertEquals(list, ml.getLatestDQList());
+
+ List<AssetLevelMetrics> ametrics = new ArrayList<AssetLevelMetrics>();
+ AssetLevelMetrics naMetric = new AssetLevelMetrics("almName", "metricType", 12.3f, 12345L, 3);
+ List<AssetLevelMetricsDetail> details = new ArrayList<AssetLevelMetricsDetail>();
+ details.add(new AssetLevelMetricsDetail(123L, 55.3f, 30.3f));
+ naMetric.setDetails(details);
+ ametrics.add(naMetric);
+ metric.setMetrics(ametrics);
+ list.add(metric);
+ assertTrue(ml.containsAsset("system", "almName"));
+ assertFalse(ml.containsAsset("system", "almName1"));
+
+ assertEquals(metric, ml.getSystemLevelMetrics("system"));
+ assertNull(ml.getSystemLevelMetrics("system1"));
+
+ List<SystemLevelMetrics> slmList = ml.getListWithLatestNAssets(5, "system", null, null);
+ assertNotNull(slmList);
+ assertTrue(slmList.size() > 0);
+
+ UserSubscription us = new UserSubscription("user");
+ List<PlatformSubscription> psList = new ArrayList<PlatformSubscription>();
+ PlatformSubscription ps = new PlatformSubscription("Apollo");
+ List<SystemSubscription> ssList = new ArrayList<SystemSubscription>();
+ SystemSubscription ss = new SystemSubscription("system");
+ ss.setSelectAll(true);
+ List<String> assets = new ArrayList<String>();
+ assets.add("asset1");
+ assets.add("asset2");
+ ss.setDataassets(assets);
+ ssList.add(ss);
+ ps.setSystems(ssList);
+ psList.add(ps);
+ us.setSubscribes(psList);
+
+ Map<String, String> tmp = new HashMap<String, String>();
+ tmp.put("almName", "asset1");
+ List<SystemLevelMetrics> slmList1 = ml.getListWithLatestNAssets(5, "system", us, tmp);
+ assertNotNull(slmList1);
+ assertTrue(slmList.size() > 0);
+
+ AssetLevelMetrics alm = ml.getListWithSpecificAssetName("almName");
+ assertNotNull(alm);
+ assertEquals(naMetric, alm);
+ AssetLevelMetrics alm1 = ml.getListWithSpecificAssetName("almName1");
+ assertNull(alm1);
+
+ alm = ml.getListWithSpecificAssetName("almName", 3);
+ assertNotNull(alm);
+ alm1 = ml.getListWithSpecificAssetName("almName1", 3);
+ assertNull(alm1);
+
+ Map<String, String> thresholds = new HashMap<String, String>();
+ thresholds.put("almName", "20.5");
+ thresholds.put("almName1", "50.2");
+ ml.updateDQFail(thresholds);
+ assertEquals(1, naMetric.getDqfail());
+
+ List<SystemLevelMetrics> heatMapList = ml.getHeatMap(thresholds);
+ assertNotNull(heatMapList);
+ assertTrue(heatMapList.size() > 0);
+
+ AssetLevelMetricsDetail detail = new AssetLevelMetricsDetail(12346L, 20.1f, 20.0f);
+ detail.setBolling(new BollingerBandsEntity(300L, 100L, 200L));
+ ml.upsertNewAssetExecute("almName", MetricType.Bollinger.toString(), 12346L, 18.6f, "system", 0, true, detail);
+ ml.upsertNewAssetExecute("almName", MetricType.Trend.toString(), 12346L, 18.6f, "system", 0, true, detail);
+ ml.upsertNewAssetExecute("almName", MetricType.MAD.toString(), 12346L, 18.6f, "system", 0, true, detail);
+ ml.upsertNewAssetExecute("almName", "", 12346L, 18.6f, "system", 0, true, detail);
+ assertEquals(1, ml.getLatestDQList().size());
+ ml.upsertNewAssetExecute("almName2", MetricType.Bollinger.toString(), 12346L, 18.6f, "system", 0, true, detail);
+ ml.upsertNewAssetExecute("almName3", MetricType.Trend.toString(), 12346L, 18.6f, "system", 0, true, detail);
+ ml.upsertNewAssetExecute("almName4", MetricType.MAD.toString(), 12346L, 18.6f, "system", 0, true, detail);
+ ml.upsertNewAssetExecute("almName5", "", 12346L, 18.6f, "system", 0, true, detail);
+ assertEquals(1, ml.getLatestDQList().size());
+ ml.upsertNewAssetExecute("almName2", "metricType2", 12346L, 18.6f, "system2", 0, true, detail);
+ assertEquals(2, ml.getLatestDQList().size());
+ }
+
+ @Test
+ public void testSystemMetadata() {
+ SystemMetadata sm = new SystemMetadata();
+
+ sm.setId("id");
+ assertEquals("id", sm.getId());
+ sm.setName("name");
+ assertEquals("name", sm.getName());
+
+ List<DataAssetIndex> daiList = new ArrayList<DataAssetIndex>();
+ sm.setAssets(daiList);
+ assertEquals(daiList, sm.getAssets());
+ }
+
+ @Test
+ public void testSystemSubscription() {
+ SystemSubscription ss = new SystemSubscription();
+
+ ss.setSystem("system");
+ assertEquals("system", ss.getSystem());
+ ss.setSelectAll(true);
+ assertEquals(true, ss.isSelectAll());
+
+ List<String> assets = new ArrayList<String>();
+ ss.setDataassets(assets);
+ assertEquals(assets, ss.getDataassets());
+ }
+
+ @Test
+ public void testValidateHiveJobConfig() {
+ ValidateHiveJobConfig config = new ValidateHiveJobConfig();
+
+ config.setDataSet("dataset");
+ assertEquals("dataset", config.getDataSet());
+
+ List<ValidateHiveJobConfigLv1Detail> details = new ArrayList<ValidateHiveJobConfigLv1Detail>();
+ List<ValidateHiveJobConfigLv2Detail> lv2Details = new ArrayList<ValidateHiveJobConfigLv2Detail>();
+ lv2Details.add(new ValidateHiveJobConfigLv2Detail(2, 13.3));
+ details.add(new ValidateHiveJobConfigLv1Detail(1, "col1", lv2Details));
+ config.setValidityReq(details);
+ assertEquals(details, config.getValidityReq());
+
+ List<PartitionConfig> partitions = new ArrayList<PartitionConfig>();
+ config.setTimePartitions(partitions);
+ assertEquals(partitions, config.getTimePartitions());
+
+ config.addColumnCalculation(1, "col1", 1);
+ assertEquals(1, config.getValidityReq().size());
+
+ // QUESTION: the result in lv2Detail is double, but getValue here returns long
+ long result = config.getValue("col1", 2);
+ assertEquals(13, result);
+ result = config.getValue("col1", 3);
+ assertEquals(Long.MIN_VALUE, result);
+
+ ValidateHiveJobConfig config1 = new ValidateHiveJobConfig("dataset1");
+ config1.addColumnCalculation(1, "col1", 1);
+ assertEquals(1, config1.getValidityReq().size());
+ }
+
+ @Test
+ public void testValidateHiveJobConfigLv1Detail() {
+ ValidateHiveJobConfigLv1Detail detail = new ValidateHiveJobConfigLv1Detail();
+
+ detail.setColId(1);
+ assertEquals(1, detail.getColId());
+ detail.setColName("name");
+ assertEquals("name", detail.getColName());
+
+ List<ValidateHiveJobConfigLv2Detail> lv2Details = new ArrayList<ValidateHiveJobConfigLv2Detail>();
+ detail.setMetrics(lv2Details);
+ assertEquals(lv2Details, detail.getMetrics());
+ }
+
+ @Test
+ public void testValidateHiveJobConfigLv2Detail() {
+ ValidateHiveJobConfigLv2Detail detail = new ValidateHiveJobConfigLv2Detail();
+
+ detail.setName(1);
+ assertEquals(1, detail.getName());
+ detail.setResult(23.2);
+ assertTrue(23.2 == detail.getResult());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/resources/application.properties
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/resources/application.properties b/griffin-core/src/test/resources/application.properties
new file mode 100644
index 0000000..1b6f147
--- /dev/null
+++ b/griffin-core/src/test/resources/application.properties
@@ -0,0 +1,31 @@
+#
+# Copyright (c) 2016 eBay Software Foundation.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#ldap query
+ldap.server=10.101.150.25
+ldap.port=389
+ldap.user=test
+ldap.pass=test
+
+####QA version
+env=qa
+job.local.folder=C:\\griffin
+#job.local.folder=/export/home/bark
+job.hdfs.folder=/user/griffin
+job.hdfs.runningfoldername=running
+job.hdfs.historyfoldername=history
+job.hdfs.failurefoldername=failure
+spring.data.mongodb.host=dart-8984.phx02.dev.ebayc3.com
+spring.data.mongodb.port=27017
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-core/src/test/resources/context.xml
----------------------------------------------------------------------
diff --git a/griffin-core/src/test/resources/context.xml b/griffin-core/src/test/resources/context.xml
new file mode 100644
index 0000000..072dcd2
--- /dev/null
+++ b/griffin-core/src/test/resources/context.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache
+ License, Version 2.0 (the "License"); you may not use this file except in
+ compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mvc="http://www.springframework.org/schema/mvc"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:task="http://www.springframework.org/schema/task"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd
+ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+ http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.1.xsd">
+
+
+ <!-- DispatcherServlet Context: defines this servlet's request-processing
+ infrastructure -->
+
+ <context:annotation-config />
+
+
+ <context:component-scan base-package="com.ebay.oss.griffin" />
+
+
+</beans>
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/a8ba6ba9/griffin-doc/FSD.md
----------------------------------------------------------------------
diff --git a/griffin-doc/FSD.md b/griffin-doc/FSD.md
new file mode 100644
index 0000000..b89f5b3
--- /dev/null
+++ b/griffin-doc/FSD.md
@@ -0,0 +1,127 @@
+#Functional Specification Document
+
+## Goals
+
+Griffin is a open source Data Quality solution for distributed data systems at any scale in both streaming or batch data context. When people use open source products (e.g. Hadoop, Spark, Kafka, Storm), they always need a data quality service to build his/her confidence on data quality processed by those platforms. Griffin creates a unified process to define and construct data quality measurement pipeline across multiple data systems to provide:
+
+- Automatic quality validation of the data
+- Data profiling and anomaly detection
+- Data quality lineage from upstream to downstream data systems.
+- Data quality health monitoring visualization
+- Shared infrastructure resource management
+
+## Background and strategic fit
+
+At eBay, when people play with big data in Hadoop (or other streaming data), data quality often becomes one big challenge. Different teams have built customized data quality tools to detect and analyze data quality issues within their own domain. We are thinking to take a platform approach to provide shared Infrastructure and generic features to solve common data quality pain points. This would enable us to build trusted data assets.
+
+Currently it\u2019s very difficult and costly to do data quality validation when we have big data flow across multi-platforms at eBay (e.g. Oracle, Hadoop, Couchbase, Cassandra, Kafka, Mongodb). Take eBay real time personalization platform as a sample, everyday we have to validate data quality status for ~600M records ( imaging we have 150M active users for our website). Data quality often becomes one big challenge both in its streaming and batch pipelines.
+
+So we conclude 3 data quality problems at eBay:
+
+1. Lack of end2end unified view of data quality measurement from multiple data sources to target applications, it usually takes a long time to identify and fix poor data quality.
+2. How to get data quality measured in streaming mode, we need to have a process and tool to visualize data quality insights through registering dataset which you want to check data quality, creating data quality measurement model, executing the data quality validation job and getting metrics insights for action taking.
+3. No Shared platform and API Service, have to apply and manage own hardware and software infrastructure.
+
+## Assumptions
+
+- Users will primarily access this application from a PC
+- We are handling textual data, no binary or encoded data
+
+## Main business process
+
+
+
+## Feature List
+
+<table class="confluenceTable" border="2"><tbody><tr><th class="confluenceTh">#</th><th class="confluenceTh">Title</th><th class="confluenceTh">User Story</th><th class="confluenceTh">Importance</th><th colspan="1" class="confluenceTh">Status</th><th class="confluenceTh">Notes</th></tr><tr><td class="confluenceTd">1</td><td class="confluenceTd">User login</td><td class="confluenceTd"><p>User can login the system so that I'm able to access<br />the subsequent features.</p></td><td class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td class="confluenceTd"><ul><li>May use NT account, or a user list maintained in DB</li><li>Remember user for 1 month</li></ul></td></tr><tr><td class="confluenceTd">2</td><td class="confluenceTd">Create model - Accuracy</td><td class="confluenceTd"><p>User can create an accuracy model, so that the metrics</p><p>can be calculated by the rule defined in the model </p></td><td class="confluenceTd">Must Have</td><td colspan="1" clas
s="confluenceTd"> </td><td class="confluenceTd"><ul><li>4 Steps in wizard:</li></ul><ol><li>Select the source dataset and fields which will be used for comparision</li><li>Select the target dataset and fields which will be used for comparision</li><li>Mapping the target fields with source, some basic functions should be <br />supported when mapping: exactly match, length(), lower(), upper(), <br />trim(), etc.</li><li>Set basic configuration for your model: name, definition,threshold, <br />schedule type, email, etc.</li></ol><ul><li>When submitting the form, need to confirm the form info</li><li>After submitting, model list should be displayed and the new model is <br />displayed as the 1st record.</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">3</td><td colspan="1" class="confluenceTd"><span>Create model - Validity</span></td><td colspan="1" class="confluenceTd"><p><span>User can create a validity model, so that the metrics </span></p><p><span>can be </span><span>cal
culated by the rule defined in the model </span></p></td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>3 steps in wizard:</li></ul><ol><li>Select the target dataset and field which want to be checked, only one<br />field should be selected.</li><li>Define your syntax check logic which will be applied on the selected field,<br />including: <strong>Null Count, Unique Count, Duplicate Count, Maximum, </strong><br /><strong>Minimum, </strong><strong>Mean, Median, regular expression match count</strong></li><li><span>Set basic configuration for your model: name, definition,threshold, </span><br /><span>schedule type, email, etc.</span><span><br /></span></li></ol><ul><li>When submitting the form, need to confirm the form info</li><li>After submitting, model list should be displayed and the new model is <br />displayed as the 1st record.</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">
4</td><td colspan="1" class="confluenceTd"><p><span>Create model - Anomaly</span></p><p><span>Detection</span></p></td><td colspan="1" class="confluenceTd"><p><span>User can create an anomaly model, so that the metrics </span></p><p><span>can be </span><span>calculated by the rule defined in the model </span></p></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>3 steps in wizard:</li></ul><ol><li><span>Selected the predefined metrics of a data asset, such as "Total Count",</span><br /><span>"Null Count", or other validity metrics(calculated by validity model)</span></li><li><span>Select a statistical techniques: <strong>History Trend Detection, MAD, Bollinger</strong></span></li><li><span>Set basic configuration for your model: name, definition,threshold, </span><br /><span>schedule type, email, etc.</span></li></ol><ul><li>When submitting the form, need to
confirm the form info</li><li>After submitting, model list should be displayed and the new model is <br />displayed as the 1st record.</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">5</td><td colspan="1" class="confluenceTd"><p>Create model - Publish</p><p>Metrics</p></td><td colspan="1" class="confluenceTd"><p><span>User can create a "publish metrics" model, so that the </span></p><p><span>metrics can be </span><span>calculated by the rule defined in the model </span></p></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Someone may already have their own metrics, so they only need to <br />manualy publish their metrics directly.</li><li>The publish URL and format should let the user know</li><li>User need to set basic configuration: name, definition,threshold, <br />schedule type, email, etc.</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">6<
/td><td colspan="1" class="confluenceTd">List models</td><td colspan="1" class="confluenceTd"><p>User can get a list of all models which contain basic model</p><p>definition info</p></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Information should be shown: Name, Org, Type, Desc, Create Date, Status</li><li>Sorted by "Create Date" desc</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">7</td><td colspan="1" class="confluenceTd">View a model</td><td colspan="1" class="confluenceTd">User can view the details of a model definition</td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>All the information should be displayed: name, definition, type, org, asset,<br />threshold, schedule type, email, owner, status, test result, daily status, etc.</li><li>Daily st
atus can be a chart</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">8</td><td colspan="1" class="confluenceTd">Delete a model</td><td colspan="1" class="confluenceTd">User can delete my own models</td><td colspan="1" class="confluenceTd">May Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Administrator should also have delete permission</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">9</td><td colspan="1" class="confluenceTd">DataAsset list</td><td colspan="1" class="confluenceTd">User can view all data assets</td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Information shown: name, type, HDFS path, org, create time, owner</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">10</td><td colspan="1" class="confluenceTd">Register DataAsset</td><td colspan="1" class="confluenceTd"><p>User can register his own data a
sset, so that he can</p><p>define models under this asset</p></td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Input these information: name, type(hdfs, hive or sql), path, folder pattern, <br /><strong>partition path</strong>, org, schema, owner</li><li>schema is a list, including name, type, desc, sample. We can also import <br />an existing schema from avro</li><li>owner is the logon user</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">11</td><td colspan="1" class="confluenceTd">Delete a DataAsset</td><td colspan="1" class="confluenceTd"><span>User can delete his own data assets</span></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"> </td></tr><tr><td colspan="1" class="confluenceTd">12</td><td colspan="1" class="confluenceTd">Edit a DataAsset</td><td colspan="1" class="conflue
nceTd"><span>User can edit his own data assets</span></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Name, type. org cannot be edited</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">13</td><td colspan="1" class="confluenceTd">Heatmap</td><td colspan="1" class="confluenceTd"><p>User can see the heatmap when I login, so that I'll know</p><p>the status of the metrics</p></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Below target, displayed as red; otherwise, green</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">14</td><td colspan="1" class="confluenceTd">Statistics of the system</td><td colspan="1" class="confluenceTd">User can see the statistics of the entire system</td><td colspan="1" class="confluenceTd">May Have</td><td colspan="1" class="conf
luenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>How many data assets totally</li><li>How many metrics totally</li><li>The healthy info of all metrics(the percentage of red/green)</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">15</td><td colspan="1" class="confluenceTd">List all metrics values</td><td colspan="1" class="confluenceTd">The application can provide the list of all metrics</td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>List format: time, name, value.</li><li>Can be organized by Org->Asset->metrics list</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">16</td><td colspan="1" class="confluenceTd">Show metrics charts</td><td colspan="1" class="confluenceTd"><span>The application can display the charts of all metrics</span></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td col
span="1" class="confluenceTd"><ul><li>A list of charts orgnized by <span>Org->Asset->metrics</span></li><li><span>Can filter by Org->Asset</span></li></ul></td></tr><tr><td colspan="1" class="confluenceTd">17</td><td colspan="1" class="confluenceTd">Show My Dashboard</td><td colspan="1" class="confluenceTd"><p>User can see "my dashboard", so that he'll know the status</p><p>of his subscribed metrics</p></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>A list of charts</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">18</td><td colspan="1" class="confluenceTd">Subscribe metrics</td><td colspan="1" class="confluenceTd"><p>User can subscribe a metrics so that it can be displayed</p><p>on "my dashboard"</p></td><td colspan="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="con
fluenceTd"><ul><li>Subscribe steps</li></ul><ol><li>Select Org->Asset<ol><li>Can select on Org level, then all the assets under this Org will be <br />selected</li><li>Can select on Asset level</li><li>Can multiple select</li></ol></li><li>After the assets are selected, all the models under these assets should<br />be displayed, so that the end user will know which metrics he's selected</li><li>Do subscription</li></ol><p> </p></td></tr><tr><td colspan="1" class="confluenceTd">19</td><td colspan="1" class="confluenceTd">Edit subscription info</td><td colspan="1" class="confluenceTd">User can edit my subscription info</td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>Can unsubscribe, and subscribe new metrics</li></ul></td></tr><tr><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">20</span></td><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255
);">Define email notification</span></td><td colspan="1" class="confluenceTd"><p><span style="color: rgb(51,102,255);">User can request to send notification email when issues</span></p><p><span style="color: rgb(51,102,255);">detected</span></p></td><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">May Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li><span style="color: rgb(51,102,255);">On My Dashboard, I can specify a threshold for each metrics</span></li><li><span style="color: rgb(51,102,255);">Can define a unified email address or different email addresses for</span><br /><span style="color: rgb(51,102,255);">different metrics</span></li></ul></td></tr><tr><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">21</span></td><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">Specify email report</span></td><td colspan="1" class="confluenceTd"><span style="c
olor: rgb(51,102,255);">User can decide the contents in email report</span></td><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">May Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li><span style="color: rgb(51,102,255);">User can select the metrics contained in the report, can also modify</span><br /><span style="color: rgb(51,102,255);">the selection</span></li><li><span style="color: rgb(51,102,255);">Specify email addresses</span></li></ul></td></tr><tr><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">22</span></td><td colspan="1" class="confluenceTd"><span style="color: rgb(51,102,255);">Report engine</span></td><td colspan="1" class="confluenceTd"><p><span style="color: rgb(51,102,255);">The application sends out the report regularly according </span></p><p><span style="color: rgb(51,102,255);">to user's subscription</span></p></td><td colspan="1" class="confluenceTd"><span
style="color: rgb(51,102,255);">May Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li><span style="color: rgb(51,102,255);">Every day or every week, the report will be sent out at the same time</span></li><li><span style="color: rgb(51,102,255);">Main contents on the report are charts</span></li></ul></td></tr><tr><td colspan="1" class="confluenceTd">23</td><td colspan="1" class="confluenceTd">Scheduler job</td><td colspan="1" class="confluenceTd"><p>The application should have the scheduler jobs to</p><p>calculate the metrics</p></td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>According to the rules defined in the models, the scheduler job has these<br />functions:</li></ul><ol><li>Knows when to calculate a metrics</li><li>Call the model engine to get metrics values</li><li>Save the metrics result</li><li>Send notification emails if n
ecessary</li></ol></td></tr><tr><td colspan="1" class="confluenceTd">24</td><td colspan="1" class="confluenceTd">Model engine</td><td colspan="1" class="confluenceTd">The application can calculate the metrics values</td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>With the rule definition, the engine can calculate the metrics values</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">25</td><td colspan="1" class="confluenceTd">Sampling</td><td colspan="1" class="confluenceTd"><p>User can download the sample data to analyze the</p><p>root cause when issue detected</p></td><td colspan="1" class="confluenceTd">Must Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"> </td></tr><tr><td colspan="1" class="confluenceTd">26</td><td colspan="1" class="confluenceTd">Profiling</td><td colspan="1" class="confluenceTd">User can profile the data</td><td colsp
an="1" class="confluenceTd"><span>Must Have</span></td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>According to the rules defined in the model, do data profiling</li></ul></td></tr><tr><td colspan="1" class="confluenceTd">27</td><td colspan="1" class="confluenceTd">User settings</td><td colspan="1" class="confluenceTd">User can specify my own settings</td><td colspan="1" class="confluenceTd">May Have</td><td colspan="1" class="confluenceTd"> </td><td colspan="1" class="confluenceTd"><ul><li>May set the unified email address for notifications</li><li>Set my default view: view the whole picture of all systems, or just my <br />concerned ones(defined in my dashboard)</li><li>......</li></ul></td></tr></tbody></table>
+
+## User interaction and design
+
+<p style="color: rgb(51,102,255);">#1 User Login</p>
+<img src="img/fsd/image2016-6-30 13-19-2.png" >
+
+<p style="color: rgb(51,102,255);">#2 Create model - Accuracy</p>
+step 1 -> step 2 -> step 3 -> step 4 ->step 5 - confirmation
+
+<img src="img/fsd/image2016-6-30 16-16-21.png" >
+<img src="img/fsd/image2016-6-30 16-17-5.png" >
+<img src="img/fsd/image2016-6-30 16-17-52.png" >
+<img src="img/fsd/image2016-6-30 16-18-20.png" >
+<img src="img/fsd/image2016-6-30 16-18-52.png" >
+
+<p style="color: rgb(51,102,255);">#3 Create model - Validity</p>
+step 1 -> step 2 -> step 3 -> step 4 - confirmation
+
+<img src="img/fsd/image2016-6-30 16-20-34.png" >
+<img src="img/fsd/image2016-6-30 16-20-53.png" >
+<img src="img/fsd/image2016-6-30 16-21-16.png" >
+<img src="img/fsd/image2016-6-30 16-21-49.png" >
+
+<p style="color: rgb(51,102,255);">#4 Create model - Anomaly Detection</p>
+step 1 -> step 2 -> step 3 -> step 4 - confirmation
+
+<img src="img/fsd/image2016-6-30 16-22-53.png" >
+<img src="img/fsd/image2016-6-30 16-23-11.png" >
+<img src="img/fsd/image2016-6-30 16-23-32.png" >
+<img src="img/fsd/image2016-6-30 16-24-7.png" >
+
+<p style="color: rgb(51,102,255);">#5 Create model - Publish Metrics</p>
+step 1 -> step 2 - confirmation
+
+<img src="img/fsd/image2016-6-30 16-25-12.png" >
+<img src="img/fsd/image2016-6-30 16-25-42.png" >
+
+<p style="color: rgb(51,102,255);">#6 List models</p>
+<img src="img/fsd/image2016-6-30 16-31-26.png" >
+
+<p style="color: rgb(51,102,255);">#7 View a model</p>
+<img src="img/fsd/image2016-6-30 16-33-44.png" >
+
+<p style="color: rgb(51,102,255);">#8 Delete model</p>
+<img src="img/fsd/image2016-6-30 16-34-58.png" >
+
+<p style="color: rgb(51,102,255);">#9 DataAsset list</p>
+<img src="img/fsd/image2016-6-30 16-35-18.png" >
+
+<p style="color: rgb(51,102,255);">#10 Register DataAsset</p>
+<i><Partition path isn't in this mock-up, will add it later ></i>
+
+<img src="img/fsd/image2016-6-30 16-35-57.png" >
+<img src="img/fsd/image2016-6-30 16-36-48.png" >
+
+<p style="color: rgb(51,102,255);">#11 Delete a DataAsset</p>
+<img src="img/fsd/image2016-6-30 16-39-17.png" >
+
+<p style="color: rgb(51,102,255);">#12 Edit a DataAsset</p>
+<img src="img/fsd/image2016-6-30 16-37-48.png" >
+
+<p style="color: rgb(51,102,255);">#13 Heatmap</p>
+<img src="img/fsd/image2016-6-30 16-39-37.png" >
+
+<p style="color: rgb(51,102,255);">#14 Statistics of the system</p>
+<img src="img/fsd/image2016-6-30 16-40-14.png" >
+
+<p style="color: rgb(51,102,255);">#15 List all metrics values</p>
+<img src="img/fsd/image2016-6-30 16-41-5.png" >
+
+<p style="color: rgb(51,102,255);">#16 Show metrics charts</p>
+chart list -> Max chart display
+
+<img src="img/fsd/image2016-6-30 16-41-57.png" >
+<img src="img/fsd/image2016-6-30 16-42-16.png" >
+
+<p style="color: rgb(51,102,255);">#17 Show My Dashboard</p>
+<img src="img/fsd/image2016-6-30 16-44-15.png" >
+
+<p style="color: rgb(51,102,255);">#18 Subscribe metrics</p>
+TBD
+
+<p style="color: rgb(51,102,255);">#19 Edit subscription info</p>
+Same UI as #18
+
+## Questions
+Below is a list of questions to be addressed as a result of this requirements document:
+<table border="2"><tbody><tr><th>Question</th><th>Outcome</th></tr></tbody></table>
+
+## Not Doing
+-