You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2017/05/11 04:38:36 UTC
incubator-griffin git commit: ut and org controller
Repository: incubator-griffin
Updated Branches:
refs/heads/master 81e9b6aee -> b5fce5247
ut and org controller
Author: Chen <xi...@lm-shc-16501061.corp.ebay.com>
Closes #23 from justACT/ut.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/b5fce524
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/b5fce524
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/b5fce524
Branch: refs/heads/master
Commit: b5fce52475bf275d6ff3fd97a1e0eee7b4a4d3fc
Parents: 81e9b6a
Author: Chen <xi...@lm-shc-16501061.corp.ebay.com>
Authored: Thu May 11 12:38:17 2017 +0800
Committer: William Guo <gu...@icloud.com>
Committed: Thu May 11 12:38:17 2017 +0800
----------------------------------------------------------------------
service/pom.xml | 28 ++++
.../griffin/core/GriffinWebApplication.java | 40 ++++--
.../griffin/core/measure/repo/MeasureRepo.java | 14 ++
.../griffin/core/metric/MetricController.java | 28 ++++
.../griffin/core/schedule/SparkSubmitJob.java | 18 ++-
.../griffin/core/service/GriffinController.java | 20 +++
.../griffin/core/measure/AssertAnnotations.java | 63 ++++++++
.../griffin/core/measure/DataConnectorTest.java | 43 ++++++
.../griffin/core/measure/ReflectTool.java | 33 +++++
.../metastore/HiveMetastoreServiceTest.java | 52 +++++++
.../core/metastore/KafkaSchemaServiceTest.java | 71 +++++++++
.../core/schedule/SparkSubmitJobTest.java | 144 +++++++++++++++++++
.../src/test/resources/application.properties | 22 +++
service/src/test/resources/context.xml | 30 ++++
14 files changed, 582 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/pom.xml
----------------------------------------------------------------------
diff --git a/service/pom.xml b/service/pom.xml
index 88d7a00..d49077f 100644
--- a/service/pom.xml
+++ b/service/pom.xml
@@ -20,6 +20,8 @@
<confluent.version>3.2.0</confluent.version>
<quartz.version>2.2.1</quartz.version>
<start-class>org.apache.griffin.core.GriffinWebApplication</start-class>
+ <powermock.version>1.6.6</powermock.version>
+ <mockito.version>1.10.19</mockito.version>
</properties>
<repositories>
@@ -118,11 +120,37 @@
<version>2.2.2</version>
</dependency>
+ <!-- test -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-module-junit4</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.powermock</groupId>
+ <artifactId>powermock-api-mockito</artifactId>
+ <version>${powermock.version}</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<profiles>
</profiles>
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java b/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
index 7efbcf9..0c2f557 100644
--- a/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
+++ b/service/src/main/java/org/apache/griffin/core/GriffinWebApplication.java
@@ -40,24 +40,36 @@ public class GriffinWebApplication implements CommandLineRunner{
public void run(String... strings) throws Exception {
- HashMap<String,String> configMap1=new HashMap<>();
- configMap1.put("database","default");
- configMap1.put("table.name","test_data_src");
- HashMap<String,String> configMap2=new HashMap<>();
- configMap2.put("database","default");
- configMap2.put("table.name","test_data_tgt");
- String configJson1 = new ObjectMapper().writeValueAsString(configMap1);
- String configJson2 = new ObjectMapper().writeValueAsString(configMap2);
+ HashMap<String,String> configMap1=new HashMap<>();
+ configMap1.put("database","default");
+ configMap1.put("table.name","test_data_src");
+ HashMap<String,String> configMap2=new HashMap<>();
+ configMap2.put("database","default");
+ configMap2.put("table.name","test_data_tgt");
+ String configJson1 = new ObjectMapper().writeValueAsString(configMap1);
+ String configJson2 = new ObjectMapper().writeValueAsString(configMap2);
- DataConnector source = new DataConnector(ConnectorType.HIVE, "1.2", configJson1);
- DataConnector target = new DataConnector(ConnectorType.HIVE, "1.2", configJson2);
+ DataConnector source = new DataConnector(ConnectorType.HIVE, "1.2", configJson1);
+ DataConnector target = new DataConnector(ConnectorType.HIVE, "1.2", configJson2);
- String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1";
+ String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1";
- EvaluateRule eRule = new EvaluateRule(1,rules);
+ EvaluateRule eRule = new EvaluateRule(1,rules);
- Measure measure = new Measure("accu1","accu1 description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1");
- measureRepo.save(measure);
+ Measure measure = new Measure("bevssoj","bevssoj description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1");
+ measureRepo.save(measure);
+
+ DataConnector source2 = new DataConnector(ConnectorType.HIVE, "1.2", configJson1);
+ DataConnector target2 = new DataConnector(ConnectorType.HIVE, "1.2", configJson2);
+ EvaluateRule eRule2 = new EvaluateRule(1,rules);
+ Measure measure2 = new Measure("test","test description", Measure.MearuseType.accuracy, "bullyeye", source2, target2, eRule2,"test1");
+ measureRepo.save(measure2);
+
+ DataConnector source3 = new DataConnector(ConnectorType.HIVE, "1.2", configJson1);
+ DataConnector target3 = new DataConnector(ConnectorType.HIVE, "1.2", configJson2);
+ EvaluateRule eRule3 = new EvaluateRule(1,rules);
+ Measure measure3 = new Measure("just_inthere","test_just_inthere description", Measure.MearuseType.accuracy, "hadoop", source3, target3, eRule3,"test1");
+ measureRepo.save(measure3);
}
@Bean
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
index 0eec264..b0f8d3c 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
@@ -2,8 +2,22 @@ package org.apache.griffin.core.measure.repo;
import org.apache.griffin.core.measure.Measure;
+import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.CrudRepository;
+import java.util.List;
+
public interface MeasureRepo extends CrudRepository<Measure, Long> {
Measure findByName(String name);
+
+ @Query("select DISTINCT m.organization from Measure m")
+ List<String> findOrganizations();
+
+ @Query("select m.name from Measure m " +
+ "where m.organization= ?1")
+ List<String> findNameByOrganization(String organization);
+
+ @Query("select m.organization from Measure m "+
+ "where m.name= ?1")
+ String findOrgByName(String measureName);
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/metric/MetricController.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/metric/MetricController.java b/service/src/main/java/org/apache/griffin/core/metric/MetricController.java
new file mode 100644
index 0000000..5b9f610
--- /dev/null
+++ b/service/src/main/java/org/apache/griffin/core/metric/MetricController.java
@@ -0,0 +1,28 @@
+package org.apache.griffin.core.metric;
+
+import org.apache.griffin.core.measure.repo.MeasureRepo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * Created by xiangrchen on 5/8/17.
+ */
+@RestController
+@RequestMapping("/metrics")
+public class MetricController {
+
+
+// @RequestMapping("/organization/{org}/{measureName}")
+private static final Logger log = LoggerFactory.getLogger(MetricController.class);
+ @Autowired
+ MeasureRepo measureRepo;
+ @RequestMapping("/org/{measureName}")
+ public String getOrgByMeasureName(@PathVariable("measureName") String measureName){
+ return measureRepo.findOrgByName(measureName);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java b/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java
index 9f34df5..a8ba755 100644
--- a/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java
+++ b/service/src/main/java/org/apache/griffin/core/schedule/SparkSubmitJob.java
@@ -85,16 +85,14 @@ public class SparkSubmitJob implements Job {
if (sourcePattern != null && !sourcePattern.isEmpty()) {
sourcePatternItemSet = sourcePattern.split("-");
- sourcePatternPartitionSizeMin = Math.min(partitionItemSet.length, sourcePatternItemSet.length);
long currentTimstamp = setCurrentTimestamp(currentSystemTimestamp);
- setDataConnectorPartitions(measure.getSource(), sourcePatternItemSet, partitionItemSet, currentTimstamp, sourcePatternPartitionSizeMin);
+ setDataConnectorPartitions(measure.getSource(), sourcePatternItemSet, partitionItemSet, currentTimstamp);
jd.getJobDataMap().put("lastTime", currentTimstamp + "");
}
if (targetPattern != null && !targetPattern.equals("")) {
targetPatternItemSet = targetPattern.split("-");
- targetPatternPartitionSizeMin = Math.min(partitionItemSet.length, targetPatternItemSet.length);
long currentTimstamp = setCurrentTimestamp(currentSystemTimestamp);
- setDataConnectorPartitions(measure.getTarget(), targetPatternItemSet, partitionItemSet, currentTimstamp, targetPatternPartitionSizeMin);
+ setDataConnectorPartitions(measure.getTarget(), targetPatternItemSet, partitionItemSet, currentTimstamp);
jd.getJobDataMap().put("lastTime", currentTimstamp + "");
}
//final String uri = "http://10.9.246.187:8998/batches";
@@ -106,13 +104,14 @@ public class SparkSubmitJob implements Job {
// {"file": "/exe/griffin-measure-batch-0.0.1-SNAPSHOT.jar", "className": "org.apache.griffin.measure.batch.Application", "args": ["/benchmark/test/env.json", "/benchmark/test/config-rdm.json"], "name": "griffin-livy", "queue": "default", "numExecutors": 2, "executorCores": 4, "driverMemory": "2g", "executorMemory": "2g", "conf": {"spark.jars.packages": "com.databricks:spark-avro_2.10:2.0.1"}, "jars": ["/livy/datanucleus-api-jdo-3.2.6.jar", "/livy/datanucleus-core-3.2.10.jar", "/livy/datanucleus-rdbms-3.2.9.jar"], "files": ["/livy/hive-site.xml"]}' -H "Content-Type: application/json"
}
- public Map<String, String> genPartitions(String[] PatternItemSet, String[] partitionItemSet, long timestamp, int comparableSizeMin) {
+ public Map<String, String> genPartitions(String[] patternItemSet, String[] partitionItemSet, long timestamp) {
+ int comparableSizeMin=Math.min(patternItemSet.length,partitionItemSet.length);
Map<String, String> res = new HashMap<>();
for (int i = 0; i < comparableSizeMin; i++) {
/**
* in order to get a standard date like 20170427 01
*/
- String pattrn = PatternItemSet[i].replace("mm", "MM");
+ String pattrn = patternItemSet[i].replace("mm", "MM");
pattrn = pattrn.replace("DD", "dd");
pattrn = pattrn.replace("hh", "HH");
SimpleDateFormat sdf = new SimpleDateFormat(pattrn);
@@ -128,11 +127,10 @@ public class SparkSubmitJob implements Job {
* @param patternItemSet
* @param partitionItemSet
* @param timestamp
- * @param patternPartitionSizeMin
* @return
*/
- private void setDataConnectorPartitions(DataConnector dc, String[] patternItemSet, String[] partitionItemSet, long timestamp, int patternPartitionSizeMin) {
- Map<String, String> partitionItemMap = genPartitions(patternItemSet, partitionItemSet, timestamp, patternPartitionSizeMin);
+ public void setDataConnectorPartitions(DataConnector dc, String[] patternItemSet, String[] partitionItemSet, long timestamp) {
+ Map<String, String> partitionItemMap = genPartitions(patternItemSet, partitionItemSet, timestamp);
String partitions = partitionItemMap.toString().substring(1, partitionItemMap.toString().length() - 1);
Map<String, String> configMap = dc.getConfig();
@@ -193,7 +191,7 @@ public class SparkSubmitJob implements Job {
sparkJobDO.setJars(jars);
List<String> files = new ArrayList<>();
-// files.add(props.getProperty("sparkJob.files_1"));
+ files.add(props.getProperty("sparkJob.files_1"));
sparkJobDO.setFiles(files);
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/main/java/org/apache/griffin/core/service/GriffinController.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/service/GriffinController.java b/service/src/main/java/org/apache/griffin/core/service/GriffinController.java
index 7a5b5ad..889b441 100644
--- a/service/src/main/java/org/apache/griffin/core/service/GriffinController.java
+++ b/service/src/main/java/org/apache/griffin/core/service/GriffinController.java
@@ -1,9 +1,16 @@
package org.apache.griffin.core.service;
+import org.apache.griffin.core.measure.repo.MeasureRepo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
+import java.util.List;
+
@RestController
public class GriffinController {
@@ -13,5 +20,18 @@ public class GriffinController {
return "0.1.0";
}
+ private static final Logger log = LoggerFactory.getLogger(GriffinController.class);
+ @Autowired
+ MeasureRepo measureRepo;
+
+ @RequestMapping("/org")
+ public List<String> getOrg(){
+ return measureRepo.findOrganizations();
+ }
+
+ @RequestMapping("/org/{org}")
+ public List<String> getMetricNameListByOrg(@PathVariable("org") String org){
+ return measureRepo.findNameByOrganization(org);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java b/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java
new file mode 100644
index 0000000..0aeb783
--- /dev/null
+++ b/service/src/test/java/org/apache/griffin/core/measure/AssertAnnotations.java
@@ -0,0 +1,63 @@
+package org.apache.griffin.core.measure;
+
+import java.lang.annotation.Annotation;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * Created by xiangrchen on 5/10/17.
+ */
+public class AssertAnnotations {
+ private static void assertAnnotations(
+ List<Class> annotationClasses, List<Annotation> annotations) {
+ // length
+ if (annotationClasses.size() != annotations.size()) {
+ throw new AssertionError(
+ String.format("Expected %d annotations, but found %d"
+ , annotationClasses.size(), annotations.size()
+ ));
+ }
+ // exists
+ annotationClasses.forEach(
+ ac -> {
+ long cnt
+ = annotations.stream()
+ .filter(a -> a.annotationType().isAssignableFrom(ac))
+ .count();
+ if (cnt == 0) {
+ throw new AssertionError(
+ String.format("No annotation of type %s found", ac.getName())
+ );
+ }
+ }
+ );
+ }
+ public static void assertType(Class c, Class... annotationClasses) {
+ assertAnnotations(
+ Arrays.asList(annotationClasses)
+ , Arrays.asList(c.getAnnotations())
+ );
+ }
+ public static void assertField(
+ Class c, String fieldName, Class... annotationClasses) {
+ try {
+ assertAnnotations(
+ Arrays.asList(annotationClasses)
+ , Arrays.asList(c.getDeclaredField(fieldName).getAnnotations())
+ );
+ } catch (NoSuchFieldException nsfe) {
+ throw new AssertionError(nsfe);
+ }
+ }
+ public static void assertMethod(
+ Class c, String getterName, Class...annotationClasses) {
+ try {
+ assertAnnotations(
+ Arrays.asList(annotationClasses)
+ , Arrays.asList(c.getDeclaredMethod(getterName).getAnnotations())
+ );
+ } catch (NoSuchMethodException nsfe) {
+ throw new AssertionError(nsfe);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java b/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java
new file mode 100644
index 0000000..c5da3c8
--- /dev/null
+++ b/service/src/test/java/org/apache/griffin/core/measure/DataConnectorTest.java
@@ -0,0 +1,43 @@
+package org.apache.griffin.core.measure;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.persistence.Entity;
+
+
+/**
+ * Created by xiangrchen on 5/10/17.
+ */
+public class DataConnectorTest {
+ @Test
+ public void typeAnnotations() {
+ // assert
+ AssertAnnotations.assertType(
+ DataConnector.class, Entity.class);
+ }
+
+ @Test
+ public void fieldAnnotations() {
+ // assert
+ AssertAnnotations.assertField(DataConnector.class, "version");
+ AssertAnnotations.assertField(DataConnector.class, "config");
+ }
+
+ @Test
+ public void entity() {
+ // setup
+ Entity a = ReflectTool.getClassAnnotation(DataConnector.class, Entity.class);
+ // assert
+ Assert.assertEquals("", a.name());
+ }
+
+// @Test
+// public void config() {
+// // setup
+// Column c = ReflectTool.getMethodAnnotation(
+// DataConnector.class, "getConfig", Column.class);
+// // assert
+// Assert.assertEquals("config", c.name());
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java b/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java
new file mode 100644
index 0000000..123b550
--- /dev/null
+++ b/service/src/test/java/org/apache/griffin/core/measure/ReflectTool.java
@@ -0,0 +1,33 @@
+package org.apache.griffin.core.measure;
+
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+
+/**
+ * Created by xiangrchen on 5/10/17.
+ */
+public class ReflectTool {
+ public static <T extends Annotation> T getMethodAnnotation(
+ Class<?> c, String methodName, Class<T> annotation) {
+ try {
+ Method m = c.getDeclaredMethod(methodName);
+ return (T)m.getAnnotation(annotation);
+ } catch (NoSuchMethodException nsme) {
+ throw new RuntimeException(nsme);
+ }
+ }
+ public static <T extends Annotation> T getFieldAnnotation(
+ Class<?> c, String fieldName, Class<T> annotation) {
+ try {
+ Field f = c.getDeclaredField(fieldName);
+ return (T)f.getAnnotation(annotation);
+ } catch (NoSuchFieldException nsme) {
+ throw new RuntimeException(nsme);
+ }
+ }
+ public static <T extends Annotation> T getClassAnnotation(
+ Class<?> c, Class<T> annotation) {
+ return (T) c.getAnnotation(annotation);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java b/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java
new file mode 100644
index 0000000..e5223bc
--- /dev/null
+++ b/service/src/test/java/org/apache/griffin/core/metastore/HiveMetastoreServiceTest.java
@@ -0,0 +1,52 @@
+package org.apache.griffin.core.metastore;
+
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.junit.Before;
+import org.junit.Test;
+import org.powermock.reflect.Whitebox;
+
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.*;
+import static org.springframework.test.util.AssertionErrors.assertEquals;
+
+/**
+ * Created by xiangrchen on 5/10/17.
+ */
+public class HiveMetastoreServiceTest {
+
+ HiveMetastoreService hiveMetastoreService;
+ @Before
+ public void setup() throws NoSuchFieldException, IllegalAccessException {
+ Field defaultDbName = HiveMetastoreService.class.getDeclaredField("defaultDbName");
+ defaultDbName.setAccessible(true);
+ hiveMetastoreService=new HiveMetastoreService();
+ defaultDbName.set(hiveMetastoreService,"default");
+ hiveMetastoreService.client=mock(HiveMetaStoreClient.class);
+ }
+
+ @Test
+ public void test_getUseDbName() throws Exception {
+ String dbName="someDbName";
+ String result = Whitebox.invokeMethod(hiveMetastoreService, "getUseDbName", dbName);
+ assertEquals("success",result,dbName);
+
+ dbName="";
+ result = Whitebox.invokeMethod(hiveMetastoreService, "getUseDbName", dbName);
+ assertEquals("success",result,"default");
+ }
+
+ @Test
+ public void test_getAllDatabases() throws MetaException {
+ List<String> res= new ArrayList<>();
+ when(hiveMetastoreService.client.getAllDatabases()).thenReturn(res);
+ hiveMetastoreService.getAllDatabases();
+ verify(hiveMetastoreService.client).getAllDatabases();
+
+ when(hiveMetastoreService.client.getAllDatabases()).thenThrow(new MetaException());
+ hiveMetastoreService.getAllDatabases();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java b/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java
new file mode 100644
index 0000000..103de3c
--- /dev/null
+++ b/service/src/test/java/org/apache/griffin/core/metastore/KafkaSchemaServiceTest.java
@@ -0,0 +1,71 @@
+package org.apache.griffin.core.metastore;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.powermock.reflect.Whitebox;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Field;
+
+import static org.springframework.test.util.AssertionErrors.assertEquals;
+
+/**
+ * Created by xiangrchen on 5/9/17.
+ */
+//@RunWith(SpringJUnit4ClassRunner.class)
+//@ContextConfiguration
+public class KafkaSchemaServiceTest {
+
+ private static final Logger log = LoggerFactory.getLogger(KafkaSchemaServiceTest.class);
+// @Value("${kafka.schema.registry.url}")
+// private String url;
+
+ private KafkaSchemaService kafkaSchemaService;
+
+ @Before
+ public void setup() throws NoSuchFieldException, IllegalAccessException {
+ Field url = KafkaSchemaService.class.getDeclaredField("url");
+ url.setAccessible(true);
+ kafkaSchemaService=new KafkaSchemaService();
+ url.set(kafkaSchemaService, "http://localhost:8080");
+ }
+
+ @Test
+ public void test_registryUrl() throws Exception {
+ String path="/user/id";
+ String result = Whitebox.invokeMethod(kafkaSchemaService, "registryUrl", path);
+ assertEquals("success",result,"http://localhost:8080"+path);
+
+ path="user/id";
+ result = Whitebox.invokeMethod(kafkaSchemaService, "registryUrl", path);
+ assertEquals("success",result,"http://localhost:8080"+"/"+path);
+
+ path="";
+ result = Whitebox.invokeMethod(kafkaSchemaService, "registryUrl", path);
+ assertEquals("success",result,path);
+ }
+
+// @Test
+// public void test_getSchemaString(){
+// int id=1;
+// RestTemplate restTemplate =mock(RestTemplate.class);
+// ResponseEntity<SchemaString> mockRes =(ResponseEntity<SchemaString>)mock(ResponseEntity.class);
+// String regUrl="http://10.65.159.119:8081"+"/schemas/ids/"+id;
+// when(restTemplate.getForEntity(regUrl, SchemaString.class)).thenReturn(mockRes);
+// SchemaString result=new SchemaString();
+// when(mockRes.getBody()).thenReturn(result);
+// kafkaSchemaService.getSchemaString(id);
+// }
+
+// @Configuration
+// @ComponentScan("org.apache.griffin.core")
+// static class MyServiceConfiguration {
+// @Bean
+// PropertyPlaceholderConfigurer propConfig() {
+// PropertyPlaceholderConfigurer ppc = new PropertyPlaceholderConfigurer();
+// ppc.setLocation(new ClassPathResource("application.properties"));
+// return ppc;
+// }
+// }
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java
----------------------------------------------------------------------
diff --git a/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java b/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java
new file mode 100644
index 0000000..ea9c50d
--- /dev/null
+++ b/service/src/test/java/org/apache/griffin/core/schedule/SparkSubmitJobTest.java
@@ -0,0 +1,144 @@
+package org.apache.griffin.core.schedule;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.griffin.core.measure.DataConnector;
+import org.apache.griffin.core.measure.EvaluateRule;
+import org.apache.griffin.core.measure.Measure;
+import org.apache.griffin.core.measure.repo.MeasureRepo;
+import org.junit.Before;
+import org.junit.Test;
+import org.quartz.JobDataMap;
+import org.quartz.JobDetail;
+import org.quartz.JobExecutionContext;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.client.RestTemplate;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.*;
+
+/**
+ * Created by xiangrchen on 5/8/17.
+ */
+//@RunWith(SpringJUnit4ClassRunner.class)
+//@ContextConfiguration(locations = {"classpath:context.xml"})
+public class SparkSubmitJobTest {
+
+ private SparkSubmitJob ssj;
+
+ @Autowired
+ MeasureRepo measureRepo;
+
+ @Before
+ public void setUp() throws IOException {
+ ssj=new SparkSubmitJob();
+ ssj.measureRepo=mock(MeasureRepo.class);
+ }
+
+ @Test
+ public void test_execute() throws JsonProcessingException {
+ JobExecutionContext context=mock(JobExecutionContext.class);
+ JobDetail jd = mock(JobDetail.class);
+ when(context.getJobDetail()).thenReturn(jd);
+
+ JobDataMap jdmap = mock(JobDataMap.class);
+ when(jd.getJobDataMap()).thenReturn(jdmap);
+
+ when(jdmap.getString("measure")).thenReturn("bevssoj");
+ when(jdmap.getString("sourcePat")).thenReturn("YYYYMMDD-HH");
+ when(jdmap.getString("targetPat")).thenReturn("YYYYMMDD-HH");
+ when(jdmap.getString("dataStartTimestamp")).thenReturn("1460174400000");
+ when(jdmap.getString("lastTime")).thenReturn("");
+ when(jdmap.getString("periodTime")).thenReturn("10");
+
+ HashMap<String,String> configMap1=new HashMap<>();
+ configMap1.put("database","default");
+ configMap1.put("table.name","test_data_src");
+ HashMap<String,String> configMap2=new HashMap<>();
+ configMap2.put("database","default");
+ configMap2.put("table.name","test_data_tgt");
+ String configJson1 = new ObjectMapper().writeValueAsString(configMap1);
+ String configJson2 = new ObjectMapper().writeValueAsString(configMap2);
+ DataConnector source = new DataConnector(DataConnector.ConnectorType.HIVE, "1.2", configJson1);
+ DataConnector target = new DataConnector(DataConnector.ConnectorType.HIVE, "1.2", configJson2);
+ String rules = "$source.uage > 100 AND $source.uid = $target.uid AND $source.uage + 12 = $target.uage + 10 + 2 AND $source.udes + 11 = $target.udes + 1 + 1";
+ EvaluateRule eRule = new EvaluateRule(1,rules);
+ Measure measure = new Measure("bevssoj","bevssoj description", Measure.MearuseType.accuracy, "bullyeye", source, target, eRule,"test1");
+
+ when(ssj.measureRepo.findByName("bevssoj")).thenReturn(measure);
+// ssj.execute(context);
+
+ RestTemplate restTemplate =mock(RestTemplate.class);
+// String uri="http://10.9.246.187:8998/batches";
+ String uri="";
+ SparkJobDO sparkJobDO=mock(SparkJobDO.class);
+ when(restTemplate.postForObject(uri, sparkJobDO, String.class)).thenReturn(null);
+
+
+ long currentSystemTimestamp=System.currentTimeMillis();
+ long currentTimstamp = ssj.setCurrentTimestamp(currentSystemTimestamp);
+
+// verify(ssj.measureRepo).findByName("bevssoj");
+// verify(jdmap,atLeast(2)).put("lastTime",currentTimstamp+"");
+ }
+
+ @Test
+ public void test_genPartitions(){
+ String[] patternItemSet={"YYYYMMDD","HH"};
+ String[] partitionItemSet={"date","hour"};
+ long timestamp=1460174400000l;
+ Map<String,String> par=ssj.genPartitions(patternItemSet,partitionItemSet,timestamp);
+ Map<String,String> verifyMap=new HashMap<>();
+ verifyMap.put("date","20160409");
+ verifyMap.put("hour","12");
+ assertEquals(verifyMap,par);
+ }
+
+ @Test
+ public void test_setDataConnectorPartitions(){
+ DataConnector dc=mock(DataConnector.class);
+ String[] patternItemSet={"YYYYMMDD","HH"};
+ String[] partitionItemSet={"date","hour"};
+ long timestamp=1460174400000l;
+ ssj.setDataConnectorPartitions(dc,patternItemSet,partitionItemSet,timestamp);
+// doNothing().when(ssj).setDataConnectorPartitions(dataConnector,patternItemSet,partitionItemSet,timestamp);
+ Map<String,String> map=new HashMap<>();
+ map.put("partitions","date=20160409, hour=12");
+ try {
+ verify(dc).setConfig(map);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Test
+ public void test_setCurrentTimestamp(){
+ long timestamp=System.currentTimeMillis();
+ ssj.eachJoblastTimestamp="";
+ System.out.println(ssj.setCurrentTimestamp(timestamp));
+ ssj.eachJoblastTimestamp="1494297256667";
+ ssj.periodTime="10";
+ System.out.println(ssj.setCurrentTimestamp(timestamp));
+ }
+
+ @Test
+ public void test_setSparkJobDO(){
+ ssj=mock(SparkSubmitJob.class);
+ doNothing().when(ssj).setSparkJobDO();
+ }
+
+ @Test
+ public void test_getsparkJobProperties(){
+ ssj=mock(SparkSubmitJob.class);
+ try {
+ when(ssj.getsparkJobProperties()).thenReturn(null);
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/resources/application.properties
----------------------------------------------------------------------
diff --git a/service/src/test/resources/application.properties b/service/src/test/resources/application.properties
new file mode 100644
index 0000000..a344a05
--- /dev/null
+++ b/service/src/test/resources/application.properties
@@ -0,0 +1,22 @@
+spring.datasource.url= jdbc:mysql://localhost:3306/metastore?autoReconnect=true&useSSL=false
+spring.datasource.username =griffin
+spring.datasource.password =123456
+
+spring.datasource.driver-class-name=com.mysql.jdbc.Driver
+
+## Hibernate ddl auto (validate,create, create-drop, update)
+
+spring.jpa.hibernate.ddl-auto = update
+spring.jpa.show-sql=true
+spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.MySQL5Dialect
+#
+#
+## Naming strategy
+spring.jpa.hibernate.naming-strategy = org.hibernate.cfg.ImprovedNamingStrategy
+
+# hive metastore
+hive.metastore.uris = thrift://10.9.246.187:9083
+hive.metastore.dbname = default
+
+# kafka schema registry
+kafka.schema.registry.url = http://10.65.159.119:8081
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/b5fce524/service/src/test/resources/context.xml
----------------------------------------------------------------------
diff --git a/service/src/test/resources/context.xml b/service/src/test/resources/context.xml
new file mode 100644
index 0000000..8d81ef8
--- /dev/null
+++ b/service/src/test/resources/context.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Copyright (c) 2016 eBay Software Foundation. Licensed under the Apache
+ License, Version 2.0 (the "License"); you may not use this file except in
+ compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+<beans xmlns="http://www.springframework.org/schema/beans"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:mvc="http://www.springframework.org/schema/mvc"
+ xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:task="http://www.springframework.org/schema/task"
+ xsi:schemaLocation="
+ http://www.springframework.org/schema/mvc http://www.springframework.org/schema/mvc/spring-mvc-3.1.xsd
+ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
+ http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context-3.1.xsd
+ http://www.springframework.org/schema/aop http://www.springframework.org/schema/aop/spring-aop-3.1.xsd
+ http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task-3.1.xsd">
+
+
+ <!-- DispatcherServlet Context: defines this servlet's request-processing
+ infrastructure -->
+
+ <context:annotation-config />
+
+
+ <context:component-scan base-package="org.apache.griffin.core" />
+
+
+</beans>