You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@griffin.apache.org by gu...@apache.org on 2018/01/22 01:51:53 UTC
[3/3] incubator-griffin git commit: fix transaction and schedule bug
and update ut
fix transaction and schedule bug and update ut
Fix Bugs:
1.fix transaction rollback failure bug
2.job schedule range null bug
3.fix time fromat without timezone bug
Author: ahutsunshine <ah...@gmail.com>
Author: He Wang <wa...@qq.com>
Closes #195 from ahutsunshine/master.
Project: http://git-wip-us.apache.org/repos/asf/incubator-griffin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-griffin/commit/71fcf93b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-griffin/tree/71fcf93b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-griffin/diff/71fcf93b
Branch: refs/heads/master
Commit: 71fcf93b99f1401d14e5baeb1f6137c6891d96fc
Parents: 87e59a5
Author: ahutsunshine <ah...@gmail.com>
Authored: Mon Jan 22 09:51:54 2018 +0800
Committer: Lionel Liu <bh...@163.com>
Committed: Mon Jan 22 09:51:54 2018 +0800
----------------------------------------------------------------------
README.md | 31 +-
griffin-doc/service/postman/griffin.json | 68 +-
.../griffin/core/config/PropertiesConfig.java | 4 +-
.../griffin/core/job/FileExistPredicator.java | 4 +-
.../apache/griffin/core/job/JobInstance.java | 12 +-
.../apache/griffin/core/job/JobServiceImpl.java | 65 +-
.../griffin/core/job/entity/GriffinJob.java | 6 +-
.../griffin/core/job/entity/JobDataSegment.java | 10 +-
.../griffin/core/job/entity/JobSchedule.java | 8 +-
.../core/job/entity/SegmentPredicate.java | 5 +
.../griffin/core/job/entity/SegmentRange.java | 8 +
.../measure/ExternalMeasureOperationImpl.java | 6 +-
.../measure/GriffinMeasureOperationImpl.java | 19 +-
.../griffin/core/measure/MeasureOperation.java | 2 +-
.../griffin/core/measure/MeasureOrgService.java | 1 -
.../core/measure/MeasureServiceImpl.java | 11 +-
.../core/measure/entity/ExternalMeasure.java | 3 +-
.../core/measure/entity/GriffinMeasure.java | 20 +-
.../griffin/core/measure/entity/Measure.java | 4 +-
.../griffin/core/measure/entity/Rule.java | 8 +-
.../griffin/core/measure/repo/MeasureRepo.java | 10 -
.../org/apache/griffin/core/util/AvroUtil.java | 30 -
.../org/apache/griffin/core/util/TimeUtil.java | 44 +-
.../src/main/resources/application.properties | 4 +-
service/src/main/resources/sparkJob.properties | 16 +-
.../core/config/PropertiesConfigTest.java | 160 ++++
.../griffin/core/job/JobInstanceTest.java | 186 ++++
.../griffin/core/job/JobServiceImplTest.java | 934 +++++++++++--------
.../griffin/core/job/SparkSubmitJobTest.java | 179 +++-
.../griffin/core/job/repo/JobRepoTest.java | 94 ++
.../core/measure/MeasureControllerTest.java | 24 +-
.../core/measure/MeasureOrgServiceImplTest.java | 212 +++--
.../core/measure/MeasureServiceImplTest.java | 336 +++++--
.../measure/repo/DataConnectorRepoTest.java | 80 ++
.../core/measure/repo/MeasureRepoTest.java | 189 ++--
.../apache/griffin/core/util/EntityHelper.java | 167 +++-
.../griffin/core/util/GriffinUtilTest.java | 87 --
.../apache/griffin/core/util/JsonUtilTest.java | 84 ++
.../griffin/core/util/PropertiesUtilTest.java | 45 +
.../apache/griffin/core/util/TimeUtilTest.java | 94 ++
.../src/test/resources/application.properties | 79 +-
service/src/test/resources/quartz.properties | 2 +-
service/src/test/resources/sparkJob.properties | 50 +
43 files changed, 2332 insertions(+), 1069 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 452b3f2..13a52eb 100644
--- a/README.md
+++ b/README.md
@@ -18,7 +18,8 @@ under the License.
-->
-## Apache Griffin
+## Apache Griffin
+[](https://travis-ci.org/apache/incubator-griffin) [](https://www.apache.org/licenses/LICENSE-2.0.html)
Apache Griffin is a model driven data quality solution for modern data systems.
It provides a standard process to define data quality measures, execute, report, as well as an unified dashboard across multiple data systems.
@@ -27,16 +28,7 @@ You can access our wiki page [here](https://cwiki.apache.org/confluence/display/
You can access our issues jira page [here](https://issues.apache.org/jira/secure/Dashboard.jspa?selectPageId=12330914).
### Contact us
-[Dev List](mailto://dev@griffin.incubator.apache.org)
-
-
-### CI
-
-
-### Repository
-Snapshot:
-
-Release:
+Email: <a href="mailto:dev@griffin.incubator.apache.org">dev@griffin.incubator.apache.org</a>
### How to run in docker
1. Install [docker](https://docs.docker.com/engine/installation/) and [docker compose](https://docs.docker.com/compose/install/).
@@ -59,10 +51,10 @@ Release:
```
docker-compose -f docker-compose-batch.yml up -d
```
-6. Now you can try griffin APIs by using postman after importing the [json files](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/postman).
+6. Now you can try griffin APIs by using postman after importing the [json files](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/service/postman).
In which you need to modify the environment `BASE_PATH` value into `<your local IP address>:38080`.
-More details about griffin docker [here](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/griffin-docker-guide.md).
+More details about griffin docker [here](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/docker/griffin-docker-guide.md).
### How to deploy and run at local
1. Install jdk (1.8 or later versions).
@@ -124,12 +116,23 @@ More details about griffin docker [here](https://github.com/apache/incubator-gri
```
http://<your IP>:8080
```
-11. Follow the steps using UI [here](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/dockerUIguide.md#webui-test-case-guide).
+11. Follow the steps using UI [here](https://github.com/apache/incubator-griffin/blob/master/griffin-doc/ui/dockerUIguide.md#webui-test-case-guide).
**Note**: The front-end UI is still under development, you can only access some basic features currently.
+### Document List
+- [Wiki](https://cwiki.apache.org/confluence/display/GRIFFIN/Apache+Griffin)
+- [Measure](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/measure)
+- [Service](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/service)
+- [UI](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/ui)
+- [Docker usage](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/docker)
+- [Postman API](https://github.com/apache/incubator-griffin/tree/master/griffin-doc/service/postman)
### Contributing
See [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to contribute code, documentation, etc.
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/griffin-doc/service/postman/griffin.json
----------------------------------------------------------------------
diff --git a/griffin-doc/service/postman/griffin.json b/griffin-doc/service/postman/griffin.json
index 88a220a..fdd172a 100644
--- a/griffin-doc/service/postman/griffin.json
+++ b/griffin-doc/service/postman/griffin.json
@@ -376,7 +376,7 @@
"tests": null,
"currentHelper": "normal",
"helperAttributes": {},
- "time": 1515399022575,
+ "time": 1516341608732,
"name": "Update measure",
"description": "`PUT /api/v1/measures`\n\n#### Request Header\nkey | value\n--- | ---\nContent-Type | application/json\n\n#### Request Body\n\nname | description | type\n--- | --- | --- \nmeasure | measure entity | Measure\n\nThere are two different measures that are griffin measure and external measure.\nIf you want to update an external measure,you can use following example json in request body.\n```\n{\n\t\"id\":1,\n \"type\": \"external\",\n \"name\": \"external_name\",\n \"description\": \" update test measure\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"metricName\": \"metricName\"\n}\n```\nPostman gives a griffin measure example in request body and response body. \n#### Response Body Sample\n```\n{\n \"code\": 204,\n \"description\": \"Update Measure Succeed\"\n}\n```\n\nIt may return failed messages.Such as,\n\n```\n {\n \"code\": 400,\n \"description\": \"Resource Not Found\"\n}\n\n```\n\nThe reason for failure may be that measure
id doesn't exist.",
"collectionId": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63",
@@ -462,14 +462,13 @@
}
],
"headers": "Content-Type: application/json\n",
- "data": "{\n \"id\": 1,\n \"name\": \"measure_official_update\",\n \"description\": \"create a measure\",\n \"organization\": \"test\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"type\": \"griffin\",\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"id\": 1,\n \"name\": \"source\",\n \"connectors\": [\n {\n \"id\": 1,\n \"name\": \"connector_name_source\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n
}\n }\n ]\n },\n {\n \"id\": 2,\n \"name\": \"target\",\n \"connectors\": [\n {\n \"id\": 2,\n \"name\": \"connector_name_target\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n }\n ],\n \"evaluate.rule\": {\n \"id\": 1,\n \"rules\": [\n {\n \"id\": 1,\n \"rule\": \"source.desc=target.desc
\",\n \"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\",\n \"details\": {}\n }\n ]\n }\n }",
+ "data": "{\n \"id\": 1,\n \"name\": \"measureName_edit\",\n \"description\": \"measure description\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"dq.type\": \"accuracy\",\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"id\": 1,\n \"name\": \"source\",\n \"connectors\": [\n {\n \"id\": 1,\n \"name\": \"connector_name_source\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 1,\n \"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n
\"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n },\n {\n \"id\": 2,\n \"name\": \"target\",\n \"connectors\": [\n {\n \"id\": 2,\n \"name\": \"connector_name_target\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 2,\n \"type\": \
"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n \"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n }\n ],\n \"evaluate.rule\": {\n \"id\": 1,\n \"rules\": [\n {\n \"id\": 1,\n \"rule\": \"source.desc=target.desc\",\n \"name\": \"rule_name\",\n \"description\": \"Total count\",\n
\"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\",\n \"details\": {}\n }\n ]\n },\n \"measure.type\": \"griffin\"\n }",
"method": "PUT",
"dataMode": "raw"
}
}
],
- "rawModeData": "{\n \"id\": 1,\n \"name\": \"measureName_test_edit\",\n \"description\": \"This is a test measure\",\n \"organization\": \"orgName\",\n \"evaluateRule\": {\n \"rules\": [\n {\n \"rule\": \"source.id = target.id and source.age = target.age and source.desc = target.desc\",\n \"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\"\n }\n ]\n },\n \"owner\": \"test\",\n \"deleted\": false,\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"name\": \"source\",\n \"connectors\": [\n {\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\
"\n }\n }\n ]\n },\n {\n \"name\": \"target\",\n \"connectors\": [\n {\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_tgt\"\n }\n }\n ]\n }\n ]\n}",
- "collection_id": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63"
+ "rawModeData": "{\n \"id\": 1,\n \"name\": \"measureName_edit\",\n \"description\": \"measure description\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"dq.type\": \"accuracy\",\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"id\": 1,\n \"name\": \"source\",\n \"connectors\": [\n {\n \"id\": 1,\n \"name\": \"connector_name_source\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 1,\n \"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n
\"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n },\n {\n \"id\": 2,\n \"name\": \"target\",\n \"connectors\": [\n {\n \"id\": 2,\n \"name\": \"connector_name_target\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 2,\n \"type\
": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n \"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n }\n ],\n \"evaluate.rule\": {\n \"id\": 1,\n \"rules\": [\n {\n \"id\": 1,\n \"rule\": \"source.desc=target.desc\",\n \"name\": \"rule_name\",\n \"description\": \"Total count\",\n
\"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\",\n \"details\": {}\n }\n ]\n },\n \"measure.type\": \"griffin\"\n }"
},
{
"id": "2bfc82ab-ec97-ee89-d6b4-db5ffefce28b",
@@ -704,7 +703,7 @@
"tests": null,
"currentHelper": "normal",
"helperAttributes": {},
- "time": 1515398727266,
+ "time": 1516341583244,
"name": "Add measure",
"description": "`POST /api/v1/measures`\n\n#### Request Header\nkey | value\n--- | ---\nContent-Type | application/json\n\n#### Request Body\n\nname | description | type\n--- | --- | --- \nmeasure | measure entity | Measure\n\nThere are two different measures that are griffin measure and external measure.\nIf you want to create an external measure,you can use following example json in request body.\n```\n{\n \"type\": \"external\",\n \"name\": \"external_name\",\n \"description\": \" test measure\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"metricName\": \"metricName\"\n}\n```\nPostman gives a griffin measure example in request body and response body. \n#### Response Body Sample\n```\n{\n \"code\": 201,\n \"description\": \"Create Measure Succeed\"\n}\n```\n\nIt may return failed messages.Such as,\n\n```\n {\n \"code\": 410,\n \"description\": \"Create Measure Failed, duplicate records\"\n}\n\n```\n\nThe reason for failure may be that measu
re name already exists.You can change measure name to make it unique.\n\n```\n {\n \"code\": 401,\n \"description\": \"Create Measure Failed\"\n}\n```\nThe reason for failure may be that connector names already exist or connector names are empty.",
"collectionId": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63",
@@ -790,14 +789,13 @@
}
],
"headers": "Content-Type: application/json\n",
- "data": "{\r\n \"name\":\"measure_name\",\r\n\t\"type\":\"griffin\",\r\n \"description\":\"create a measure\",\r\n \"organization\":\"test\",\r\n \"evaluate.rule\":{\r\n \"rules\":[\r\n {\r\n \"rule\":\"source.desc=target.desc\",\r\n \"dsl.type\":\"griffin-dsl\",\r\n \"dq.type\":\"accuracy\",\r\n \"details\":{}\r\n }\r\n ]\r\n },\r\n \"owner\":\"test\",\r\n \"process.type\":\"batch\",\r\n \"data.sources\":[\r\n {\r\n \"name\":\"source\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_source\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n \"database\":\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#
YYYYMMdd# AND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n },\r\n {\r\n \"name\":\"target\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_target\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n \"database\":\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#YYYYMMdd# AND hour=
#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n }\r\n ]\r\n}",
+ "data": "{\r\n \"name\":\"measureName\",\r\n\t\"measure.type\":\"griffin\",\r\n \"description\":\"measure description\",\r\n \"organization\":\"orgName\",\r\n\t\"owner\":\"test\",\r\n \"process.type\":\"batch\",\r\n\t\"dq.type\":\"accuracy\",\r\n \"evaluate.rule\":{\r\n \"rules\":[\r\n {\r\n\t\t\t\t\"name\":\"rule_name\",\r\n \"rule\":\"source.desc=target.desc\",\r\n\t\t\t\t\"description\":\"Total count\",\r\n \"dsl.type\":\"griffin-dsl\",\r\n \"dq.type\":\"accuracy\",\r\n \"details\":{}\r\n }\r\n ]\r\n },\r\n \r\n \"data.sources\":[\r\n {\r\n \"name\":\"source\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_source\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n
\"database\":\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#YYYYMMdd# AND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n },\r\n {\r\n \"name\":\"target\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_target\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n \"database\":\"d
efault\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#YYYYMMdd# AND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n }\r\n ]\r\n}",
"method": "POST",
"dataMode": "raw"
}
}
],
- "rawModeData": "{\r\n \"name\":\"measure_official\",\r\n\t\"type\":\"griffin\",\r\n \"description\":\"create a measure\",\r\n \"organization\":\"test\",\r\n \"evaluate.rule\":{\r\n \"rules\":[\r\n {\r\n \"rule\":\"source.desc=target.desc\",\r\n \"dsl.type\":\"griffin-dsl\",\r\n \"dq.type\":\"accuracy\",\r\n \"details\":{}\r\n }\r\n ]\r\n },\r\n \"owner\":\"test\",\r\n \"process.type\":\"batch\",\r\n \"data.sources\":[\r\n {\r\n \"name\":\"source\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_source\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n \"database\":\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\
":\"dt=#YYYYMMdd# AND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n },\r\n {\r\n \"name\":\"target\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_target\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n \"database\":\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#YYYYMMdd# A
ND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n }\r\n ]\r\n}",
- "collection_id": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63"
+ "rawModeData": "{\r\n \"name\":\"measureName\",\r\n\t\"measure.type\":\"griffin\",\r\n \"description\":\"measure description\",\r\n \"organization\":\"orgName\",\r\n\t\"owner\":\"test\",\r\n \"process.type\":\"batch\",\r\n\t\"dq.type\":\"accuracy\",\r\n \"evaluate.rule\":{\r\n \"rules\":[\r\n {\r\n\t\t\t\t\"name\":\"rule_name\",\r\n \"rule\":\"source.desc=target.desc\",\r\n\t\t\t\t\"description\":\"Total count\",\r\n \"dsl.type\":\"griffin-dsl\",\r\n \"dq.type\":\"accuracy\",\r\n \"details\":{}\r\n }\r\n ]\r\n },\r\n \r\n \"data.sources\":[\r\n {\r\n \"name\":\"source\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_source\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n
\"database\":\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#YYYYMMdd# AND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n },\r\n {\r\n \"name\":\"target\",\r\n \"connectors\":[\r\n {\r\n\t\t\t\t\t\"name\":\"connector_name_target\",\r\n \"type\":\"HIVE\",\r\n \"version\":\"1.2\",\r\n\t\t\t\t\t\"data.unit\":\"1h\",\r\n \"config\":{\r\n \"database\"
:\"default\",\r\n \"table.name\":\"demo_src\",\r\n \"where\":\"dt=#YYYYMMdd# AND hour=#HH#\"\r\n },\r\n \"predicates\":[\r\n {\r\n \"type\":\"file.exist\",\r\n \"config\":{\r\n \"root.path\":\"hdfs:///griffin/demo_src\",\r\n \"path\":\"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\r\n }\r\n }\r\n ]\r\n }\r\n ]\r\n }\r\n ]\r\n}"
},
{
"id": "45aef93d-2bcf-4a1f-245a-29611d3d740e",
@@ -1092,7 +1090,7 @@
],
"cookies": [],
"mime": "",
- "text": "[{\"id\":2,\"name\":\"measureName_test_edit\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":18,\"rules\":[{\"id\":10,\"rule\":\"source.id==target.id\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":35,\"name\":\"source\",\"connectors\":[{\"id\":19,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":36,\"name\":\"target\",\"connectors\":[{\"id\":20,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]},{\"id\":6,\"name\":\"third_measure\",\"description\":null,\"organization\":\"ebay\",\"evaluateRule\":{\"id\":6,\"rules\":[{\"id\":6,\"rule\":\"source.id=target.id AND source.age=target.age\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\
",\"data.sources\":[{\"id\":11,\"name\":\"source\",\"connectors\":[{\"id\":11,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":12,\"name\":\"target\",\"connectors\":[{\"id\":12,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]},{\"id\":8,\"name\":\"measure1\",\"description\":null,\"organization\":\"test\",\"evaluateRule\":{\"id\":8,\"rules\":[{\"id\":8,\"rule\":\"source.age=target.age\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":15,\"name\":\"source\",\"connectors\":[{\"id\":15,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":16,\"name\":\"target\",\"connectors\":[{\"id\":16,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]},{\"id\":9,\"
name\":\"measureName_test_edit\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":14,\"rules\":[]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":27,\"name\":null,\"connectors\":[]},{\"id\":28,\"name\":null,\"connectors\":[]}]},{\"id\":10,\"name\":\"measureName1\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":19,\"rules\":[{\"id\":11,\"rule\":\"source.id==target.id\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":37,\"name\":\"source\",\"connectors\":[{\"id\":21,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":38,\"name\":\"target\",\"connectors\":[{\"id\":22,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]}]",
+ "text": "[\n {\n \"measure.type\": \"griffin\",\n \"id\": 1,\n \"name\": \"measureName\",\n \"description\": \"measure description\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"dq.type\": \"accuracy\",\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"id\": 1,\n \"name\": \"source\",\n \"connectors\": [\n {\n \"id\": 1,\n \"name\": \"connector_name_source\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 1,\n \"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/d
emo_src\",\n \"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n },\n {\n \"id\": 2,\n \"name\": \"target\",\n \"connectors\": [\n {\n \"id\": 2,\n \"name\": \"connector_name_target\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 2,\n
\"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n \"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n }\n ],\n \"evaluate.rule\": {\n \"id\": 1,\n \"rules\": [\n {\n \"id\": 1,\n \"rule\": \"source.desc=target.desc\",\n \"name\": \"rule_name\",\n \"descri
ption\": \"Total count\",\n \"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\",\n \"details\": {}\n }\n ]\n }\n },\n {\n \"measure.type\": \"external\",\n \"id\": 2,\n \"name\": \"external_name\",\n \"description\": \" test measure\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"metricName\": \"metricName\"\n }\n]",
"language": "json",
"rawDataType": "text",
"previewType": "text",
@@ -1106,9 +1104,24 @@
"isSample": true,
"scrollToResult": false,
"runTests": false,
- "request": "738b5d6d-4fea-85af-89a8-949468d3cde2",
- "owner": "503523",
- "requestObject": "{\"url\":\"{{BASE_PATH}}/api/v1/measures/owner/:owner\",\"pathVariables\":{\"owner\":\"test\"},\"pathVariableData\":[{\"key\":\"owner\",\"value\":\"test\"}],\"queryParams\":[],\"headerData\":[],\"headers\":\"\",\"data\":null,\"method\":\"GET\",\"dataMode\":\"params\"}"
+ "request": {
+ "url": "{{BASE_PATH}}/api/v1/measures/owner/:owner",
+ "pathVariables": {
+ "owner": "test"
+ },
+ "pathVariableData": [
+ {
+ "key": "owner",
+ "value": "test"
+ }
+ ],
+ "queryParams": [],
+ "headerData": [],
+ "headers": "",
+ "data": null,
+ "method": "GET",
+ "dataMode": "params"
+ }
}
],
"isFromCollection": true,
@@ -2301,7 +2314,7 @@
],
"cookies": [],
"mime": "",
- "text": "{\"id\":1,\"name\":\"measureName_test_edit\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":20,\"rules\":[{\"id\":12,\"rule\":\"source.id = target.id and source.age = target.age and source.desc = target.desc\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":39,\"name\":\"source\",\"connectors\":[{\"id\":23,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":40,\"name\":\"target\",\"connectors\":[{\"id\":24,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]}",
+ "text": "{\n \"measure.type\": \"griffin\",\n \"id\": 1,\n \"name\": \"measureName\",\n \"description\": \"measure description\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"dq.type\": \"accuracy\",\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"id\": 1,\n \"name\": \"source\",\n \"connectors\": [\n {\n \"id\": 1,\n \"name\": \"connector_name_source\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 1,\n \"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n \"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n
}\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n },\n {\n \"id\": 2,\n \"name\": \"target\",\n \"connectors\": [\n {\n \"id\": 2,\n \"name\": \"connector_name_target\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 2,\n \"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n \
"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n }\n ],\n \"evaluate.rule\": {\n \"id\": 1,\n \"rules\": [\n {\n \"id\": 1,\n \"rule\": \"source.desc=target.desc\",\n \"name\": \"rule_name\",\n \"description\": \"Total count\",\n \"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\",\n \"details\": {}\n }\n ]\n }\n}",
"language": "json",
"rawDataType": "text",
"previewType": "text",
@@ -2332,8 +2345,7 @@
"data": null,
"method": "GET",
"dataMode": "params"
- },
- "owner": "503523"
+ }
}
],
"collection_id": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63"
@@ -2804,6 +2816,7 @@
"headers": "",
"headerData": [],
"url": "{{BASE_PATH}}/api/v1/measures",
+ "folder": "523a7f9f-1970-018e-9241-57caa3d6ea60",
"queryParams": [],
"preRequestScript": null,
"pathVariables": {},
@@ -2814,8 +2827,8 @@
"version": 2,
"tests": null,
"currentHelper": "normal",
- "helperAttributes": "{}",
- "time": 1508997057521,
+ "helperAttributes": {},
+ "time": 1516340119702,
"name": "Get measures",
"description": "`GET /api/v1/measures`",
"collectionId": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63",
@@ -2873,7 +2886,7 @@
],
"cookies": [],
"mime": "",
- "text": "[{\"id\":2,\"name\":\"measureName_test_edit\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":18,\"rules\":[{\"id\":10,\"rule\":\"source.id==target.id\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":35,\"name\":\"source\",\"connectors\":[{\"id\":19,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":36,\"name\":\"target\",\"connectors\":[{\"id\":20,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]},{\"id\":6,\"name\":\"third_measure\",\"description\":null,\"organization\":\"ebay\",\"evaluateRule\":{\"id\":6,\"rules\":[{\"id\":6,\"rule\":\"source.id=target.id AND source.age=target.age\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\
",\"data.sources\":[{\"id\":11,\"name\":\"source\",\"connectors\":[{\"id\":11,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":12,\"name\":\"target\",\"connectors\":[{\"id\":12,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]},{\"id\":8,\"name\":\"measure1\",\"description\":null,\"organization\":\"test\",\"evaluateRule\":{\"id\":8,\"rules\":[{\"id\":8,\"rule\":\"source.age=target.age\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":15,\"name\":\"source\",\"connectors\":[{\"id\":15,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":16,\"name\":\"target\",\"connectors\":[{\"id\":16,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]},{\"id\":9,\"
name\":\"measureName_test_edit\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":14,\"rules\":[]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":27,\"name\":null,\"connectors\":[]},{\"id\":28,\"name\":null,\"connectors\":[]}]},{\"id\":10,\"name\":\"measureName1\",\"description\":\"This is a test measure\",\"organization\":\"orgName\",\"evaluateRule\":{\"id\":19,\"rules\":[{\"id\":11,\"rule\":\"source.id==target.id\",\"dsl.type\":\"griffin-dsl\",\"dq.type\":\"accuracy\"}]},\"owner\":\"test\",\"deleted\":false,\"process.type\":\"batch\",\"data.sources\":[{\"id\":37,\"name\":\"source\",\"connectors\":[{\"id\":21,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_src\"}}]},{\"id\":38,\"name\":\"target\",\"connectors\":[{\"id\":22,\"type\":\"HIVE\",\"version\":\"1.2\",\"config\":{\"database\":\"default\",\"table.name\":\"demo_tgt\"}}]}]}]",
+ "text": "[\n {\n \"id\": 1,\n \"name\": \"measureName\",\n \"description\": \"measure description\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"dq.type\": \"accuracy\",\n \"process.type\": \"batch\",\n \"data.sources\": [\n {\n \"id\": 1,\n \"name\": \"source\",\n \"connectors\": [\n {\n \"id\": 1,\n \"name\": \"connector_name_source\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 1,\n \"type\": \"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n
\"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n },\n {\n \"id\": 2,\n \"name\": \"target\",\n \"connectors\": [\n {\n \"id\": 2,\n \"name\": \"connector_name_target\",\n \"type\": \"HIVE\",\n \"version\": \"1.2\",\n \"predicates\": [\n {\n \"id\": 2,\n \"type\":
\"file.exist\",\n \"config\": {\n \"root.path\": \"hdfs:///griffin/demo_src\",\n \"path\": \"/dt=#YYYYMMdd#/hour=#HH#/_DONE\"\n }\n }\n ],\n \"data.unit\": \"1h\",\n \"config\": {\n \"database\": \"default\",\n \"table.name\": \"demo_src\",\n \"where\": \"dt=#YYYYMMdd# AND hour=#HH#\"\n }\n }\n ]\n }\n ],\n \"evaluate.rule\": {\n \"id\": 1,\n \"rules\": [\n {\n \"id\": 1,\n \"rule\": \"source.desc=target.desc\",\n \"name\": \"rule_name\",\n \"description\": \"Total count\",\n
\"dsl.type\": \"griffin-dsl\",\n \"dq.type\": \"accuracy\",\n \"details\": {}\n }\n ]\n },\n \"measure.type\": \"griffin\"\n },\n {\n \"id\": 2,\n \"name\": \"external_name\",\n \"description\": \" test measure\",\n \"organization\": \"orgName\",\n \"owner\": \"test\",\n \"deleted\": false,\n \"metricName\": \"metricName\",\n \"measure.type\": \"external\"\n }\n]",
"language": "json",
"rawDataType": "text",
"previewType": "text",
@@ -2887,14 +2900,19 @@
"isSample": true,
"scrollToResult": false,
"runTests": false,
- "request": "d4242bb8-d273-6bdd-588a-ec5367c3fe57",
- "owner": "503523",
- "requestObject": "{\"url\":\"{{BASE_PATH}}/api/v1/measures\",\"pathVariables\":{},\"pathVariableData\":[],\"queryParams\":[],\"headerData\":[],\"headers\":\"\",\"data\":null,\"method\":\"GET\",\"dataMode\":\"params\"}"
+ "request": {
+ "url": "{{BASE_PATH}}/api/v1/measures",
+ "pathVariables": {},
+ "pathVariableData": [],
+ "queryParams": [],
+ "headerData": [],
+ "headers": "",
+ "data": null,
+ "method": "GET",
+ "dataMode": "params"
+ }
}
- ],
- "collection_id": "a743e1b9-583f-6bd7-e2ae-f03a1f807c63",
- "isFromCollection": true,
- "folder": "523a7f9f-1970-018e-9241-57caa3d6ea60"
+ ]
},
{
"id": "f989dff6-0847-cc8a-0989-ccae76f33562",
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/config/PropertiesConfig.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/config/PropertiesConfig.java b/service/src/main/java/org/apache/griffin/core/config/PropertiesConfig.java
index 95b8676..bfaba35 100644
--- a/service/src/main/java/org/apache/griffin/core/config/PropertiesConfig.java
+++ b/service/src/main/java/org/apache/griffin/core/config/PropertiesConfig.java
@@ -47,14 +47,14 @@ public class PropertiesConfig {
this.location = location;
}
- private String getPath(String defaultPath, String name) {
+ private String getPath(String defaultPath, String name) throws FileNotFoundException {
String path = defaultPath;
File file = new File(location);
LOGGER.info("File absolute path:" + file.getAbsolutePath());
File[] files = file.listFiles();
if (files == null || files.length == 0) {
LOGGER.error("The defaultPath {} does not exist.Please check your config in application.properties.", location);
- throw new NullPointerException();
+ throw new FileNotFoundException();
}
for (File f : files) {
if (f.getName().equals(name)) {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/FileExistPredicator.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/FileExistPredicator.java b/service/src/main/java/org/apache/griffin/core/job/FileExistPredicator.java
index a97d812..7354176 100644
--- a/service/src/main/java/org/apache/griffin/core/job/FileExistPredicator.java
+++ b/service/src/main/java/org/apache/griffin/core/job/FileExistPredicator.java
@@ -32,8 +32,8 @@ import static org.apache.griffin.core.job.JobInstance.PATH_CONNECTOR_CHARACTER;
public class FileExistPredicator implements Predicator {
private static final Logger LOGGER = LoggerFactory.getLogger(FileExistPredicator.class);
- public static final String PREDICT_PATH = "path";
- public static final String PREDICT_ROOT_PATH = "root.path";
+ private static final String PREDICT_PATH = "path";
+ private static final String PREDICT_ROOT_PATH = "root.path";
private SegmentPredicate predicate;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/JobInstance.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/JobInstance.java b/service/src/main/java/org/apache/griffin/core/job/JobInstance.java
index 0c8b554..ba0b1fb 100644
--- a/service/src/main/java/org/apache/griffin/core/job/JobInstance.java
+++ b/service/src/main/java/org/apache/griffin/core/job/JobInstance.java
@@ -52,10 +52,10 @@ import static org.quartz.TriggerKey.triggerKey;
@DisallowConcurrentExecution
public class JobInstance implements Job {
private static final Logger LOGGER = LoggerFactory.getLogger(JobInstance.class);
- static final String MEASURE_KEY = "measure";
- static final String PREDICATES_KEY = "predicts";
- static final String PREDICATE_JOB_NAME = "predicateJobName";
- static final String JOB_NAME = "jobName";
+ public static final String MEASURE_KEY = "measure";
+ public static final String PREDICATES_KEY = "predicts";
+ public static final String PREDICATE_JOB_NAME = "predicateJobName";
+ public static final String JOB_NAME = "jobName";
static final String PATH_CONNECTOR_CHARACTER = ",";
@Autowired
@@ -201,14 +201,14 @@ public class JobInstance implements Job {
* @param conf map with file predicate,data split and partitions info
* @param sampleTs collection of data split start timestamp
* @return all config data combine,like {"where": "year=2017 AND month=11 AND dt=15 AND hour=09,year=2017 AND month=11 AND dt=15 AND hour=10"}
- * or like {"path": "/year=#2017/month=11/dt=15/hour=09/_DONE,/year=#2017/month=11/dt=15/hour=10/_DONE"}
+ * or like {"path": "/year=2017/month=11/dt=15/hour=09/_DONE,/year=2017/month=11/dt=15/hour=10/_DONE"}
*/
private void genConfMap(Map<String, String> conf, Long[] sampleTs) {
for (Map.Entry<String, String> entry : conf.entrySet()) {
String value = entry.getValue();
Set<String> set = new HashSet<>();
for (Long timestamp : sampleTs) {
- set.add(TimeUtil.format(value, timestamp));
+ set.add(TimeUtil.format(value, timestamp,jobSchedule.getTimeZone()));
}
conf.put(entry.getKey(), StringUtils.join(set, PATH_CONNECTOR_CHARACTER));
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
index 92188b4..ef2fb9f 100644
--- a/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/job/JobServiceImpl.java
@@ -38,6 +38,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.DataAccessException;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Sort;
@@ -62,8 +63,8 @@ import static org.quartz.TriggerKey.triggerKey;
@Service
public class JobServiceImpl implements JobService {
private static final Logger LOGGER = LoggerFactory.getLogger(JobServiceImpl.class);
- static final String JOB_SCHEDULE_ID = "jobScheduleId";
- static final String GRIFFIN_JOB_ID = "griffinJobId";
+ public static final String JOB_SCHEDULE_ID = "jobScheduleId";
+ public static final String GRIFFIN_JOB_ID = "griffinJobId";
static final int MAX_PAGE_SIZE = 1024;
static final int DEFAULT_PAGE_SIZE = 10;
@@ -143,26 +144,22 @@ public class JobServiceImpl implements JobService {
public GriffinOperationMessage addJob(JobSchedule js) throws Exception {
Long measureId = js.getMeasureId();
GriffinMeasure measure = getMeasureIfValid(measureId);
- if (measure != null && addJob(js, measure)) {
+ if (measure != null) {
+ String qName = getQuartzName(js);
+ String qGroup = getQuartzGroupName();
+ TriggerKey triggerKey = triggerKey(qName, qGroup);
+ if (!isJobScheduleParamValid(js, measure) || factory.getObject().checkExists(triggerKey)) {
+ return CREATE_JOB_FAIL;
+ }
+ GriffinJob job = new GriffinJob(measure.getId(), js.getJobName(), qName, qGroup, false);
+ job = jobRepo.save(job);
+ js = jobScheduleRepo.save(js);
+ addJob(triggerKey, js, job);
return CREATE_JOB_SUCCESS;
}
return CREATE_JOB_FAIL;
}
- private boolean addJob(JobSchedule js, GriffinMeasure measure) throws Exception {
- String qName = getQuartzName(js);
- String qGroup = getQuartzGroupName();
- TriggerKey triggerKey = triggerKey(qName, qGroup);
- if (!isJobScheduleParamValid(js, measure) || factory.getObject().checkExists(triggerKey)) {
- return false;
- }
- GriffinJob job = new GriffinJob(measure.getId(), js.getJobName(), qName, qGroup, false);
- jobRepo.save(job);
- js = jobScheduleRepo.save(js);
- addJob(triggerKey, js, job);
- return true;
- }
-
private void addJob(TriggerKey triggerKey, JobSchedule js, GriffinJob job) throws Exception {
Scheduler scheduler = factory.getObject();
JobDetail jobDetail = addJobDetail(scheduler, triggerKey, js, job);
@@ -185,7 +182,7 @@ public class JobServiceImpl implements JobService {
return false;
}
List<String> names = getConnectorNames(measure);
- return isConnectorNamesValid(js.getSegments(), names);
+ return names != null && isConnectorNamesValid(js.getSegments(), names);
}
private boolean isJobNameValid(String jobName) {
@@ -212,11 +209,18 @@ public class JobServiceImpl implements JobService {
}
private boolean isConnectorNamesValid(List<JobDataSegment> segments, List<String> names) {
+ Set<String> dcSets = new HashSet<>();
for (JobDataSegment segment : segments) {
- if (!isConnectorNameValid(segment.getDataConnectorName(), names)) {
+ String dcName = segment.getDataConnectorName();
+ dcSets.add(dcName);
+ if (!isConnectorNameValid(dcName, names)) {
return false;
}
}
+ if (dcSets.size() < segments.size()) {
+ LOGGER.warn("Connector names in job data segment cannot be repeated.");
+ return false;
+ }
return true;
}
@@ -239,11 +243,11 @@ public class JobServiceImpl implements JobService {
sets.add(dc.getName());
});
}
- names.addAll(sets);
- if (names.size() < sets.size()) {
- LOGGER.error("Connector names cannot be repeated.");
- throw new IllegalArgumentException();
+ if (sets.size() < sources.size()) {
+ LOGGER.warn("Connector names cannot be repeated.");
+ return null;
}
+ names.addAll(sets);
return names;
}
@@ -403,7 +407,7 @@ public class JobServiceImpl implements JobService {
private boolean deleteJob(String group, String name) throws SchedulerException {
Scheduler scheduler = factory.getObject();
JobKey jobKey = new JobKey(name, group);
- if (scheduler.checkExists(jobKey)) {
+ if (!scheduler.checkExists(jobKey)) {
LOGGER.warn("Job({},{}) does not exist.", jobKey.getGroup(), jobKey.getName());
return true;
}
@@ -424,10 +428,11 @@ public class JobServiceImpl implements JobService {
LOGGER.info("Measure id {} has no related jobs.", measureId);
return true;
}
+ boolean status = true;
for (GriffinJob job : jobs) {
- deleteJob(job);
+ status = status && deleteJob(job);
}
- return true;
+ return status;
}
@Override
@@ -445,12 +450,13 @@ public class JobServiceImpl implements JobService {
@Scheduled(fixedDelayString = "${jobInstance.expired.milliseconds}")
public void deleteExpiredJobInstance() {
- List<JobInstanceBean> instances = jobInstanceRepo.findByExpireTmsLessThanEqual(System.currentTimeMillis());
+ Long timeMills = System.currentTimeMillis();
+ List<JobInstanceBean> instances = jobInstanceRepo.findByExpireTmsLessThanEqual(timeMills);
if (!pauseJob(instances)) {
LOGGER.error("Pause job failure.");
return;
}
- jobInstanceRepo.deleteByExpireTimestamp(System.currentTimeMillis());
+ jobInstanceRepo.deleteByExpireTimestamp(timeMills);
LOGGER.info("Delete expired job instances success.");
}
@@ -485,6 +491,8 @@ public class JobServiceImpl implements JobService {
LOGGER.error("Job instance json converts to map failed. {}", e.getMessage());
} catch (IllegalArgumentException e) {
LOGGER.error("Livy status is illegal. {}", e.getMessage());
+ } catch (Exception e) {
+ LOGGER.error("Sync job instances failure. {}",e.getMessage());
}
}
@@ -499,6 +507,7 @@ public class JobServiceImpl implements JobService {
instance.setAppUri(appUri);
}
jobInstanceRepo.save(instance);
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/entity/GriffinJob.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/GriffinJob.java b/service/src/main/java/org/apache/griffin/core/job/entity/GriffinJob.java
index 65d8e15..ee5e107 100644
--- a/service/src/main/java/org/apache/griffin/core/job/entity/GriffinJob.java
+++ b/service/src/main/java/org/apache/griffin/core/job/entity/GriffinJob.java
@@ -65,11 +65,11 @@ public class GriffinJob extends AbstractJob {
super();
}
- public GriffinJob(Long measureId, String jobName, String qJobName, String qGroupName, boolean deleted) {
+ public GriffinJob(Long measureId, String jobName, String quartzName, String quartzGroup, boolean deleted) {
super(measureId, jobName, deleted);
this.metricName = jobName;
- this.quartzName = qJobName;
- this.quartzGroup = qGroupName;
+ this.quartzName = quartzName;
+ this.quartzGroup = quartzGroup;
}
public GriffinJob(Long jobId, Long measureId, String jobName, String qJobName, String qGroupName, boolean deleted) {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java b/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java
index b0f81cb..e765702 100644
--- a/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java
+++ b/service/src/main/java/org/apache/griffin/core/job/entity/JobDataSegment.java
@@ -40,7 +40,7 @@ public class JobDataSegment extends AbstractAuditableEntity {
@OneToOne(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE, CascadeType.MERGE})
@JoinColumn(name = "segment_range_id")
- private SegmentRange segmentRange;
+ private SegmentRange segmentRange = new SegmentRange();
@JsonProperty("as.baseline")
public Boolean getBaseline() {
@@ -80,7 +80,13 @@ public class JobDataSegment extends AbstractAuditableEntity {
}
public JobDataSegment(String dataConnectorName, boolean baseline) {
- this.dataConnectorName =dataConnectorName;
+ this.dataConnectorName = dataConnectorName;
+ this.baseline = baseline;
+ }
+
+ public JobDataSegment(String dataConnectorName, boolean baseline, SegmentRange segmentRange) {
+ this.dataConnectorName = dataConnectorName;
this.baseline = baseline;
+ this.segmentRange = segmentRange;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java b/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java
index 1406b5e..d1dd44f 100644
--- a/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java
+++ b/service/src/main/java/org/apache/griffin/core/job/entity/JobSchedule.java
@@ -30,11 +30,7 @@ import org.apache.griffin.core.util.PropertiesUtil;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Configurable;
-import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.core.io.ClassPathResource;
-import org.springframework.stereotype.Component;
import javax.persistence.*;
import javax.validation.constraints.NotNull;
@@ -155,7 +151,7 @@ public class JobSchedule extends AbstractAuditableEntity {
*/
private Map<String, Object> defaultPredicatesConfig() throws JsonProcessingException {
String path = "/application.properties";
- Properties appConf = PropertiesUtil.getProperties(path,new ClassPathResource(path));
+ Properties appConf = PropertiesUtil.getProperties(path, new ClassPathResource(path));
Map<String, Object> scheduleConf = new HashMap<>();
Map<String, Object> map = new HashMap<>();
map.put("interval", appConf.getProperty("predicate.job.interval"));
@@ -176,7 +172,7 @@ public class JobSchedule extends AbstractAuditableEntity {
public JobSchedule() throws JsonProcessingException {
}
- public JobSchedule(Long measureId, String jobName, String cronExpression,String timeZone, List<JobDataSegment> segments) throws JsonProcessingException {
+ public JobSchedule(Long measureId, String jobName, String cronExpression, String timeZone, List<JobDataSegment> segments) throws JsonProcessingException {
this.measureId = measureId;
this.jobName = jobName;
this.cronExpression = cronExpression;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/entity/SegmentPredicate.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/SegmentPredicate.java b/service/src/main/java/org/apache/griffin/core/job/entity/SegmentPredicate.java
index 0f5a624..78b2794 100644
--- a/service/src/main/java/org/apache/griffin/core/job/entity/SegmentPredicate.java
+++ b/service/src/main/java/org/apache/griffin/core/job/entity/SegmentPredicate.java
@@ -78,4 +78,9 @@ public class SegmentPredicate extends AbstractAuditableEntity {
public SegmentPredicate() {
}
+
+ public SegmentPredicate(String type, Map configMap) throws JsonProcessingException {
+ this.type = type;
+ setConfigMap(configMap);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/job/entity/SegmentRange.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/job/entity/SegmentRange.java b/service/src/main/java/org/apache/griffin/core/job/entity/SegmentRange.java
index b8ca5cf..5393f22 100644
--- a/service/src/main/java/org/apache/griffin/core/job/entity/SegmentRange.java
+++ b/service/src/main/java/org/apache/griffin/core/job/entity/SegmentRange.java
@@ -50,4 +50,12 @@ public class SegmentRange extends AbstractAuditableEntity {
this.length = length;
}
+ public SegmentRange(String begin, String length) {
+ this.begin = begin;
+ this.length = length;
+ }
+
+ SegmentRange() {
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperationImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperationImpl.java b/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperationImpl.java
index ca9aae1..ab04567 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperationImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/ExternalMeasureOperationImpl.java
@@ -81,17 +81,17 @@ public class ExternalMeasureOperationImpl implements MeasureOperation {
}
@Override
- public Boolean delete(Measure measure) {
+ public GriffinOperationMessage delete(Measure measure) {
try {
ExternalMeasure em = (ExternalMeasure) measure;
em.setDeleted(true);
em.getVirtualJob().setDeleted(true);
measureRepo.save(em);
- return true;
+ return DELETE_MEASURE_BY_ID_SUCCESS;
} catch (Exception e) {
LOGGER.error("Failed to delete measure. {}", e.getMessage());
}
- return false;
+ return DELETE_MEASURE_BY_ID_FAIL;
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperationImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperationImpl.java b/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperationImpl.java
index f21b60d..b5d9805 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperationImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/GriffinMeasureOperationImpl.java
@@ -77,14 +77,19 @@ public class GriffinMeasureOperationImpl implements MeasureOperation {
}
@Override
- public Boolean delete(Measure measure) {
- boolean pauseStatus = jobService.deleteJobsRelateToMeasure(measure.getId());
- if (!pauseStatus) {
- return false;
+ public GriffinOperationMessage delete(Measure measure) {
+ try {
+ boolean pauseStatus = jobService.deleteJobsRelateToMeasure(measure.getId());
+ if (!pauseStatus) {
+ return DELETE_MEASURE_BY_ID_FAIL;
+ }
+ measure.setDeleted(true);
+ measureRepo.save(measure);
+ return DELETE_MEASURE_BY_ID_SUCCESS;
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage());
}
- measure.setDeleted(true);
- measureRepo.save(measure);
- return true;
+ return DELETE_MEASURE_BY_ID_FAIL;
}
private boolean isConnectorNamesValid(GriffinMeasure measure) {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/MeasureOperation.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureOperation.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureOperation.java
index 80f1f30..81e9f06 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/MeasureOperation.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureOperation.java
@@ -29,6 +29,6 @@ public interface MeasureOperation {
GriffinOperationMessage update(Measure measure);
- Boolean delete(Measure measure);
+ GriffinOperationMessage delete(Measure measure);
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgService.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgService.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgService.java
index 754f3d1..228d5bf 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgService.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureOrgService.java
@@ -19,7 +19,6 @@ under the License.
package org.apache.griffin.core.measure;
-import java.io.Serializable;
import java.util.List;
import java.util.Map;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java b/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java
index 34a780d..86dc9a9 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/MeasureServiceImpl.java
@@ -94,15 +94,8 @@ public class MeasureServiceImpl implements MeasureService {
if (measure == null) {
return RESOURCE_NOT_FOUND;
}
- try {
- MeasureOperation op = getOperation(measure);
- if (op.delete(measure)) {
- return DELETE_MEASURE_BY_ID_SUCCESS;
- }
- } catch (Exception e) {
- LOGGER.error("Delete measure id: {} name: {} failure. {}", measure.getId(), measure.getName(), e.getMessage());
- }
- return DELETE_MEASURE_BY_ID_FAIL;
+ MeasureOperation op = getOperation(measure);
+ return op.delete(measure);
}
private MeasureOperation getOperation(Measure measure) {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/entity/ExternalMeasure.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/ExternalMeasure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/ExternalMeasure.java
index eb4a19d..2339df0 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/entity/ExternalMeasure.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/entity/ExternalMeasure.java
@@ -43,9 +43,10 @@ public class ExternalMeasure extends Measure {
super();
}
- public ExternalMeasure(String name, String description, String organization, String owner, String metricName) {
+ public ExternalMeasure(String name, String description, String organization, String owner, String metricName,VirtualJob vj) {
super(name, description, organization, owner);
this.metricName = metricName;
+ this.virtualJob = vj;
}
public String getMetricName() {
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
index c448c0b..6b060b5 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/entity/GriffinMeasure.java
@@ -34,13 +34,14 @@ import java.util.List;
@Entity
public class GriffinMeasure extends Measure {
+ private String dqType;
+
private String processType;
@Transient
@JsonInclude(JsonInclude.Include.NON_NULL)
private Long timestamp;
-
@NotNull
@OneToMany(fetch = FetchType.EAGER, cascade = {CascadeType.PERSIST, CascadeType.REMOVE, CascadeType.MERGE})
@JoinColumn(name = "measure_id")
@@ -51,6 +52,16 @@ public class GriffinMeasure extends Measure {
@JoinColumn(name = "evaluate_rule_id")
private EvaluateRule evaluateRule;
+ @JsonProperty("dq.type")
+ public String getDqType() {
+ return dqType;
+ }
+
+ @JsonProperty("dq.type")
+ public void setDqType(String dqType) {
+ this.dqType = dqType;
+ }
+
@JsonProperty("process.type")
public String getProcessType() {
return processType;
@@ -104,10 +115,9 @@ public class GriffinMeasure extends Measure {
super();
}
- public GriffinMeasure(Long measureId,String name, String description, String organization, String processType, String owner, List<DataSource> dataSources, EvaluateRule evaluateRule) {
- super(name, description, organization, owner);
- this.setId(measureId);
- this.processType = processType;
+ public GriffinMeasure(String name, String owner, List<DataSource> dataSources, EvaluateRule evaluateRule) {
+ this.name = name;
+ this.owner = owner;
this.dataSources = dataSources;
this.evaluateRule = evaluateRule;
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
index cf2daec..a5c97a1 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/entity/Measure.java
@@ -19,6 +19,7 @@ under the License.
package org.apache.griffin.core.measure.entity;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
@@ -29,7 +30,7 @@ import javax.validation.constraints.NotNull;
@Entity
@Inheritance(strategy = InheritanceType.JOINED)
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type")
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "measure.type")
@JsonSubTypes({@JsonSubTypes.Type(value = GriffinMeasure.class, name = "griffin"), @JsonSubTypes.Type(value = ExternalMeasure.class, name = "external")})
public abstract class Measure extends AbstractAuditableEntity {
private static final long serialVersionUID = -4748881017029815714L;
@@ -95,5 +96,6 @@ public abstract class Measure extends AbstractAuditableEntity {
this.owner = owner;
}
+ @JsonProperty("measure.type")
public abstract String getType();
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java b/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java
index f0c6516..8a61bfb 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/entity/Rule.java
@@ -40,25 +40,23 @@ public class Rule extends AbstractAuditableEntity {
private String dqType;
- @Column(length = 10 * 1024)
+ @Column(length = 8 * 1024)
private String rule;
- @JsonIgnore
private String name;
- @JsonIgnore
+ @Column(length = 1024)
private String description;
@JsonIgnore
@Access(AccessType.PROPERTY)
- @Column(length = 10 * 1024)
+ @Column(length = 1024)
private String details;
@Transient
@JsonInclude(JsonInclude.Include.NON_NULL)
private Map<String, Object> detailsMap;
-
@JsonProperty("dsl.type")
public String getDslType() {
return dslType;
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
index 976bec2..c88cd3a 100644
--- a/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
+++ b/service/src/main/java/org/apache/griffin/core/measure/repo/MeasureRepo.java
@@ -42,14 +42,4 @@ public interface MeasureRepo<T extends Measure> extends CrudRepository<T, Long>
@Query("select m.name from #{#entityName} m " +
"where m.organization= ?1 and m.deleted= ?2")
List<String> findNameByOrganization(String organization, Boolean deleted);
-
- @Query("select m.organization from #{#entityName} m " +
- "where m.name= ?1")
- String findOrgByName(String measureName);
-
-// @Modifying
-// @Transactional
-// @Query("update Measure m "+
-// "set m.description= ?2,m.organization= ?3,m.source= ?4,m.target= ?5,m.evaluateRule= ?6 where m.id= ?1")
-// void update(Long Id, String description, String organization, DataConnector source, DataConnector target, EvaluateRule evaluateRule);
}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/util/AvroUtil.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/util/AvroUtil.java b/service/src/main/java/org/apache/griffin/core/util/AvroUtil.java
deleted file mode 100644
index e1fbf99..0000000
--- a/service/src/main/java/org/apache/griffin/core/util/AvroUtil.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
-*/
-
-package org.apache.griffin.core.util;
-
-import org.apache.avro.Schema;
-
-public class AvroUtil {
-
- public static Schema schemaOf(String schema) {
- return new Schema.Parser().parse(schema);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-griffin/blob/71fcf93b/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java
----------------------------------------------------------------------
diff --git a/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java b/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java
index 859fe5b..5f4396a 100644
--- a/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java
+++ b/service/src/main/java/org/apache/griffin/core/util/TimeUtil.java
@@ -23,10 +23,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.IllegalFormatException;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -55,8 +52,8 @@ public class TimeUtil {
list.add(group.toLowerCase());
}
long time = 0;
- for (int i = 0; i < list.size(); i++) {
- long t = milliseconds(list.get(i).toLowerCase());
+ for (String aList : list) {
+ long t = milliseconds(aList.toLowerCase());
if (positive) {
time += t;
} else {
@@ -67,23 +64,18 @@ public class TimeUtil {
}
private static Long milliseconds(String str) {
- try {
- if (str.endsWith("ms")) {
- return milliseconds(Long.parseLong(str.substring(0, str.length() - 2)), TimeUnit.MILLISECONDS);
- } else if (str.endsWith("s")) {
- return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.SECONDS);
- } else if (str.endsWith("m")) {
- return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.MINUTES);
- } else if (str.endsWith("h")) {
- return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.HOURS);
- } else if (str.endsWith("d")) {
- return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.DAYS);
- } else {
- LOGGER.error("Time string format error.It only supports d(day),h(hour),m(minute),s(second),ms(millsecond).Please check your time format.)");
- throw new IllegalArgumentException();
- }
- } catch (Exception e) {
- LOGGER.error("Parse exception occur. {}",e);
+ if (str.endsWith("ms")) {
+ return milliseconds(Long.parseLong(str.substring(0, str.length() - 2)), TimeUnit.MILLISECONDS);
+ } else if (str.endsWith("s")) {
+ return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.SECONDS);
+ } else if (str.endsWith("m")) {
+ return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.MINUTES);
+ } else if (str.endsWith("h")) {
+ return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.HOURS);
+ } else if (str.endsWith("d")) {
+ return milliseconds(Long.parseLong(str.substring(0, str.length() - 1)), TimeUnit.DAYS);
+ } else {
+ LOGGER.error("Time string format error.It only supports d(day),h(hour),m(minute),s(second),ms(millsecond).Please check your time format.)");
return 0L;
}
}
@@ -92,7 +84,7 @@ public class TimeUtil {
return unit.toMillis(duration);
}
- public static String format(String timeFormat, long time) {
+ public static String format(String timeFormat, long time,String timeZone) {
String timePattern = "#(?:\\\\#|[^#])*#";
Date t = new Date(time);
Pattern ptn = Pattern.compile(timePattern);
@@ -103,11 +95,11 @@ public class TimeUtil {
String content = group.substring(1, group.length() - 1);
String pattern = refreshEscapeHashTag(content);
SimpleDateFormat sdf = new SimpleDateFormat(pattern);
+ sdf.setTimeZone(TimeZone.getTimeZone(timeZone));
matcher.appendReplacement(sb, sdf.format(t));
}
matcher.appendTail(sb);
- String endString = refreshEscapeHashTag(sb.toString());
- return endString;
+ return refreshEscapeHashTag(sb.toString());
}
private static String refreshEscapeHashTag(String str) {