You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by GitBox <gi...@apache.org> on 2022/08/12 07:26:29 UTC
[GitHub] [dolphinscheduler] guodongym opened a new pull request, #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
guodongym opened a new pull request, #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157
<!--Thanks very much for contributing to Apache DolphinScheduler. Please review https://dolphinscheduler.apache.org/en-us/community/development/pull-request.html before opening a pull request.-->
## Purpose of the pull request
add dataSource for AWS Athena
close #10273
<!--(For example: This pull request adds checkstyle plugin).-->
## Brief change log
* Add the ATHENA backend
* Add the ATHENA test case
* Add the ATHENA frontend
<!--*(for example:)*
- *Add maven-checkstyle-plugin to root pom.xml*
-->
## Verify this pull request
<!--*(Please pick either of the following options)*-->
This pull request is code cleanup without any test coverage.
*(or)*
This pull request is already covered by existing tests, such as *(please describe tests)*.
(or)
This change added tests and can be verified as follows:
<!--*(example:)*
- *Added dolphinscheduler-dao tests for end-to-end.*
- *Added CronUtilsTest to verify the change.*
- *Manually verified the change by testing locally.* -->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r939648959
##########
docs/docs/en/guide/datasource/athena.md:
##########
@@ -0,0 +1,21 @@
+# AWS Athena
+
+
+
+## Datasource Parameters
+
+| **Datasource** | **Description** |
+| --- | --- |
+| Datasource | Select ATHENA. |
+| Datasource name | Enter the name of the DataSource. |
+| Description | Enter a description of the DataSource. |
+| Username | Set the AWS access key. |
+| Password | Set the AWS secret access key. |
+| AwsRegion | Set the AWS region. |
+| Database name | Enter the database name of the ATHENA connection. |
+| Jdbc connection parameters | Parameter settings for ATHENA connection, in JSON format. |
+
+## Native Supported
+
+No, read section example in [datasource-setting](../howto/datasource-setting.md) `DataSource Center` section to activate this datasource.
Review Comment:
If not native supported, can we provider the JDBC connection name to download, better add some download link for it
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import com.google.auto.service.AutoService;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@AutoService(DataSourceProcessor.class)
+public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
+ return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class);
+ }
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ AthenaConnectionParam
+ connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson);
+
+ AthenaDataSourceParamDTO
+ athenaDatasourceParamDTO = new AthenaDataSourceParamDTO();
+ athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion());
+ athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ athenaDatasourceParamDTO.setUserName(connectionParams.getUser());
+ athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther()));
+
+ return athenaDatasourceParamDTO;
+ }
+
+ @Override
+ public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
+ AthenaDataSourceParamDTO athenaParam = (AthenaDataSourceParamDTO) datasourceParam;
+ String address = String.format("%s%s=%s;", Constants.JDBC_ATHENA, "AwsRegion", athenaParam.getAwsRegion());
+
+ AthenaConnectionParam
+ athenaConnectionParam = new AthenaConnectionParam();
+ athenaConnectionParam.setUser(athenaParam.getUserName());
+ athenaConnectionParam.setPassword(PasswordUtils.encodePassword(athenaParam.getPassword()));
+ athenaConnectionParam.setAwsRegion(athenaParam.getAwsRegion());
+ athenaConnectionParam.setOther(this.transformOther(athenaParam.getOther()));
+ athenaConnectionParam.setAddress(address);
+ athenaConnectionParam.setJdbcUrl(address);
+ athenaConnectionParam.setDatabase(athenaParam.getDatabase());
+ athenaConnectionParam.setDriverClassName(this.getDatasourceDriver());
+ athenaConnectionParam.setValidationQuery(this.getValidationQuery());
+ athenaConnectionParam.setProps(athenaParam.getOther());
+
+ return athenaConnectionParam;
+ }
+
+ @Override
+ public ConnectionParam createConnectionParams(String connectionJson) {
+ return JSONUtils.parseObject(connectionJson, AthenaConnectionParam.class);
+ }
+
+ @Override
+ public String getDatasourceDriver() {
+ return Constants.COM_ATHENA_JDBC_DRIVER;
+ }
+
+ @Override
+ public String getValidationQuery() {
+ return Constants.ATHENA_VALIDATION_QUERY;
+ }
+
+ @Override
+ public String getJdbcUrl(ConnectionParam connectionParam) {
+ AthenaConnectionParam
+ athenaConnectionParam = (AthenaConnectionParam) connectionParam;
+ if (!StringUtils.isEmpty(athenaConnectionParam.getOther())) {
+ return String.format("%s%s", athenaConnectionParam.getJdbcUrl(), athenaConnectionParam.getOther());
+ }
+ return athenaConnectionParam.getJdbcUrl();
+ }
+
+ @Override
+ public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
+ AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam;
+ Class.forName(this.getDatasourceDriver());
+ return DriverManager.getConnection(this.getJdbcUrl(connectionParam),
+ athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword()));
+ }
+
+ @Override
+ public DbType getDbType() {
+ return DbType.ATHENA;
+ }
+
+ @Override
+ public DataSourceProcessor create() {
+ return new AthenaDataSourceProcessor();
+ }
+
+ private String transformOther(Map<String, String> otherMap) {
+ if (MapUtils.isNotEmpty(otherMap)) {
+ List<String> list = new ArrayList<>(otherMap.size());
+ otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
+ return String.join(Constants.SEMICOLON, list);
+ }
+ return null;
+ }
+
+ private Map<String, String> parseOther(String other) {
+ Map<String, String> otherMap = new LinkedHashMap<>();
+ if (StringUtils.isEmpty(other)) {
+ return otherMap;
+ }
+ String[] configs = other.split(Constants.SEMICOLON);
+ for (String config : configs) {
+ otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]);
+ }
+ return otherMap;
+ }
+
+ @Override
+ protected void checkHost(String host) {
+ // Do not need to set the host, nothing to do
+ }
+
+ @Override
+ protected void checkDatabasePatter(String database) {
+ // Do not need to set the database, nothing to do
+ }
Review Comment:
we can remove those method
##########
docs/docs/en/guide/datasource/athena.md:
##########
@@ -0,0 +1,21 @@
+# AWS Athena
+
+
Review Comment:
```suggestion
```
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import com.google.auto.service.AutoService;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@AutoService(DataSourceProcessor.class)
+public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
+ return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class);
+ }
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ AthenaConnectionParam
+ connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson);
+
+ AthenaDataSourceParamDTO
+ athenaDatasourceParamDTO = new AthenaDataSourceParamDTO();
+ athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion());
+ athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ athenaDatasourceParamDTO.setUserName(connectionParams.getUser());
+ athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther()));
Review Comment:
Can we add a new constructor for the class AthenaDataSourceParamDTO? we pass AthenaConnectionParam to it and it see attribute inside the constructor?
I find out we all data source using the `setAttr` to set attribute, maybe we should discuss it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1207351644
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [62.0% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] codecov-commenter commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
codecov-commenter commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1195759409
# [Codecov](https://codecov.io/gh/apache/dolphinscheduler/pull/11157?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) Report
> Merging [#11157](https://codecov.io/gh/apache/dolphinscheduler/pull/11157?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) (ef44aa8) into [dev](https://codecov.io/gh/apache/dolphinscheduler/commit/764a24384fe8be566407d2816d2f965105b578d6?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) (764a243) will **decrease** coverage by `0.10%`.
> The diff coverage is `60.00%`.
```diff
@@ Coverage Diff @@
## dev #11157 +/- ##
============================================
- Coverage 40.39% 40.28% -0.11%
- Complexity 4877 4940 +63
============================================
Files 950 988 +38
Lines 37186 37590 +404
Branches 4079 4128 +49
============================================
+ Hits 15021 15145 +124
- Misses 20642 20909 +267
- Partials 1523 1536 +13
```
| [Impacted Files](https://codecov.io/gh/apache/dolphinscheduler/pull/11157?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) | Coverage Δ | |
|---|---|---|
| [...ce/api/datasource/AbstractDataSourceProcessor.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXBpL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2FwaS9kYXRhc291cmNlL0Fic3RyYWN0RGF0YVNvdXJjZVByb2Nlc3Nvci5qYXZh) | `17.39% <0.00%> (ø)` | |
| [...gin/datasource/athena/AthenaDataSourceChannel.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXRoZW5hL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2F0aGVuYS9BdGhlbmFEYXRhU291cmNlQ2hhbm5lbC5qYXZh) | `0.00% <0.00%> (ø)` | |
| [...asource/athena/AthenaDataSourceChannelFactory.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXRoZW5hL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2F0aGVuYS9BdGhlbmFEYXRhU291cmNlQ2hhbm5lbEZhY3RvcnkuamF2YQ==) | `0.00% <0.00%> (ø)` | |
| [...ugin/datasource/athena/AthenaDataSourceClient.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXRoZW5hL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2F0aGVuYS9BdGhlbmFEYXRhU291cmNlQ2xpZW50LmphdmE=) | `0.00% <0.00%> (ø)` | |
| [.../org/apache/dolphinscheduler/spi/enums/DbType.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1zcGkvc3JjL21haW4vamF2YS9vcmcvYXBhY2hlL2RvbHBoaW5zY2hlZHVsZXIvc3BpL2VudW1zL0RiVHlwZS5qYXZh) | `0.00% <0.00%> (ø)` | |
| [...g/apache/dolphinscheduler/spi/utils/Constants.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1zcGkvc3JjL21haW4vamF2YS9vcmcvYXBhY2hlL2RvbHBoaW5zY2hlZHVsZXIvc3BpL3V0aWxzL0NvbnN0YW50cy5qYXZh) | `0.00% <ø> (ø)` | |
| [...asource/athena/param/AthenaDataSourceParamDTO.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXRoZW5hL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2F0aGVuYS9wYXJhbS9BdGhlbmFEYXRhU291cmNlUGFyYW1EVE8uamF2YQ==) | `66.66% <66.66%> (ø)` | |
| [...source/athena/param/AthenaDataSourceProcessor.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXRoZW5hL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2F0aGVuYS9wYXJhbS9BdGhlbmFEYXRhU291cmNlUHJvY2Vzc29yLmphdmE=) | `69.38% <69.38%> (ø)` | |
| [...datasource/athena/param/AthenaConnectionParam.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1kYXRhc291cmNlLXBsdWdpbi9kb2xwaGluc2NoZWR1bGVyLWRhdGFzb3VyY2UtYXRoZW5hL3NyYy9tYWluL2phdmEvb3JnL2FwYWNoZS9kb2xwaGluc2NoZWR1bGVyL3BsdWdpbi9kYXRhc291cmNlL2F0aGVuYS9wYXJhbS9BdGhlbmFDb25uZWN0aW9uUGFyYW0uamF2YQ==) | `80.00% <80.00%> (ø)` | |
| [...erver/master/processor/queue/TaskEventService.java](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation#diff-ZG9scGhpbnNjaGVkdWxlci1tYXN0ZXIvc3JjL21haW4vamF2YS9vcmcvYXBhY2hlL2RvbHBoaW5zY2hlZHVsZXIvc2VydmVyL21hc3Rlci9wcm9jZXNzb3IvcXVldWUvVGFza0V2ZW50U2VydmljZS5qYXZh) | `69.64% <0.00%> (-10.72%)` | :arrow_down: |
| ... and [51 more](https://codecov.io/gh/apache/dolphinscheduler/pull/11157/diff?src=pr&el=tree-more&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation) | |
Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=The+Apache+Software+Foundation).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1207348936
> Hi @guodongym , Thx for contributing this great feature! It seems there is no related docs for this PR, could you please add the docs? Thx again.
The document has been added
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1208229596
SonarCloud Quality Gate failed. [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [21 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [9 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [37 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [2101 Code Smells](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [37.9% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [2.4% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1212831530
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [62.0% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1229419254
@songjianet @Amy0104 @devosend do you have time to take a look the FE part?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r970288115
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java:
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1203434575
> Hi @guodongym , Thx for contributing this great feature! It seems there is no related docs for this PR, could you please add the docs? Thx again.
Yes I can
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1230447930
Restart the failed test
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1246702930
SonarCloud Quality Gate failed. [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [5 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [29 Code Smells](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [44.9% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1195764391
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [61.5% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym closed pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym closed pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
URL: https://github.com/apache/dolphinscheduler/pull/11157
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r971908937
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import com.google.auto.service.AutoService;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@AutoService(DataSourceProcessor.class)
+public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
+ return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class);
+ }
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ AthenaConnectionParam
+ connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson);
+
+ AthenaDataSourceParamDTO
+ athenaDatasourceParamDTO = new AthenaDataSourceParamDTO();
+ athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion());
+ athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ athenaDatasourceParamDTO.setUserName(connectionParams.getUser());
+ athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther()));
+
+ return athenaDatasourceParamDTO;
+ }
+
+ @Override
+ public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
+ AthenaDataSourceParamDTO athenaParam = (AthenaDataSourceParamDTO) datasourceParam;
+ String address = String.format("%s%s=%s;", Constants.JDBC_ATHENA, "AwsRegion", athenaParam.getAwsRegion());
+
+ AthenaConnectionParam
+ athenaConnectionParam = new AthenaConnectionParam();
+ athenaConnectionParam.setUser(athenaParam.getUserName());
+ athenaConnectionParam.setPassword(PasswordUtils.encodePassword(athenaParam.getPassword()));
+ athenaConnectionParam.setAwsRegion(athenaParam.getAwsRegion());
+ athenaConnectionParam.setOther(this.transformOther(athenaParam.getOther()));
+ athenaConnectionParam.setAddress(address);
+ athenaConnectionParam.setJdbcUrl(address);
+ athenaConnectionParam.setDatabase(athenaParam.getDatabase());
+ athenaConnectionParam.setDriverClassName(this.getDatasourceDriver());
+ athenaConnectionParam.setValidationQuery(this.getValidationQuery());
+ athenaConnectionParam.setProps(athenaParam.getOther());
+
+ return athenaConnectionParam;
+ }
+
+ @Override
+ public ConnectionParam createConnectionParams(String connectionJson) {
+ return JSONUtils.parseObject(connectionJson, AthenaConnectionParam.class);
+ }
+
+ @Override
+ public String getDatasourceDriver() {
+ return Constants.COM_ATHENA_JDBC_DRIVER;
+ }
+
+ @Override
+ public String getValidationQuery() {
+ return Constants.ATHENA_VALIDATION_QUERY;
+ }
+
+ @Override
+ public String getJdbcUrl(ConnectionParam connectionParam) {
+ AthenaConnectionParam
+ athenaConnectionParam = (AthenaConnectionParam) connectionParam;
+ if (!StringUtils.isEmpty(athenaConnectionParam.getOther())) {
+ return String.format("%s%s", athenaConnectionParam.getJdbcUrl(), athenaConnectionParam.getOther());
+ }
+ return athenaConnectionParam.getJdbcUrl();
+ }
+
+ @Override
+ public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
+ AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam;
+ Class.forName(this.getDatasourceDriver());
+ return DriverManager.getConnection(this.getJdbcUrl(connectionParam),
+ athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword()));
+ }
+
+ @Override
+ public DbType getDbType() {
+ return DbType.ATHENA;
+ }
+
+ @Override
+ public DataSourceProcessor create() {
+ return new AthenaDataSourceProcessor();
+ }
+
+ private String transformOther(Map<String, String> otherMap) {
+ if (MapUtils.isNotEmpty(otherMap)) {
+ List<String> list = new ArrayList<>(otherMap.size());
+ otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
+ return String.join(Constants.SEMICOLON, list);
+ }
+ return null;
+ }
+
+ private Map<String, String> parseOther(String other) {
+ Map<String, String> otherMap = new LinkedHashMap<>();
+ if (StringUtils.isEmpty(other)) {
+ return otherMap;
+ }
+ String[] configs = other.split(Constants.SEMICOLON);
+ for (String config : configs) {
+ otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]);
+ }
+ return otherMap;
+ }
+
+ @Override
+ protected void checkHost(String host) {
+ // Do not need to set the host, nothing to do
+ }
+
+ @Override
+ protected void checkDatabasePatter(String database) {
+ // Do not need to set the database, nothing to do
+ }
Review Comment:
👌
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1246699804
SonarCloud Quality Gate failed. [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [5 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [29 Code Smells](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [45.0% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] caishunfeng commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
caishunfeng commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1211511726
> SonarCloud Quality Gate failed. [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
>
> [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [21 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [9 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [37 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [2101 Code Smells](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
>
> [](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [37.9% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density
&view=list) [2.4% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
I found the sonar result is wrong, seems checked all the code, how can we retry this check again?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r940276809
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import com.google.auto.service.AutoService;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@AutoService(DataSourceProcessor.class)
+public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
+ return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class);
+ }
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ AthenaConnectionParam
+ connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson);
+
+ AthenaDataSourceParamDTO
+ athenaDatasourceParamDTO = new AthenaDataSourceParamDTO();
+ athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion());
+ athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ athenaDatasourceParamDTO.setUserName(connectionParams.getUser());
+ athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther()));
Review Comment:
I find out we all data source using the `setAttr` to set attribute, so i implemented in the same way,I'm not sure if it needs to be changed uniformly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r971909996
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/test/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessorTest.java:
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+
+import java.sql.DriverManager;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
Review Comment:
mark resolve
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r971909551
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java:
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
Review Comment:
resolve
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1229971501
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [62.0% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r940261494
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import com.google.auto.service.AutoService;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@AutoService(DataSourceProcessor.class)
+public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
+ return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class);
+ }
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ AthenaConnectionParam
+ connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson);
+
+ AthenaDataSourceParamDTO
+ athenaDatasourceParamDTO = new AthenaDataSourceParamDTO();
+ athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion());
+ athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ athenaDatasourceParamDTO.setUserName(connectionParams.getUser());
+ athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther()));
+
+ return athenaDatasourceParamDTO;
+ }
+
+ @Override
+ public BaseConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
+ AthenaDataSourceParamDTO athenaParam = (AthenaDataSourceParamDTO) datasourceParam;
+ String address = String.format("%s%s=%s;", Constants.JDBC_ATHENA, "AwsRegion", athenaParam.getAwsRegion());
+
+ AthenaConnectionParam
+ athenaConnectionParam = new AthenaConnectionParam();
+ athenaConnectionParam.setUser(athenaParam.getUserName());
+ athenaConnectionParam.setPassword(PasswordUtils.encodePassword(athenaParam.getPassword()));
+ athenaConnectionParam.setAwsRegion(athenaParam.getAwsRegion());
+ athenaConnectionParam.setOther(this.transformOther(athenaParam.getOther()));
+ athenaConnectionParam.setAddress(address);
+ athenaConnectionParam.setJdbcUrl(address);
+ athenaConnectionParam.setDatabase(athenaParam.getDatabase());
+ athenaConnectionParam.setDriverClassName(this.getDatasourceDriver());
+ athenaConnectionParam.setValidationQuery(this.getValidationQuery());
+ athenaConnectionParam.setProps(athenaParam.getOther());
+
+ return athenaConnectionParam;
+ }
+
+ @Override
+ public ConnectionParam createConnectionParams(String connectionJson) {
+ return JSONUtils.parseObject(connectionJson, AthenaConnectionParam.class);
+ }
+
+ @Override
+ public String getDatasourceDriver() {
+ return Constants.COM_ATHENA_JDBC_DRIVER;
+ }
+
+ @Override
+ public String getValidationQuery() {
+ return Constants.ATHENA_VALIDATION_QUERY;
+ }
+
+ @Override
+ public String getJdbcUrl(ConnectionParam connectionParam) {
+ AthenaConnectionParam
+ athenaConnectionParam = (AthenaConnectionParam) connectionParam;
+ if (!StringUtils.isEmpty(athenaConnectionParam.getOther())) {
+ return String.format("%s%s", athenaConnectionParam.getJdbcUrl(), athenaConnectionParam.getOther());
+ }
+ return athenaConnectionParam.getJdbcUrl();
+ }
+
+ @Override
+ public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException {
+ AthenaConnectionParam athenaConnectionParam = (AthenaConnectionParam) connectionParam;
+ Class.forName(this.getDatasourceDriver());
+ return DriverManager.getConnection(this.getJdbcUrl(connectionParam),
+ athenaConnectionParam.getUser(), PasswordUtils.decodePassword(athenaConnectionParam.getPassword()));
+ }
+
+ @Override
+ public DbType getDbType() {
+ return DbType.ATHENA;
+ }
+
+ @Override
+ public DataSourceProcessor create() {
+ return new AthenaDataSourceProcessor();
+ }
+
+ private String transformOther(Map<String, String> otherMap) {
+ if (MapUtils.isNotEmpty(otherMap)) {
+ List<String> list = new ArrayList<>(otherMap.size());
+ otherMap.forEach((key, value) -> list.add(String.format("%s=%s", key, value)));
+ return String.join(Constants.SEMICOLON, list);
+ }
+ return null;
+ }
+
+ private Map<String, String> parseOther(String other) {
+ Map<String, String> otherMap = new LinkedHashMap<>();
+ if (StringUtils.isEmpty(other)) {
+ return otherMap;
+ }
+ String[] configs = other.split(Constants.SEMICOLON);
+ for (String config : configs) {
+ otherMap.put(config.split(Constants.EQUAL_SIGN)[0], config.split(Constants.EQUAL_SIGN)[1]);
+ }
+ return otherMap;
+ }
+
+ @Override
+ protected void checkHost(String host) {
+ // Do not need to set the host, nothing to do
+ }
+
+ @Override
+ protected void checkDatabasePatter(String database) {
+ // Do not need to set the database, nothing to do
+ }
Review Comment:
if remove those method, will cause the parent class parameter check error, `org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor#checkDatasourceParam` , I find POSTGRESQL are also problems with database verification
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1212833647
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [62.0% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] kezhenxu94 merged pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
kezhenxu94 merged PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1195764494
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [61.5% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1208229609
Please retry analysis of this Pull-Request directly on [SonarCloud](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1229908695
I add missing doc side bar in 8def3b2
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1229909023
And will merge PR when it pass
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1212834813
> > SonarCloud Quality Gate failed. [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
> > [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [21 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [9 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [37 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [2101 Code Smells](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
> > [](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [37.9% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_densi
ty&view=list) [2.4% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
>
> I found the sonar result is wrong, seems checked all the code, how can we retry this check again?
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] guodongym commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
guodongym commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r970288647
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/test/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessorTest.java:
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+
+import java.sql.DriverManager;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
Review Comment:
waiting for you to refactor later
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] EricGao888 commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
EricGao888 commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r965526952
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/test/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessorTest.java:
##########
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+
+import java.sql.DriverManager;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.powermock.api.mockito.PowerMockito;
+import org.powermock.core.classloader.annotations.PrepareForTest;
+import org.powermock.modules.junit4.PowerMockRunner;
+
+@RunWith(PowerMockRunner.class)
Review Comment:
Better avoid using `powermock`, see: https://github.com/apache/dolphinscheduler/issues/11405 . But it's fine to me if you don't want to rewrite it at this moment. I could refactor this later in issue #11405
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceParamDTO.java:
##########
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+
Review Comment:
We could use `lombok` `@Data` here to help the data class look neat.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] sonarcloud[bot] commented on pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
sonarcloud[bot] commented on PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#issuecomment-1229973317
Kudos, SonarCloud Quality Gate passed! [](https://sonarcloud.io/dashboard?id=apache-dolphinscheduler&pullRequest=11157)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG) [0 Bugs](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=BUG)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY) [0 Vulnerabilities](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=VULNERABILITY)
[](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT) [0 Security Hotspots](https://sonarcloud.io/project/security_hotspots?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=SECURITY_HOTSPOT)
[](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL) [1 Code Smell](https://sonarcloud.io/project/issues?id=apache-dolphinscheduler&pullRequest=11157&resolved=false&types=CODE_SMELL)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list) [62.0% Coverage](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_coverage&view=list)
[](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list) [3.2% Duplication](https://sonarcloud.io/component_measures?id=apache-dolphinscheduler&pullRequest=11157&metric=new_duplicated_lines_density&view=list)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [dolphinscheduler] zhongjiajie commented on a diff in pull request #11157: [Feature-10273][Datasource] Add dataSource for AWS Athena
Posted by GitBox <gi...@apache.org>.
zhongjiajie commented on code in PR #11157:
URL: https://github.com/apache/dolphinscheduler/pull/11157#discussion_r971908425
##########
dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-athena/src/main/java/org/apache/dolphinscheduler/plugin/datasource/athena/param/AthenaDataSourceProcessor.java:
##########
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.datasource.athena.param;
+
+import com.google.auto.service.AutoService;
+import org.apache.commons.collections4.MapUtils;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
+import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
+import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
+import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
+import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
+import org.apache.dolphinscheduler.spi.enums.DbType;
+import org.apache.dolphinscheduler.spi.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.JSONUtils;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+@AutoService(DataSourceProcessor.class)
+public class AthenaDataSourceProcessor extends AbstractDataSourceProcessor {
+
+ @Override
+ public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
+ return JSONUtils.parseObject(paramJson, AthenaDataSourceParamDTO.class);
+ }
+
+ @Override
+ public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
+ AthenaConnectionParam
+ connectionParams = (AthenaConnectionParam) this.createConnectionParams(connectionJson);
+
+ AthenaDataSourceParamDTO
+ athenaDatasourceParamDTO = new AthenaDataSourceParamDTO();
+ athenaDatasourceParamDTO.setAwsRegion(connectionParams.getAwsRegion());
+ athenaDatasourceParamDTO.setDatabase(connectionParams.getDatabase());
+ athenaDatasourceParamDTO.setUserName(connectionParams.getUser());
+ athenaDatasourceParamDTO.setOther(this.parseOther(connectionParams.getOther()));
Review Comment:
get it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@dolphinscheduler.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org