You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by pe...@apache.org on 2017/11/22 02:43:26 UTC
ranger git commit: RANGER-1848:Implement getLinkList/getJobList in
SqoopClient for Ranger Sqoop2 plugin
Repository: ranger
Updated Branches:
refs/heads/master 41e220381 -> ad2731811
RANGER-1848:Implement getLinkList/getJobList in SqoopClient for Ranger Sqoop2 plugin
Signed-off-by: peng.jianhua <pe...@zte.com.cn>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/ad273181
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/ad273181
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/ad273181
Branch: refs/heads/master
Commit: ad2731811d56a2fa357cf06bea5e5c2f1fc1b98d
Parents: 41e2203
Author: zhangqiang2 <zh...@zte.com.cn>
Authored: Tue Nov 21 11:01:02 2017 +0800
Committer: peng.jianhua <pe...@zte.com.cn>
Committed: Wed Nov 22 10:34:11 2017 +0800
----------------------------------------------------------------------
.../services/sqoop/client/SqoopClient.java | 139 +++++++++++++++----
.../client/json/model/SqoopJobResponse.java | 104 ++++++++++++++
.../client/json/model/SqoopJobsResponse.java | 37 +++++
.../client/json/model/SqoopLinkResponse.java | 70 ++++++++++
.../client/json/model/SqoopLinksResponse.java | 37 +++++
5 files changed, 358 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java
----------------------------------------------------------------------
diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java
index 640d5db..3563b31 100644
--- a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java
+++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java
@@ -21,6 +21,7 @@ package org.apache.ranger.services.sqoop.client;
import java.security.PrivilegedAction;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -38,6 +39,10 @@ import org.apache.ranger.plugin.client.BaseClient;
import org.apache.ranger.plugin.client.HadoopException;
import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorResponse;
import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorsResponse;
+import org.apache.ranger.services.sqoop.client.json.model.SqoopJobResponse;
+import org.apache.ranger.services.sqoop.client.json.model.SqoopJobsResponse;
+import org.apache.ranger.services.sqoop.client.json.model.SqoopLinkResponse;
+import org.apache.ranger.services.sqoop.client.json.model.SqoopLinksResponse;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -53,6 +58,10 @@ public class SqoopClient extends BaseClient {
private static final String SQOOP_CONNECTOR_API_ENDPOINT = "/sqoop/v1/connector/all";
+ private static final String SQOOP_LINK_API_ENDPOINT = "/sqoop/v1/link/all";
+
+ private static final String SQOOP_JOB_API_ENDPOINT = "/sqoop/v1/job/all";
+
private static final String ERROR_MESSAGE = " You can still save the repository and start creating "
+ "policies, but you would not be able to use autocomplete for "
+ "resource names. Check ranger_admin.log for more info.";
@@ -80,12 +89,12 @@ public class SqoopClient extends BaseClient {
public List<String> getConnectorList(final String connectorMatching, final List<String> existingConnectors) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Getting sqoop connector list for connectorMatching: " + connectorMatching
- + ", existingConnectors: " + existingConnectors);
+ LOG.debug("Get sqoop connector list for connectorMatching: " + connectorMatching + ", existingConnectors: "
+ + existingConnectors);
}
Subject subj = getLoginSubject();
if (subj == null) {
- return null;
+ return Collections.emptyList();
}
List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() {
@@ -95,33 +104,106 @@ public class SqoopClient extends BaseClient {
ClientResponse response = getClientResponse(sqoopUrl, SQOOP_CONNECTOR_API_ENDPOINT, userName);
- SqoopConnectorsResponse sqoopConnectorsResponse = getSqoopConnectorResponse(response);
- if (sqoopConnectorsResponse == null) {
- return null;
+ SqoopConnectorsResponse sqoopConnectorsResponse = getSqoopResourceResponse(response,
+ SqoopConnectorsResponse.class);
+ if (sqoopConnectorsResponse == null || CollectionUtils.isEmpty(sqoopConnectorsResponse.getConnectors())) {
+ return Collections.emptyList();
+ }
+ List<String> connectorResponses = new ArrayList<>();
+ for (SqoopConnectorResponse sqoopConnectorResponse : sqoopConnectorsResponse.getConnectors()) {
+ connectorResponses.add(sqoopConnectorResponse.getName());
}
- List<SqoopConnectorResponse> connectorResponses = sqoopConnectorsResponse.getConnectors();
+
List<String> connectors = null;
if (CollectionUtils.isNotEmpty(connectorResponses)) {
- connectors = getConnectorFromResponse(connectorMatching, existingConnectors, connectorResponses);
+ connectors = filterResourceFromResponse(connectorMatching, existingConnectors, connectorResponses);
}
return connectors;
}
});
if (LOG.isDebugEnabled()) {
- LOG.debug("Getting sqoop project connector result: " + ret);
+ LOG.debug("Get sqoop connector list result: " + ret);
}
return ret;
}
public List<String> getLinkList(final String linkMatching, final List<String> existingLinks) {
- // TODO
- return null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Get sqoop link list for linkMatching: " + linkMatching + ", existingLinks: " + existingLinks);
+ }
+ Subject subj = getLoginSubject();
+ if (subj == null) {
+ return Collections.emptyList();
+ }
+
+ List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() {
+
+ @Override
+ public List<String> run() {
+
+ ClientResponse response = getClientResponse(sqoopUrl, SQOOP_LINK_API_ENDPOINT, userName);
+
+ SqoopLinksResponse sqoopLinksResponse = getSqoopResourceResponse(response, SqoopLinksResponse.class);
+ if (sqoopLinksResponse == null || CollectionUtils.isEmpty(sqoopLinksResponse.getLinks())) {
+ return Collections.emptyList();
+ }
+ List<String> linkResponses = new ArrayList<>();
+ for (SqoopLinkResponse sqoopLinkResponse : sqoopLinksResponse.getLinks()) {
+ linkResponses.add(sqoopLinkResponse.getName());
+ }
+
+ List<String> links = null;
+ if (CollectionUtils.isNotEmpty(linkResponses)) {
+ links = filterResourceFromResponse(linkMatching, existingLinks, linkResponses);
+ }
+ return links;
+ }
+ });
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Get sqoop link list result: " + ret);
+ }
+ return ret;
}
public List<String> getJobList(final String jobMatching, final List<String> existingJobs) {
- // TODO
- return null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Get sqoop job list for jobMatching: " + jobMatching + ", existingJobs: " + existingJobs);
+ }
+ Subject subj = getLoginSubject();
+ if (subj == null) {
+ return Collections.emptyList();
+ }
+
+ List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() {
+
+ @Override
+ public List<String> run() {
+
+ ClientResponse response = getClientResponse(sqoopUrl, SQOOP_JOB_API_ENDPOINT, userName);
+
+ SqoopJobsResponse sqoopJobsResponse = getSqoopResourceResponse(response, SqoopJobsResponse.class);
+ if (sqoopJobsResponse == null || CollectionUtils.isEmpty(sqoopJobsResponse.getJobs())) {
+ return Collections.emptyList();
+ }
+ List<String> jobResponses = new ArrayList<>();
+ for (SqoopJobResponse sqoopJobResponse : sqoopJobsResponse.getJobs()) {
+ jobResponses.add(sqoopJobResponse.getName());
+ }
+
+ List<String> jobs = null;
+ if (CollectionUtils.isNotEmpty(jobResponses)) {
+ jobs = filterResourceFromResponse(jobMatching, existingJobs, jobResponses);
+ }
+ return jobs;
+ }
+ });
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Get sqoop job list result: " + ret);
+ }
+ return ret;
}
private static ClientResponse getClientResponse(String sqoopUrl, String sqoopApi, String userName) {
@@ -182,14 +264,14 @@ public class SqoopClient extends BaseClient {
return response;
}
- private SqoopConnectorsResponse getSqoopConnectorResponse(ClientResponse response) {
- SqoopConnectorsResponse sqoopConnectorsResponse = null;
+ private <T> T getSqoopResourceResponse(ClientResponse response, Class<T> classOfT) {
+ T resource = null;
try {
if (response != null && response.getStatus() == HttpStatus.SC_OK) {
String jsonString = response.getEntity(String.class);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
- sqoopConnectorsResponse = gson.fromJson(jsonString, SqoopConnectorsResponse.class);
+ resource = gson.fromJson(jsonString, classOfT);
} else {
String msgDesc = "Unable to get a valid response for " + "expected mime type : [" + EXPECTED_MIME_TYPE
+ "], sqoopUrl: " + sqoopUrl + " - got null response.";
@@ -201,7 +283,7 @@ public class SqoopClient extends BaseClient {
} catch (HadoopException he) {
throw he;
} catch (Throwable t) {
- String msgDesc = "Exception while getting sqoop project response, sqoopUrl: " + sqoopUrl;
+ String msgDesc = "Exception while getting sqoop resource response, sqoopUrl: " + sqoopUrl;
HadoopException hdpException = new HadoopException(msgDesc, t);
LOG.error(msgDesc, t);
@@ -214,26 +296,25 @@ public class SqoopClient extends BaseClient {
response.close();
}
}
- return sqoopConnectorsResponse;
+ return resource;
}
- private static List<String> getConnectorFromResponse(String connectorMatching, List<String> existingConnectors,
- List<SqoopConnectorResponse> connectorResponses) {
- List<String> connectors = new ArrayList<String>();
- for (SqoopConnectorResponse connectorResponse : connectorResponses) {
- String connectorName = connectorResponse.getName();
- if (CollectionUtils.isNotEmpty(existingConnectors) && existingConnectors.contains(connectorName)) {
+ private static List<String> filterResourceFromResponse(String resourceMatching, List<String> existingResources,
+ List<String> resourceResponses) {
+ List<String> resources = new ArrayList<String>();
+ for (String resourceResponse : resourceResponses) {
+ if (CollectionUtils.isNotEmpty(existingResources) && existingResources.contains(resourceResponse)) {
continue;
}
- if (StringUtils.isEmpty(connectorMatching) || connectorMatching.startsWith("*")
- || connectorName.toLowerCase().startsWith(connectorMatching.toLowerCase())) {
+ if (StringUtils.isEmpty(resourceMatching) || resourceMatching.startsWith("*")
+ || resourceResponse.toLowerCase().startsWith(resourceMatching.toLowerCase())) {
if (LOG.isDebugEnabled()) {
- LOG.debug("getConnectorFromResponse(): Adding sqoop connector " + connectorName);
+ LOG.debug("filterResourceFromResponse(): Adding sqoop resource " + resourceResponse);
}
- connectors.add(connectorName);
+ resources.add(resourceResponse);
}
}
- return connectors;
+ return resources;
}
public static Map<String, Object> connectionTest(String serviceName, Map<String, String> configs) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java
----------------------------------------------------------------------
diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java
new file mode 100644
index 0000000..b78ca16
--- /dev/null
+++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobResponse.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.services.sqoop.client.json.model;
+
+import com.google.gson.annotations.SerializedName;
+
+public class SqoopJobResponse {
+ private Long id;
+
+ private String name;
+
+ @SerializedName("from-link-name")
+ private String fromLinkName;
+
+ @SerializedName("to-link-name")
+ private String toLinkName;
+
+ @SerializedName("from-connector-name")
+ private String fromConnectorName;
+
+ @SerializedName("to-connector-name")
+ private String toConnectorName;
+
+ @SerializedName("creation-user")
+ private String creationUser;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getFromLinkName() {
+ return fromLinkName;
+ }
+
+ public void setFromLinkName(String fromLinkName) {
+ this.fromLinkName = fromLinkName;
+ }
+
+ public String getToLinkName() {
+ return toLinkName;
+ }
+
+ public void setToLinkName(String toLinkName) {
+ this.toLinkName = toLinkName;
+ }
+
+ public String getFromConnectorName() {
+ return fromConnectorName;
+ }
+
+ public void setFromConnectorName(String fromConnectorName) {
+ this.fromConnectorName = fromConnectorName;
+ }
+
+ public String getToConnectorName() {
+ return toConnectorName;
+ }
+
+ public void setToConnectorName(String toConnectorName) {
+ this.toConnectorName = toConnectorName;
+ }
+
+ public String getCreationUser() {
+ return creationUser;
+ }
+
+ public void setCreationUser(String creationUser) {
+ this.creationUser = creationUser;
+ }
+
+ @Override
+ public String toString() {
+ return "SqoopJobResponse [id=" + id + ", name=" + name + ", fromLinkName=" + fromLinkName + ", toLinkName="
+ + toLinkName + ", fromConnectorName=" + fromConnectorName + ", toConnectorName=" + toConnectorName
+ + ", creationUser=" + creationUser + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java
----------------------------------------------------------------------
diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java
new file mode 100644
index 0000000..a2de043
--- /dev/null
+++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopJobsResponse.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.services.sqoop.client.json.model;
+
+import java.util.List;
+
+public class SqoopJobsResponse {
+ private List<SqoopJobResponse> jobs;
+
+ public List<SqoopJobResponse> getJobs() {
+ return jobs;
+ }
+
+ public void setJobs(List<SqoopJobResponse> jobs) {
+ this.jobs = jobs;
+ }
+
+ @Override
+ public String toString() {
+ return "SqoopJobsResponse [jobs=" + jobs + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java
----------------------------------------------------------------------
diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java
new file mode 100644
index 0000000..0866a3e
--- /dev/null
+++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinkResponse.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.services.sqoop.client.json.model;
+
+import com.google.gson.annotations.SerializedName;
+
+public class SqoopLinkResponse {
+ private Long id;
+
+ private String name;
+
+ @SerializedName("connector-name")
+ private String connectorName;
+
+ @SerializedName("creation-user")
+ private String creationUser;
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getConnectorName() {
+ return connectorName;
+ }
+
+ public void setConnectorName(String connectorName) {
+ this.connectorName = connectorName;
+ }
+
+ public String getCreationUser() {
+ return creationUser;
+ }
+
+ public void setCreationUser(String creationUser) {
+ this.creationUser = creationUser;
+ }
+
+ @Override
+ public String toString() {
+ return "SqoopLinkResponse [id=" + id + ", name=" + name + ", connectorName=" + connectorName
+ + ", creationUser=" + creationUser + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/ranger/blob/ad273181/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java
----------------------------------------------------------------------
diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java
new file mode 100644
index 0000000..9fda8c2
--- /dev/null
+++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopLinksResponse.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ranger.services.sqoop.client.json.model;
+
+import java.util.List;
+
+public class SqoopLinksResponse {
+ private List<SqoopLinkResponse> links;
+
+ public List<SqoopLinkResponse> getLinks() {
+ return links;
+ }
+
+ public void setLinks(List<SqoopLinkResponse> links) {
+ this.links = links;
+ }
+
+ @Override
+ public String toString() {
+ return "SqoopLinksResponse [links=" + links + "]";
+ }
+}