You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by bo...@apache.org on 2015/04/03 00:30:23 UTC
[1/2] incubator-ranger git commit: RANGER-246 - Ranger Admin side
support Kafka integration
Repository: incubator-ranger
Updated Branches:
refs/heads/master 0d9a31a93 -> 399ed34b2
RANGER-246 - Ranger Admin side support Kafka integration
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/10b57288
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/10b57288
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/10b57288
Branch: refs/heads/master
Commit: 10b57288904da5e832300850e799f8debb4cc909
Parents: be34cc2
Author: Don Bosco Durai <bo...@apache.org>
Authored: Thu Apr 2 12:18:33 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Thu Apr 2 12:18:33 2015 -0700
----------------------------------------------------------------------
.../service-defs/ranger-servicedef-kafka.json | 112 +++++++++
.../service-defs/ranger-servicedef-solr.json | 13 +-
plugin-kafka/.gitignore | 1 +
.../conf/ranger-kafka-audit-changes.cfg | 36 +++
plugin-kafka/conf/ranger-kafka-audit.xml | 235 +++++++++++++++++++
.../conf/ranger-kafka-security-changes.cfg | 26 ++
plugin-kafka/conf/ranger-kafka-security.xml | 67 ++++++
.../conf/ranger-policymgr-ssl-changes.cfg | 23 ++
plugin-kafka/conf/ranger-policymgr-ssl.xml | 63 +++++
plugin-kafka/pom.xml | 56 +++++
plugin-kafka/scripts/install.properties | 112 +++++++++
.../scripts/kafka-plugin-install.properties | 23 ++
.../kafka/authorizer/RangerKafkaAuthorizer.java | 33 +++
.../services/kafka/RangerServiceKafka.java | 78 ++++++
.../kafka/client/ServiceKafkaClient.java | 199 ++++++++++++++++
.../kafka/client/ServiceKafkaConnectionMgr.java | 55 +++++
.../services/solr/client/ServiceSolrClient.java | 2 +-
pom.xml | 2 +
src/main/assembly/plugin-kafka.xml | 157 +++++++++++++
19 files changed, 1285 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
----------------------------------------------------------------------
diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
new file mode 100644
index 0000000..11b5d15
--- /dev/null
+++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-kafka.json
@@ -0,0 +1,112 @@
+{
+ "name":"kafka",
+ "implClass":"org.apache.ranger.services.kafka.RangerServiceKafka",
+ "label":"Kafka",
+ "description":"Apache Kafka",
+ "resources":[
+ {
+ "id":1,
+ "name":"topic",
+ "type":"string",
+ "level":1,
+ "mandatory":true,
+ "lookupSupported":true,
+ "recursiveSupported":false,
+ "excludesSupported":true,
+ "matcher":"org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher",
+ "matcherOptions":{
+ "wildCard":true,
+ "ignoreCase":true
+ },
+ "validationRegEx":"",
+ "validationMessage":"",
+ "uiHint":"",
+ "label":"Topic",
+ "description":"Topic"
+ }
+
+ ],
+ "accessTypes":[
+ {
+ "name":"publish",
+ "label":"Publish"
+ },
+ {
+ "name":"consume",
+ "label":"Consume"
+ },
+ {
+ "name":"create",
+ "label":"Create"
+ },
+ {
+ "name":"delete",
+ "label":"Delete"
+ },
+ {
+ "name":"configure",
+ "label":"Configure"
+ },
+ {
+ "name":"describe",
+ "label":"Describe"
+ },
+ {
+ "name":"replicate",
+ "label":"Replicate"
+ },
+ {
+ "name":"connect",
+ "label":"Connect"
+ }
+
+ ],
+ "configs":[
+ {
+ "name":"username",
+ "type":"string",
+ "mandatory":true,
+ "label":"Username"
+ },
+ {
+ "name":"password",
+ "type":"password",
+ "mandatory":true,
+ "label":"Password"
+ },
+ {
+ "name":"zookeeper.connect",
+ "type":"string",
+ "mandatory":true,
+ "defaultValue":"localhost:2181",
+ "label":"Zookeeper Connect String"
+ },
+ {
+ "name":"certificate.cn",
+ "type":"string",
+ "mandatory":false,
+ "label":"Common Name for Certificate"
+ }
+
+ ],
+ "enums":[
+
+ ],
+ "contextEnrichers":[
+
+ ],
+ "policyConditions":[
+ {
+ "id": 1,
+ "name": "ip-range",
+ "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher",
+ "evaluatorOptions": { },
+ "validationRegEx":"",
+ "validationMessage": "",
+ "uiHint":"",
+ "label": "IP Address Range",
+ "description": "IP Address Range"
+ }
+
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
----------------------------------------------------------------------
diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
index 19e9b63..793ec28 100644
--- a/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
+++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-solr.json
@@ -118,15 +118,14 @@
"policyConditions":[
{
"id": 1,
- "name":"ip-range",
- "evaluator":"org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher",
- "evaluatorOptions":"",
+ "name": "ip-range",
+ "evaluator": "org.apache.ranger.plugin.conditionevaluator.RangerIpMatcher",
+ "evaluatorOptions": { },
"validationRegEx":"",
- "validationMessage":"",
+ "validationMessage": "",
"uiHint":"",
- "label":"IP Address Range",
- "description":"IP Address Range"
+ "label": "IP Address Range",
+ "description": "IP Address Range"
}
-
]
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/.gitignore
----------------------------------------------------------------------
diff --git a/plugin-kafka/.gitignore b/plugin-kafka/.gitignore
new file mode 100644
index 0000000..ea8c4bf
--- /dev/null
+++ b/plugin-kafka/.gitignore
@@ -0,0 +1 @@
+/target
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/ranger-kafka-audit-changes.cfg b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
new file mode 100644
index 0000000..7c0c430
--- /dev/null
+++ b/plugin-kafka/conf/ranger-kafka-audit-changes.cfg
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT_DB_JDBC_URL% mod create-if-not-exists
+xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists
+xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists
+xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
+xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT_DB_JDBC_DRIVER% mod create-if-not-exists
+
+xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.directory %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.directory %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists
+xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists
+xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-audit.xml
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/ranger-kafka-audit.xml b/plugin-kafka/conf/ranger-kafka-audit.xml
new file mode 100644
index 0000000..bff6cbf
--- /dev/null
+++ b/plugin-kafka/conf/ranger-kafka-audit.xml
@@ -0,0 +1,235 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+ <property>
+ <name>xasecure.audit.is.enabled</name>
+ <value>true</value>
+ </property>
+
+
+ <!-- DB audit provider configuration -->
+ <property>
+ <name>xasecure.audit.db.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.is.async</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.db.batch.size</name>
+ <value>100</value>
+ </property>
+
+ <!-- Properties whose name begin with "xasecure.audit.jpa." are used to configure JPA -->
+ <property>
+ <name>xasecure.audit.jpa.javax.persistence.jdbc.url</name>
+ <value>jdbc:mysql://localhost:3306/ranger_audit</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.jpa.javax.persistence.jdbc.user</name>
+ <value>rangerlogger</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.jpa.javax.persistence.jdbc.password</name>
+ <value>none</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.jpa.javax.persistence.jdbc.driver</name>
+ <value>com.mysql.jdbc.Driver</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.credential.provider.file</name>
+ <value>jceks://file/etc/ranger/kafkadev/auditcred.jceks</value>
+ </property>
+
+
+
+ <!-- HDFS audit provider configuration -->
+ <property>
+ <name>xasecure.audit.hdfs.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.is.async</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.queue.size</name>
+ <value>1048576</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.encoding</name>
+ <value></value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.directory</name>
+ <value>hdfs://NAMENODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.file</name>
+ <value>%hostname%-audit.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+ <value>900</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
+ <value>86400</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.directory</name>
+ <value>/var/log/kafka/audit</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file</name>
+ <value>%time:yyyyMMdd-HHmm.ss%.log</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.file.buffer.size.bytes</name>
+ <value>8192</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+ <value>60</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
+ <value>600</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.directory</name>
+ <value>/var/log/kafka/audit/archive</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name>
+ <value>10</value>
+ </property>
+
+
+ <!-- Log4j audit provider configuration -->
+ <property>
+ <name>xasecure.audit.log4j.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.log4j.is.async</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.log4j.async.max.queue.size</name>
+ <value>10240</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
+ <value>30000</value>
+ </property>
+
+
+ <!-- Kafka audit provider configuration -->
+ <property>
+ <name>xasecure.audit.kafka.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.broker_list</name>
+ <value>localhost:9092</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.kafka.topic_name</name>
+ <value>ranger_audits</value>
+ </property>
+
+ <!-- Ranger audit provider configuration -->
+ <property>
+ <name>xasecure.audit.solr.is.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.async.max.queue.size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.async.max.flush.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <name>xasecure.audit.solr.solr_url</name>
+ <value>http://localhost:6083/solr/ranger_audits</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-security-changes.cfg
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/ranger-kafka-security-changes.cfg b/plugin-kafka/conf/ranger-kafka-security-changes.cfg
new file mode 100644
index 0000000..fe36616
--- /dev/null
+++ b/plugin-kafka/conf/ranger-kafka-security-changes.cfg
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Change the original policy parameter to work with policy manager based.
+#
+#
+ranger.plugin.kafka.service.name %REPOSITORY_NAME% mod create-if-not-exists
+
+ranger.plugin.kafka.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient mod create-if-not-exists
+
+ranger.plugin.kafka.policy.rest.url %POLICY_MGR_URL% mod create-if-not-exists
+ranger.plugin.kafka.policy.rest.ssl.config.file /etc/kafka/conf/ranger-policymgr-ssl.xml mod create-if-not-exists
+ranger.plugin.kafka.policy.pollIntervalMs 30000 mod create-if-not-exists
+ranger.plugin.kafka.policy.cache.dir %POLICY_CACHE_FILE_PATH% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-kafka-security.xml
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/ranger-kafka-security.xml b/plugin-kafka/conf/ranger-kafka-security.xml
new file mode 100644
index 0000000..b11a71a
--- /dev/null
+++ b/plugin-kafka/conf/ranger-kafka-security.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+ <property>
+ <name>ranger.plugin.kafka.service.name</name>
+ <value>kafkadev</value>
+ <description>
+ Name of the Ranger service containing policies for this KAFKA instance
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.kafka.policy.source.impl</name>
+ <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value>
+ <description>
+ Class to retrieve policies from the source
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.kafka.policy.rest.url</name>
+ <value>http://policymanagerhost:port</value>
+ <description>
+ URL to Ranger Admin
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.kafka.policy.rest.ssl.config.file</name>
+ <value>/etc/kafka/conf/ranger-policymgr-ssl.xml</value>
+ <description>
+ Path to the file containing SSL details to contact Ranger Admin
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.kafka.policy.pollIntervalMs</name>
+ <value>30000</value>
+ <description>
+ How often to poll for changes in policies?
+ </description>
+ </property>
+
+ <property>
+ <name>ranger.plugin.kafka.policy.cache.dir</name>
+ <value>/etc/ranger/kafkadev/policycache</value>
+ <description>
+ Directory where Ranger policies are cached after successful retrieval from the source
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg b/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg
new file mode 100644
index 0000000..ec4eeab
--- /dev/null
+++ b/plugin-kafka/conf/ranger-policymgr-ssl-changes.cfg
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SSL Params
+#
+xasecure.policymgr.clientssl.keystore %SSL_KEYSTORE_FILE_PATH% mod create-if-not-exists
+xasecure.policymgr.clientssl.keystore.password %SSL_KEYSTORE_PASSWORD% mod create-if-not-exists
+xasecure.policymgr.clientssl.keystore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
+xasecure.policymgr.clientssl.truststore %SSL_TRUSTSTORE_FILE_PATH% mod create-if-not-exists
+xasecure.policymgr.clientssl.truststore.password %SSL_TRUSTSTORE_PASSWORD% mod create-if-not-exists
+xasecure.policymgr.clientssl.truststore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/conf/ranger-policymgr-ssl.xml
----------------------------------------------------------------------
diff --git a/plugin-kafka/conf/ranger-policymgr-ssl.xml b/plugin-kafka/conf/ranger-policymgr-ssl.xml
new file mode 100644
index 0000000..e80f1c3
--- /dev/null
+++ b/plugin-kafka/conf/ranger-policymgr-ssl.xml
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
+ <!-- The following properties are used for 2-way SSL client server validation -->
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore</name>
+ <value>kafkadev-clientcert.jks</value>
+ <description>
+ Java Keystore files
+ </description>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.password</name>
+ <value>none</value>
+ <description>
+ password for keystore
+ </description>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore</name>
+ <value>cacerts-xasecure.jks</value>
+ <description>
+ java truststore file
+ </description>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.password</name>
+ <value>none</value>
+ <description>
+ java truststore password
+ </description>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.keystore.credential.file</name>
+ <value>jceks://file/tmp/keystore-kafkadev-ssl.jceks</value>
+ <description>
+ java keystore credential file
+ </description>
+ </property>
+ <property>
+ <name>xasecure.policymgr.clientssl.truststore.credential.file</name>
+ <value>jceks://file/tmp/truststore-kafkadev-ssl.jceks</value>
+ <description>
+ java truststore credential file
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/plugin-kafka/pom.xml b/plugin-kafka/pom.xml
new file mode 100644
index 0000000..e9ea265
--- /dev/null
+++ b/plugin-kafka/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <groupId>security_plugins.ranger-kafka-plugin</groupId>
+ <artifactId>ranger-kafka-plugin</artifactId>
+ <name>KAFKA Security Plugin</name>
+ <description>KAFKA Security Plugin</description>
+ <packaging>jar</packaging>
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ </properties>
+ <parent>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>ranger</artifactId>
+ <version>0.5.0</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <dependencies>
+ <dependency>
+ <groupId>security_plugins.ranger-plugins-common</groupId>
+ <artifactId>ranger-plugins-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>security_plugins.ranger-plugins-audit</groupId>
+ <artifactId>ranger-plugins-audit</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.ranger</groupId>
+ <artifactId>credentialbuilder</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.kafka</groupId>
+ <artifactId>kafka_2.10</artifactId>
+ <version>${kafka.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/scripts/install.properties
----------------------------------------------------------------------
diff --git a/plugin-kafka/scripts/install.properties b/plugin-kafka/scripts/install.properties
new file mode 100644
index 0000000..506c638
--- /dev/null
+++ b/plugin-kafka/scripts/install.properties
@@ -0,0 +1,112 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# Location of Policy Manager URL
+#
+# Example:
+# POLICY_MGR_URL=http://policymanager.xasecure.net:6080
+#
+POLICY_MGR_URL=
+
+#
+# Location of db client library (please check the location of the jar file)
+#
+# Example:
+# SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar
+# SQL_CONNECTOR_JAR=/usr/share/java/ojdbc6.jar
+#
+SQL_CONNECTOR_JAR=/usr/share/java/mysql-connector-java.jar
+
+#
+# This is the repository name created within policy manager
+#
+# Example:
+# REPOSITORY_NAME=kafkadev
+#
+REPOSITORY_NAME=
+
+#
+# AUDIT DB Configuration
+#
+# This information should match with the one you specified during the PolicyManager Installation
+#
+# Example:
+# XAAUDIT.DB.IS_ENABLED=true
+# XAAUDIT.DB.FLAVOUR=MYSQL
+# XAAUDIT.DB.FLAVOUR=ORACLE
+# XAAUDIT.DB.HOSTNAME=localhost
+# XAAUDIT.DB.DATABASE_NAME=ranger_audit
+# XAAUDIT.DB.USER_NAME=rangerlogger
+# XAAUDIT.DB.PASSWORD=rangerlogger
+#
+XAAUDIT.DB.IS_ENABLED=false
+XAAUDIT.DB.FLAVOUR=MYSQL
+XAAUDIT.DB.HOSTNAME=
+XAAUDIT.DB.DATABASE_NAME=
+XAAUDIT.DB.USER_NAME=
+XAAUDIT.DB.PASSWORD=
+
+#
+# Audit to HDFS Configuration
+#
+# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens
+# that start with __REPLACE__ with appropriate values
+# XAAUDIT.HDFS.IS_ENABLED=true
+# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%
+# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit
+# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit/archive
+#
+# Example:
+# XAAUDIT.HDFS.IS_ENABLED=true
+# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd%
+# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/kafka/audit
+# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/kafka/audit/archive
+#
+XAAUDIT.HDFS.IS_ENABLED=false
+XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%
+XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit
+XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/kafka/audit/archive
+
+XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
+XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
+XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
+XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
+XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
+
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
+#
+# SSL Client Certificate Information
+#
+# Example:
+# SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+# SSL_KEYSTORE_PASSWORD=none
+# SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+# SSL_TRUSTSTORE_PASSWORD=none
+#
+# You do not need use SSL between agent and security admin tool, please leave these sample value as it is.
+#
+SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks
+SSL_KEYSTORE_PASSWORD=myKeyFilePassword
+SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks
+SSL_TRUSTSTORE_PASSWORD=changeit
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/scripts/kafka-plugin-install.properties
----------------------------------------------------------------------
diff --git a/plugin-kafka/scripts/kafka-plugin-install.properties b/plugin-kafka/scripts/kafka-plugin-install.properties
new file mode 100644
index 0000000..f4df857
--- /dev/null
+++ b/plugin-kafka/scripts/kafka-plugin-install.properties
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# installation properties for this plugin
+
+#
+# Name of the directory where the component's lib and conf directory exist.
+# This location should be relative to the parent of the directory containing
+# the plugin installation files.
+#
+COMPONENT_INSTALL_DIR_NAME=kafka
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
----------------------------------------------------------------------
diff --git a/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
new file mode 100644
index 0000000..40c2204
--- /dev/null
+++ b/plugin-kafka/src/main/java/org/apache/ranger/authorization/kafka/authorizer/RangerKafkaAuthorizer.java
@@ -0,0 +1,33 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.authorization.kafka.authorizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class RangerKafkaAuthorizer /*KafkaAuthorizationPlugin*/ {
+
+ private static final Log LOG = LogFactory.getLog(RangerKafkaAuthorizer.class);
+
+ //private static volatile RangerKafkaPlugin kafkaPlugin = null;
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java
----------------------------------------------------------------------
diff --git a/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java
new file mode 100644
index 0000000..ea6d316
--- /dev/null
+++ b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/RangerServiceKafka.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.services.kafka;
+
+import java.util.HashMap;
+import java.util.List;
+import org.apache.ranger.plugin.model.RangerService;
+import org.apache.ranger.plugin.model.RangerServiceDef;
+import org.apache.ranger.plugin.service.RangerBaseService;
+import org.apache.ranger.plugin.service.ResourceLookupContext;
+import org.apache.ranger.services.kafka.client.ServiceKafkaClient;
+import org.apache.ranger.services.kafka.client.ServiceKafkaConnectionMgr;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class RangerServiceKafka extends RangerBaseService {
+
+ private static final Log LOG = LogFactory.getLog(RangerServiceKafka.class);
+
+ public RangerServiceKafka() {
+ super();
+ }
+
+ @Override
+ public void init(RangerServiceDef serviceDef, RangerService service) {
+ super.init(serviceDef, service);
+ }
+
+ @Override
+ public HashMap<String, Object> validateConfig() throws Exception {
+ HashMap<String, Object> ret = new HashMap<String, Object>();
+ String serviceName = getServiceName();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> RangerServiceKafka.validateConfig Service: ("
+ + serviceName + " )");
+ }
+ if (configs != null) {
+ try {
+ ret = ServiceKafkaConnectionMgr.testConnection(serviceName,
+ configs);
+ } catch (Exception e) {
+ LOG.error("<== RangerServiceKafka.validateConfig Error:" + e);
+ throw e;
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== RangerServiceKafka.validateConfig Response : (" + ret
+ + " )");
+ }
+ return ret;
+ }
+
+ @Override
+ public List<String> lookupResource(ResourceLookupContext context)
+ throws Exception {
+
+ ServiceKafkaClient serviceKafkaClient = ServiceKafkaConnectionMgr
+ .getKafkaClient(serviceName, configs);
+ return serviceKafkaClient.getResources(context);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java
----------------------------------------------------------------------
diff --git a/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java
new file mode 100644
index 0000000..a62bd95
--- /dev/null
+++ b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaClient.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.services.kafka.client;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+import kafka.utils.ZkUtils;
+
+import org.I0Itec.zkclient.ZkClient;
+import org.apache.log4j.Logger;
+import org.apache.ranger.plugin.client.BaseClient;
+import org.apache.ranger.plugin.service.ResourceLookupContext;
+import org.apache.ranger.plugin.util.TimedEventUtil;
+
+import scala.collection.Iterator;
+import scala.collection.Seq;
+
+public class ServiceKafkaClient {
+ public static final Logger LOG = Logger.getLogger(ServiceKafkaClient.class);
+
+ enum RESOURCE_TYPE {
+ TOPIC
+ }
+
+ String serviceName = null;
+ String zookeeperConnect = null;
+ private static final String errMessage = " You can still save the repository and start creating "
+ + "policies, but you would not be able to use autocomplete for "
+ + "resource names. Check server logs for more info.";
+
+ private static final String TOPIC_KEY = "topic";
+ private static final long LOOKUP_TIMEOUT_SEC = 5;
+
+ public ServiceKafkaClient(String serviceName, String zookeeperConnect) {
+ this.serviceName = serviceName;
+ this.zookeeperConnect = zookeeperConnect;
+ }
+
+ public HashMap<String, Object> testConnection() throws Exception {
+ String errMsg = errMessage;
+ boolean connectivityStatus = false;
+ HashMap<String, Object> responseData = new HashMap<String, Object>();
+ try {
+ getTopicList(null);
+ // If it doesn't throw exception, then assume the instance is
+ // reachable
+ String successMsg = "TestConnection Successful";
+ BaseClient.generateResponseDataMap(connectivityStatus, successMsg,
+ successMsg, null, null, responseData);
+ } catch (IOException e) {
+ LOG.error("Error connecting to Kafka. kafkaClient=" + this, e);
+ String failureMsg = "Unable to connect to Kafka instance."
+ + e.getMessage();
+ BaseClient.generateResponseDataMap(connectivityStatus, failureMsg,
+ failureMsg + errMsg, null, null, responseData);
+ }
+ return responseData;
+ }
+
+ public List<String> getTopicList(List<String> ignoreTopicList)
+ throws Exception {
+
+ List<String> list = new ArrayList<String>();
+
+ ZkClient zkClient = new ZkClient(zookeeperConnect);
+ try {
+ Seq<String> topicList = ZkUtils.getChildrenParentMayNotExist(
+ zkClient, ZkUtils.BrokerTopicsPath());
+
+ Iterator<String> iter = topicList.iterator();
+ while (iter.hasNext()) {
+ String topic = iter.next();
+ if (ignoreTopicList == null || !ignoreTopicList.contains(topic)) {
+ list.add(topic);
+ }
+ }
+ } finally {
+ try {
+ zkClient.close();
+ } catch (Exception ex) {
+ LOG.error("Error closing zookeeper", ex);
+ }
+ }
+ return list;
+ }
+
+ /**
+ * @param serviceName
+ * @param context
+ * @return
+ */
+ public List<String> getResources(ResourceLookupContext context) {
+
+ String userInput = context.getUserInput();
+ String resource = context.getResourceName();
+ Map<String, List<String>> resourceMap = context.getResources();
+ List<String> resultList = null;
+ List<String> topicList = null;
+
+ RESOURCE_TYPE lookupResource = RESOURCE_TYPE.TOPIC;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getResources() UserInput: \"" + userInput
+ + "\" resource : " + resource + " resourceMap: "
+ + resourceMap);
+ }
+
+ if (userInput != null && resource != null) {
+ if (resourceMap != null && !resourceMap.isEmpty()) {
+ topicList = resourceMap.get(TOPIC_KEY);
+ }
+ switch (resource.trim().toLowerCase()) {
+ case TOPIC_KEY:
+ lookupResource = RESOURCE_TYPE.TOPIC;
+ break;
+ default:
+ break;
+ }
+ }
+
+ if (userInput != null) {
+ try {
+ Callable<List<String>> callableObj = null;
+ final String userInputFinal = userInput;
+
+ final List<String> finalTopicList = topicList;
+
+ if (lookupResource == RESOURCE_TYPE.TOPIC) {
+ // get the topic list for given Input
+ callableObj = new Callable<List<String>>() {
+ @Override
+ public List<String> call() {
+ List<String> retList = new ArrayList<String>();
+ try {
+ List<String> list = getTopicList(finalTopicList);
+ if (userInputFinal != null
+ && !userInputFinal.isEmpty()) {
+ for (String value : list) {
+ if (value.startsWith(userInputFinal)) {
+ retList.add(value);
+ }
+ }
+ } else {
+ retList.addAll(list);
+ }
+ } catch (Exception ex) {
+ LOG.error("Error getting topic.", ex);
+ }
+ return retList;
+ };
+ };
+ }
+ // If we need to do lookup
+ if (callableObj != null) {
+ synchronized (this) {
+ resultList = TimedEventUtil.timedTask(callableObj,
+ LOOKUP_TIMEOUT_SEC, TimeUnit.SECONDS);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error("Unable to get hive resources.", e);
+ }
+ }
+
+ return resultList;
+ }
+
+ @Override
+ public String toString() {
+ return "ServiceKafkaClient [serviceName=" + serviceName
+ + ", zookeeperConnect=" + zookeeperConnect + "]";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java
----------------------------------------------------------------------
diff --git a/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java
new file mode 100644
index 0000000..a6d9d55
--- /dev/null
+++ b/plugin-kafka/src/main/java/org/apache/ranger/services/kafka/client/ServiceKafkaConnectionMgr.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.services.kafka.client;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.log4j.Logger;
+
+public class ServiceKafkaConnectionMgr {
+ public static final Logger LOG = Logger
+ .getLogger(ServiceKafkaConnectionMgr.class);
+
+ static public ServiceKafkaClient getKafkaClient(String serviceName,
+ Map<String, String> configs) throws Exception {
+ String zookeeperConnect = configs.get("zookeeper.connect");
+ if (zookeeperConnect != null) {
+ ServiceKafkaClient serviceKafkaClient = new ServiceKafkaClient(
+ serviceName, zookeeperConnect);
+ return serviceKafkaClient;
+ }
+ throw new Exception("Required properties are not set for "
+ + serviceName + ". URL or Zookeeper information not provided.");
+ }
+
+ /**
+ * @param serviceName
+ * @param configs
+ * @return
+ */
+ public static HashMap<String, Object> testConnection(String serviceName,
+ Map<String, String> configs) throws Exception {
+ ServiceKafkaClient serviceKafkaClient = getKafkaClient(serviceName,
+ configs);
+ return serviceKafkaClient.testConnection();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
----------------------------------------------------------------------
diff --git a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
index 3696299..2e6d0ac 100644
--- a/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
+++ b/plugin-solr/src/main/java/org/apache/ranger/services/solr/client/ServiceSolrClient.java
@@ -196,7 +196,7 @@ public class ServiceSolrClient {
RESOURCE_TYPE lookupResource = RESOURCE_TYPE.COLLECTION;
if (LOG.isDebugEnabled()) {
- LOG.debug("<== HiveResourceMgr.getHiveResources() UserInput: \""
+ LOG.debug("<== getResources() UserInput: \""
+ userInput + "\" resource : " + resource
+ " resourceMap: " + resourceMap);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8845eed..09899c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -87,6 +87,7 @@
<module>hive-agent</module>
<module>knox-agent</module>
<module>storm-agent</module>
+ <module>plugin-kafka</module>
<module>plugin-solr</module>
<module>plugin-yarn</module>
<module>ranger_solrj</module>
@@ -364,6 +365,7 @@
<descriptor>src/main/assembly/hbase-agent.xml</descriptor>
<descriptor>src/main/assembly/knox-agent.xml</descriptor>
<descriptor>src/main/assembly/storm-agent.xml</descriptor>
+ <descriptor>src/main/assembly/plugin-kafka.xml</descriptor>
<descriptor>src/main/assembly/plugin-yarn.xml</descriptor>
<descriptor>src/main/assembly/plugin-solr.xml</descriptor>
<descriptor>src/main/assembly/admin-web.xml</descriptor>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/10b57288/src/main/assembly/plugin-kafka.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/plugin-kafka.xml b/src/main/assembly/plugin-kafka.xml
new file mode 100644
index 0000000..b61039d
--- /dev/null
+++ b/src/main/assembly/plugin-kafka.xml
@@ -0,0 +1,157 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<assembly>
+ <id>kafka-plugin</id>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+ <baseDirectory>${project.name}-${project.version}-kafka-plugin</baseDirectory>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <moduleSets>
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <unpack>false</unpack>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/lib</outputDirectory>
+ <unpack>false</unpack>
+ <includes>
+ <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include>
+ <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include>
+ <include>org.apache.hadoop:hadoop-common-plus:jar:${hadoop-common.version}</include>
+ <include>com.google.code.gson:gson</include>
+ <include>org.eclipse.persistence:eclipselink</include>
+ <include>org.eclipse.persistence:javax.persistence</include>
+ <include>commons-collections:commons-collections</include>
+ <include>com.sun.jersey:jersey-bundle</include>
+ <include>commons-logging:commons-logging:jar:${commons.logging.version}</include>
+ <include>com.google.guava:guava:jar:${guava.version}</include>
+ <include>org.apache.httpcomponents:httpclient:jar:${httpcomponent.httpclient.version}</include>
+ <include>org.apache.httpcomponents:httpcore:jar:${httpcomponent.httpcore.version}</include>
+ <include>org.apache.httpcomponents:httpmime:jar:${httpcomponent.httpmime.version}</include>
+ <include>org.noggit:noggit:jar:${noggit.version}</include>
+ </includes>
+ </dependencySet>
+ <dependencySet>
+ <outputDirectory>/install/lib</outputDirectory>
+ <unpack>false</unpack>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <includes>
+ <include>commons-cli:commons-cli</include>
+ <include>commons-collections:commons-collections</include>
+ <include>commons-configuration:commons-configuration:jar:${commons.configuration.version}</include>
+ <include>commons-io:commons-io:jar:${commons.io.version}</include>
+ <include>commons-lang:commons-lang:jar:${commons.lang.version}</include>
+ <include>commons-logging:commons-logging</include>
+ <include>com.google.guava:guava:jar:${guava.version}</include>
+ <include>org.hamcrest:hamcrest-all</include>
+ <include>junit:junit</include>
+ <include>org.slf4j:slf4j-api:jar:${slf4j-api.version}</include>
+ <include>org.apache.hadoop:hadoop-common:jar:${hadoop-common.version}</include>
+ <include>org.apache.hadoop:hadoop-auth:jar:${hadoop-common.version}</include>
+ <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include>
+ <include>org.apache.ranger:credentialbuilder</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ <outputDirectory>/lib</outputDirectory>
+ </binaries>
+ <includes>
+ <include>org.apache.ranger:ranger_solrj</include>
+ <include>security_plugins.ranger-plugins-audit:ranger-plugins-audit</include>
+ <include>security_plugins.ranger-plugins-cred:ranger-plugins-cred</include>
+ <include>security_plugins.ranger-plugins-impl:ranger-plugins-impl</include>
+ <include>security_plugins.ranger-plugins-common:ranger-plugins-common</include>
+ <include>security_plugins.ranger-kafka-plugin:ranger-kafka-plugin</include>
+ </includes>
+ </moduleSet>
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <outputDirectory>/install/lib</outputDirectory>
+ <unpack>false</unpack>
+ </binaries>
+ <includes>
+ <include>security_plugins.ranger-plugins-installer:ranger-plugins-installer</include>
+ <include>org.apache.ranger:credentialbuilder</include>
+ </includes>
+ </moduleSet>
+ </moduleSets>
+ <fileSets>
+ <!-- conf.templates for enable -->
+ <fileSet>
+ <outputDirectory>/install/conf.templates/enable</outputDirectory>
+ <directory>plugin-kafka/conf</directory>
+ <excludes>
+ <exclude>*.sh</exclude>
+ </excludes>
+ <fileMode>700</fileMode>
+ </fileSet>
+ <fileSet>
+ <outputDirectory>/install/conf.templates/disable</outputDirectory>
+ <directory>plugin-kafka/disable-conf</directory>
+ <fileMode>700</fileMode>
+ </fileSet>
+ <fileSet>
+ <outputDirectory>/install/conf.templates/default</outputDirectory>
+ <directory>plugin-kafka/template</directory>
+ <fileMode>700</fileMode>
+ </fileSet>
+ <!-- version file -->
+ <fileSet>
+ <outputDirectory>/</outputDirectory>
+ <directory>${project.build.outputDirectory}</directory>
+ <includes>
+ <include>version</include>
+ </includes>
+ <fileMode>444</fileMode>
+ </fileSet>
+ </fileSets>
+ <!-- enable/disable script for Plugin -->
+ <files>
+ <file>
+ <source>agents-common/scripts/enable-agent.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>enable-kafka-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>agents-common/scripts/enable-agent.sh</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>disable-kafka-plugin.sh</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>plugin-kafka/scripts/install.properties</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>install.properties</destName>
+ <fileMode>755</fileMode>
+ </file>
+ <file>
+ <source>plugin-kafka/scripts/kafka-plugin-install.properties</source>
+ <outputDirectory>/</outputDirectory>
+ <destName>kafka-plugin-install.properties</destName>
+ <fileMode>755</fileMode>
+ </file>
+ </files>
+</assembly>
[2/2] incubator-ranger git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-ranger.git
Posted by bo...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-ranger.git
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/399ed34b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/399ed34b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/399ed34b
Branch: refs/heads/master
Commit: 399ed34b29fa96bac4970251044ad1476d056f57
Parents: 10b5728 0d9a31a
Author: Don Bosco Durai <bo...@apache.org>
Authored: Thu Apr 2 15:30:04 2015 -0700
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Thu Apr 2 15:30:04 2015 -0700
----------------------------------------------------------------------
.../RangerPolicyResourceSignature.java | 19 ++
.../model/validation/RangerPolicyValidator.java | 112 +++++----
.../validation/RangerServiceDefValidator.java | 19 ++
.../ranger/plugin/util/RangerObjectFactory.java | 19 ++
.../TestRangerPolicyResourceSignature.java | 19 ++
.../validation/TestRangerPolicyValidator.java | 228 +++++++++++--------
.../TestRangerServiceDefValidator.java | 19 ++
.../org/apache/ranger/rest/ServiceREST.java | 10 +-
8 files changed, 306 insertions(+), 139 deletions(-)
----------------------------------------------------------------------