You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by bo...@apache.org on 2015/12/10 21:17:53 UTC
[3/3] incubator-ranger git commit: RANGER-728: Update Solr script to
resolve issues with ZK and creating collection
RANGER-728: Update Solr script to resolve issues with ZK and creating
collection
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/269617d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/269617d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/269617d5
Branch: refs/heads/master
Commit: 269617d5dbf13fcbf9600efca72bc5a803f49a92
Parents: bb81459
Author: Don Bosco Durai <bo...@apache.org>
Authored: Wed Dec 9 16:23:11 2015 -0800
Committer: Don Bosco Durai <bo...@apache.org>
Committed: Thu Dec 10 12:17:19 2015 -0800
----------------------------------------------------------------------
.../create_hdfs_folders_for_audit_secure.sh | 4 +-
.../solr_for_audit_setup/conf/managed-schema | 92 +
.../solr_for_audit_setup/conf/schema.xml | 118 --
.../solr_for_audit_setup/conf/solrconfig.xml | 13 +
.../solr_for_audit_setup/conf/solrconfig.xml.j2 | 1878 ++++++++++++++++++
.../solr_for_audit_setup/install.properties | 3 +
.../resources/log4j.properties.j2 | 40 +
.../resources/log4j.properties.template | 39 -
.../contrib/solr_for_audit_setup/setup.sh | 31 +-
.../scripts/add_ranger_audits_conf_to_zk.sh.j2 | 63 +
.../add_ranger_audits_conf_to_zk.sh.template | 63 -
.../create_ranger_audits_collection.sh.j2 | 33 +
.../create_ranger_audits_collection.sh.template | 33 -
.../solr_cloud/scripts/solr.in.sh.j2 | 116 ++
.../solr_cloud/scripts/solr.sh.j2 | 21 +
.../solr_cloud/scripts/start_solr.sh.j2 | 32 +
.../solr_cloud/scripts/start_solr.sh.template | 39 -
.../solr_cloud/scripts/stop_solr.sh.j2 | 33 +
.../solr_cloud/scripts/stop_solr.sh.template | 35 -
.../solr_for_audit_setup/solr_cloud/solr.xml.j2 | 26 +
.../solr_cloud/solr.xml.template | 26 -
.../ranger_audits/core.properties.j2 | 20 +
.../ranger_audits/core.properties.template | 20 -
.../solr_standalone/scripts/solr.in.sh.j2 | 116 ++
.../solr_standalone/scripts/solr.sh.j2 | 21 +
.../solr_standalone/scripts/start_solr.sh.j2 | 33 +
.../scripts/start_solr.sh.template | 38 -
.../solr_standalone/scripts/stop_solr.sh.j2 | 33 +
.../scripts/stop_solr.sh.template | 35 -
29 files changed, 2593 insertions(+), 461 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
----------------------------------------------------------------------
diff --git a/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh b/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
index 12a4c93..9a5bdd5 100755
--- a/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
+++ b/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
@@ -14,9 +14,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-#Usage: Run this script as user hdfs or the HDFS admin user.
+#Usage: Use this script in kerberos enabled hadoop only. Run this script after kinit'ing as hdfs user
#This script creates the folders in HDFS required by Apache Ranger for writing Audit records
-#Note 1: Use this script only for non-kerberos environment. In non-kerberos environment, Ranger KMS writes the audit logs as user "HTTP"
+#Note 1: Use this script only for kerberos environment. In kerberos environment, Ranger KMS writes the audit logs as user "HTTP"
#Note 2: Please update the below variables according to your environment
HBASE_USER_GROUP=hbase:hbase
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
new file mode 100644
index 0000000..7b9769a
--- /dev/null
+++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<schema name="ranger-audit-schema" version="1.5">
+ <uniqueKey>id</uniqueKey>
+ <fieldType name="binary" class="solr.BinaryField"/>
+ <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+ <fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/>
+ <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/>
+ <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="key_lower_case" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+ <analyzer>
+ <tokenizer class="solr.KeywordTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+ <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+ <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.StandardTokenizerFactory"/>
+ <filter class="solr.LowerCaseFilterFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+ <analyzer>
+ <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+ </analyzer>
+ </fieldType>
+ <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
+ <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" multiValued="true" positionIncrementGap="0"/>
+ <field name="_expire_at_" type="tdate" multiValued="false" stored="true"/>
+ <field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/>
+ <field name="_version_" type="long" indexed="true" stored="true"/>
+ <field name="access" type="key_lower_case" multiValued="false"/>
+ <field name="action" type="key_lower_case" multiValued="false"/>
+ <field name="agent" type="key_lower_case" multiValued="false"/>
+ <field name="agentHost" type="key_lower_case" multiValued="false"/>
+ <field name="cliIP" type="key_lower_case" multiValued="false"/>
+ <field name="cliType" type="key_lower_case" multiValued="false"/>
+ <field name="cluster" type="key_lower_case" multiValued="false"/>
+ <field name="reqContext" type="key_lower_case" multiValued="true"/>
+ <field name="enforcer" type="key_lower_case" multiValued="false"/>
+ <field name="event_count" type="tlong" multiValued="false" docValues="true"/>
+ <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
+ <field name="evtTime" type="tdate"/>
+ <field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
+ <field name="logType" type="key_lower_case" multiValued="false"/>
+ <field name="policy" type="tlong" docValues="true"/>
+ <field name="proxyUsers" type="key_lower_case" multiValued="true"/>
+ <field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
+ <field name="repo" type="key_lower_case" multiValued="false"/>
+ <field name="repoType" type="tint" multiValued="false" docValues="true"/>
+ <field name="req_caller_id" type="key_lower_case" multiValued="false"/>
+ <field name="req_self_id" type="key_lower_case" multiValued="false"/>
+ <field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
+ <field name="reqUser" type="key_lower_case" multiValued="false"/>
+ <field name="resType" type="key_lower_case" multiValued="false"/>
+ <field name="resource" type="key_lower_case" multiValued="false"/>
+ <field name="result" type="tint" multiValued="false"/>
+ <field name="seq_num" type="tlong" multiValued="false" docValues="true"/>
+ <field name="sess" type="key_lower_case" multiValued="false"/>
+ <field name="tags" type="key_lower_case" multiValued="true"/>
+ <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
+ <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/>
+</schema>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/solr_for_audit_setup/conf/schema.xml
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/schema.xml b/security-admin/contrib/solr_for_audit_setup/conf/schema.xml
deleted file mode 100644
index df56974..0000000
--- a/security-admin/contrib/solr_for_audit_setup/conf/schema.xml
+++ /dev/null
@@ -1,118 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Trimmed schema.xml to include only the fields used by RangerAudit and also to make it more manageable -->
-<schema name="ranger-audit-schema" version="1.5">
- <fields>
- <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
- <field name="_version_" type="long" indexed="true" stored="true"/>
-
- <field name="resource" type="key_lower_case" multiValued="false"/>
- <field name="cluster" type="key_lower_case" multiValued="false"/>
- <field name="resType" type="key_lower_case" multiValued="false"/>
- <field name="reqUser" type="key_lower_case" multiValued="false"/>
- <field name="evtTime" type="tdate"/>
- <field name="policy" type="tlong" docValues="true"/>
- <field name="sess" type="key_lower_case" multiValued="false"/>
- <field name="access" type="key_lower_case" multiValued="false"/>
- <field name="result" type="tint" multiValued="false"/>
- <field name="reason" type="text_std_token_lower_case" omitNorms="false" multiValued="false"/>
- <field name="enforcer" type="key_lower_case" multiValued="false"/>
- <field name="repo" type="key_lower_case" multiValued="false"/>
- <field name="cliIP" type="key_lower_case" multiValued="false"/>
- <field name="cliType" type="key_lower_case" multiValued="false"/>
- <field name="action" type="key_lower_case" multiValued="false"/>
- <field name="agent" type="key_lower_case" multiValued="false"/>
- <field name="agentHost" type="key_lower_case" multiValued="false"/>
- <field name="logType" type="key_lower_case" multiValued="false"/>
- <field name="repoType" type="tint" multiValued="false" docValues="true"/>
- <field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
- <field name="seq_num" type="tlong" multiValued="false" docValues="true"/>
- <field name="tags" type="key_lower_case" multiValued="true"/>
- <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
- <field name="event_count" type="tlong" multiValued="false" docValues="true"/>
- <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
-
- <field name="text" type="text_std_token_lower_case" indexed="true" stored="false" multiValued="true"/>
- </fields>
-
- <uniqueKey>id</uniqueKey>
-
- <types>
-
- <!-- Updated from text_general and having only StandardTokenizer and LowerCaseFilter-->
- <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true">
- <analyzer>
- <tokenizer class="solr.StandardTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory" />
- </analyzer>
- </fieldType>
-
- <!-- Keyword tokenizer won't split the string. So is like full string search -->
- <fieldType name="key_lower_case" class="solr.TextField"
- sortMissingLast="true" omitNorms="true">
- <analyzer>
- <tokenizer class="solr.KeywordTokenizerFactory"/>
- <filter class="solr.LowerCaseFilterFactory" />
- </analyzer>
- </fieldType>
-
- <!-- The StrField type is not analyzed, but indexed/stored verbatim.
- It supports doc values but in that case the field needs to be
- single-valued and either required or have a default value.
- -->
- <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
-
- <!-- boolean type: "true" or "false" -->
- <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
-
- <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
-
- <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
- <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
-
- <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
- <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
-
- <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0" multiValued="true"/>
- <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
- <!-- A Trie based date field for faster date range queries and date faceting. -->
- <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
- <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0" multiValued="true"/>
- <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
- <fieldtype name="binary" class="solr.BinaryField"/>
- <fieldType name="random" class="solr.RandomSortField" indexed="true" />
- <!-- A text field that only splits on whitespace for exact matching of words -->
- <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
- <analyzer>
- <tokenizer class="solr.WhitespaceTokenizerFactory"/>
- </analyzer>
- </fieldType>
- <!-- since fields of this type are by default not stored or indexed,
- any data added to them will be ignored outright. -->
- <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
-
- </types>
-</schema>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml b/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
index 0991eba..7e71f9a 100644
--- a/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
+++ b/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
@@ -1642,6 +1642,19 @@
See http://wiki.apache.org/solr/GuessingFieldTypes
-->
<updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
+ <processor class="solr.DefaultValueUpdateProcessorFactory">
+ <str name="fieldName">_ttl_</str>
+ <str name="value">+1095DAYS</str>
+ </processor>
+ <processor class="solr.processor.DocExpirationUpdateProcessorFactory">
+ <int name="autoDeletePeriodSeconds">300</int>
+ <str name="ttlFieldName">_ttl_</str>
+ <str name="expirationFieldName">_expire_at_</str>
+ </processor>
+ <processor class="solr.FirstFieldValueUpdateProcessorFactory">
+ <str name="fieldName">_expire_at_</str>
+ </processor>
+
<processor class="solr.RemoveBlankFieldUpdateProcessorFactory"/>
<processor class="solr.ParseBooleanFieldUpdateProcessorFactory"/>
<processor class="solr.ParseLongFieldUpdateProcessorFactory"/>