You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/10/03 14:36:16 UTC
[13/16] ambari git commit: AMBARI-18348. Ranger audit log collection
is not used in Log Search (Miklos Gergely via oleewere)
AMBARI-18348. Ranger audit log collection is not used in Log Search (Miklos Gergely via oleewere)
Change-Id: I50b4c2c375d1657d620cfd1cab30943688ef079a
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a83bf2b4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a83bf2b4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a83bf2b4
Branch: refs/heads/logsearch-ga
Commit: a83bf2b4e667841e4cce47e489c48b25100368d2
Parents: 5bc031c
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Mon Oct 3 16:13:08 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Mon Oct 3 16:33:39 2016 +0200
----------------------------------------------------------------------
.../ambari/logfeeder/filter/FilterKeyValue.java | 41 +++-
.../logfeeder/filter/FilterKeyValueTest.java | 29 ++-
.../configuration/logsearch-properties.xml | 19 +-
.../LOGSEARCH/0.5.0/package/scripts/params.py | 5 +
.../0.5.0/package/scripts/setup_logsearch.py | 19 +-
.../templates/input.config-ambari.json.j2 | 235 ++++++++++++++++++-
.../test/python/stacks/2.4/configs/default.json | 2 +
ambari-web/app/data/HDP2/site_properties.js | 22 +-
8 files changed, 339 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
index c9c3f2c..5bb15ff 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
@@ -21,7 +21,7 @@ package org.apache.ambari.logfeeder.filter;
import java.util.List;
import java.util.Map;
-import java.util.StringTokenizer;
+import java.util.regex.Pattern;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
@@ -37,7 +37,8 @@ public class FilterKeyValue extends Filter {
private String sourceField = null;
private String valueSplit = "=";
private String fieldSplit = "\t";
-
+ private String valueBorders = null;
+
private MetricData errorMetric = new MetricData("filter.error.keyvalue", false);
@Override
@@ -47,6 +48,7 @@ public class FilterKeyValue extends Filter {
sourceField = getStringValue("source_field");
valueSplit = getStringValue("value_split", valueSplit);
fieldSplit = getStringValue("field_split", fieldSplit);
+ valueBorders = getStringValue("value_borders");
LOG.info("init() done. source_field=" + sourceField + ", value_split=" + valueSplit + ", " + ", field_split=" +
fieldSplit + ", " + getShortDescription());
@@ -68,18 +70,16 @@ public class FilterKeyValue extends Filter {
}
Object valueObj = jsonObj.get(sourceField);
if (valueObj != null) {
- StringTokenizer fieldTokenizer = new StringTokenizer(valueObj.toString(), fieldSplit);
- while (fieldTokenizer.hasMoreTokens()) {
- String nv = fieldTokenizer.nextToken();
- StringTokenizer nvTokenizer = new StringTokenizer(nv, valueSplit);
- while (nvTokenizer.hasMoreTokens()) {
- String name = nvTokenizer.nextToken();
- if (nvTokenizer.hasMoreTokens()) {
- String value = nvTokenizer.nextToken();
+ String splitPattern = Pattern.quote(fieldSplit);
+ String[] tokens = valueObj.toString().split(splitPattern);
+ for (String nv : tokens) {
+ String[] nameValue = getNameValue(nv);
+ String name = nameValue != null && nameValue.length == 2 ? nameValue[0] : null;
+ String value = nameValue != null && nameValue.length == 2 ? nameValue[1] : null;
+ if (name != null && value != null) {
jsonObj.put(name, value);
- } else {
- logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + valueObj);
- }
+ } else {
+ logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + valueObj);
}
}
}
@@ -87,6 +87,21 @@ public class FilterKeyValue extends Filter {
statMetric.value++;
}
+ private String[] getNameValue(String nv) {
+ if (valueBorders != null) {
+ if (nv.charAt(nv.length() - 1) == valueBorders.charAt(1)) {
+ String splitPattern = Pattern.quote("" + valueBorders.charAt(0));
+ return nv.substring(0, nv.length() - 1).split(splitPattern);
+ } else {
+ return null;
+ }
+ }
+ else {
+ String splitPattern = Pattern.quote(valueSplit);
+ return nv.split(splitPattern);
+ }
+ }
+
private void logParseError(String inputStr) {
errorMetric.value++;
String logMessageKey = this.getClass().getSimpleName() + "_PARSEERROR";
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
index 30cee42..05647e6 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,8 +21,8 @@ package org.apache.ambari.logfeeder.filter;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.OutputManager;
+import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
@@ -77,6 +77,31 @@ public class FilterKeyValueTest {
}
@Test
+ public void testFilterKeyValue_extractionWithBorders() throws Exception {
+ LOG.info("testFilterKeyValue_extractionWithBorders()");
+
+ Map<String, Object> config = new HashMap<String, Object>();
+ config.put("source_field", "keyValueField");
+ config.put("field_split", "&");
+ config.put("value_borders", "()");
+ init(config);
+
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ EasyMock.expectLastCall();
+ EasyMock.replay(mockOutputManager);
+
+ filterKeyValue.apply("{ keyValueField: 'name1(value1)&name2(value2)' }", new InputMarker(null, null, 0));
+
+ EasyMock.verify(mockOutputManager);
+ Map<String, Object> jsonParams = capture.getValue();
+
+ assertEquals("Original missing!", "name1(value1)&name2(value2)", jsonParams.remove("keyValueField"));
+ assertEquals("Incorrect extraction: name1", "value1", jsonParams.remove("name1"));
+ assertEquals("Incorrect extraction: name2", "value2", jsonParams.remove("name2"));
+ assertTrue("jsonParams are not empty!", jsonParams.isEmpty());
+ }
+
+ @Test
public void testFilterKeyValue_missingSourceField() throws Exception {
LOG.info("testFilterKeyValue_missingSourceField()");
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
index e521c8e..febeffd 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
@@ -113,6 +113,23 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>logsearch.solr.audit.logs.use.ranger</name>
+ <value>false</value>
+ <display-name>Ranger Audit Logs Enabled</display-name>
+ <description>Use Ranger Audit collection. This is supported only if Ranger Solr is installed in SolrCloud mode</description>
+ <value-attributes>
+ <type>boolean</type>
+ </value-attributes>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>logsearch.ranger.audit.logs.collection.name</name>
+ <value>ranger_audits</value>
+ <display-name>Log Search Solr Ranger Audit Logs Collection</display-name>
+ <description>Name for the ranger audit logs collection</description>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>logsearch.logfeeder.include.default.level</name>
<value>FATAL,ERROR,WARN</value>
<description>Include default Log Feeder Log Levels for Log Search. Used for bootstrapping the configuration only. (levels: FATAL,ERROR,WARN,INFO,DEBUG,TRACE)</description>
@@ -133,7 +150,7 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>logsearch.auth.external_auth.enable</name>
+ <name>logsearch.auth.external_auth.enabled</name>
<value>true</value>
<display-name>External authentication</display-name>
<description>Enable external authentication</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 8b0ce75..7200d7b 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -205,6 +205,11 @@ logsearch_properties['logsearch.solr.collection.history'] = 'history'
logsearch_properties['logsearch.solr.history.config.name'] = 'history'
logsearch_properties['logsearch.collection.history.replication.factor'] = '1'
+if logsearch_properties['logsearch.solr.audit.logs.use.ranger'] == 'false':
+ del logsearch_properties['logsearch.ranger.audit.logs.collection.name']
+
+del logsearch_properties['logsearch.solr.audit.logs.use.ranger']
+
logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts'])
logsearch_properties['logsearch.solr.jmx.port'] = infra_solr_jmx_port
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index 8d84093..6c1a936 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -38,13 +38,14 @@ def setup_logsearch():
)
Directory([params.logsearch_dir, params.logsearch_server_conf, params.logsearch_config_set_dir],
- mode=0755,
- cd_access='a',
- owner=params.logsearch_user,
- group=params.user_group,
- create_parents=True,
- recursive_ownership=True
- )
+ mode=0755,
+ cd_access='a',
+ owner=params.logsearch_user,
+ group=params.user_group,
+ create_parents=True,
+ recursive_ownership=True
+ )
+
File(params.logsearch_log,
mode=0644,
owner=params.logsearch_user,
@@ -53,8 +54,8 @@ def setup_logsearch():
)
PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"),
- properties=params.logsearch_properties
- )
+ properties=params.logsearch_properties
+ )
File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),
content=Template("HadoopServiceConfig.json.j2"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
index 34f4b30..e2646ff 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ambari.json.j2
@@ -52,8 +52,12 @@
},
{
"type":"ambari_audit",
- "rowtype":"service",
+ "rowtype":"audit",
"add_fields":{
+ "logType":"AmbariAudit",
+ "enforcer":"ambari-acl",
+ "repoType":"1",
+ "repo":"ambari",
"level":"INFO"
},
"path":"{{ambari_server_log_dir}}/ambari-audit.log"
@@ -237,10 +241,10 @@
},
"log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
- "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
- "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime},%{SPACE}%{GREEDYDATA:log_message}",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime},%{SPACE}%{GREEDYDATA:log_message}",
"post_map_values":{
- "logtime":{
+ "evtTime":{
"map_date":{
"target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX"
}
@@ -249,6 +253,229 @@
}
+ },
+ {
+ "filter":"keyvalue",
+ "sort_order":1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "ambari_audit"
+ ]
+
+ }
+
+ },
+ "source_field":"log_message",
+ "field_split":", ",
+ "value_borders":"()",
+ "post_map_values":{
+ "User":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+ },
+ "Hostname":{
+ "map_fieldname":{
+ "new_fieldname":"host"
+ }
+ },
+ "RemoteIp":{
+ "map_fieldname":{
+ "new_fieldname":"cliIP"
+ }
+ },
+ "RequestType":{
+ "map_fieldname":{
+ "new_fieldname":"cliType"
+ }
+ },
+ "RequestId":{
+ "map_fieldname":{
+ "new_fieldname":"request_id"
+ }
+ },
+ "TaskId":{
+ "map_fieldname":{
+ "new_fieldname":"task_id"
+ }
+ },
+ "Operation":{
+ "map_fieldname":{
+ "new_fieldname":"action"
+ }
+ },
+ "url":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+ },
+ "ResourcePath":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+ },
+ "Status":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"Success",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Successfully queued",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"QUEUED",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"PENDING",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"COMPLETED",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"IN_PROGRESS",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Failed",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"Failed to queue",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"HOLDING",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"HOLDING_FAILED",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"HOLDING_TIMEDOUT",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"FAILED",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"TIMEDOUT",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"ABORTED",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"SKIPPED_FAILED",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+ }
+ ],
+ "ResultStatus":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"200 OK",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"201 Created",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"202 Accepted",
+ "post_value":"1"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"400 Bad Request",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"401 Unauthorized",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"403 Forbidden",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"404 Not Found",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"409 Resource Conflict",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"500 Internal Server Error",
+ "post_value":"0"
+ }
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+ }
+ ]
+
+ }
+
}
]
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-server/src/test/python/stacks/2.4/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json
index 8ac6fa7..1ebbda9 100644
--- a/ambari-server/src/test/python/stacks/2.4/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json
@@ -289,6 +289,8 @@
"logsearch.collection.service.logs.replication.factor": "1",
"logsearch.collection.audit.logs.numshards": "10",
"logsearch.collection.audit.logs.replication.factor": "1",
+ "logsearch.solr.audit.logs.use.ranger": "false",
+ "logsearch.ranger.audit.logs.collection.name": "ranger_audits",
"logsearch.solr.metrics.collector.hosts" : "{metrics_collector_hosts}",
"logsearch.auth.external_auth.enabled" : "false",
"logsearch.auth.external_auth.host_url" : "{ambari_server_auth_host_url}",
http://git-wip-us.apache.org/repos/asf/ambari/blob/a83bf2b4/ambari-web/app/data/HDP2/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js
index 9311e75..fac40c4 100644
--- a/ambari-web/app/data/HDP2/site_properties.js
+++ b/ambari-web/app/data/HDP2/site_properties.js
@@ -2208,33 +2208,47 @@ var hdp2properties = [
"index": 5
},
{
- "name": "logsearch.solr.metrics.collector.hosts",
+ "name": "logsearch.solr.audit.logs.use.ranger",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
"index": 6
},
{
- "name": "logsearch.auth.external_auth.enable",
+ "name": "logsearch.ranger.audit.logs.collection.name",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
"index": 7
},
{
- "name": "logsearch.auth.external_auth.host_url",
+ "name": "logsearch.solr.metrics.collector.hosts",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
"index": 8
},
{
- "name": "logsearch.auth.external_auth.login_url",
+ "name": "logsearch.auth.external_auth.enable",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
"index": 9
},
+ {
+ "name": "logsearch.auth.external_auth.host_url",
+ "serviceName": "LOGSEARCH",
+ "filename": "logsearch-properties.xml",
+ "category": "Advanced logsearch-properties",
+ "index": 10
+ },
+ {
+ "name": "logsearch.auth.external_auth.login_url",
+ "serviceName": "LOGSEARCH",
+ "filename": "logsearch-properties.xml",
+ "category": "Advanced logsearch-properties",
+ "index": 11
+ },
/*infra-solr-client-log4j*/
{
"name": "infra_solr_client_log_dir",