You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2017/05/31 07:10:13 UTC
ambari git commit: AMBARI-21144 Create .md files to describe Log
Search input configurations (mgergely)
Repository: ambari
Updated Branches:
refs/heads/trunk 5845d5463 -> f0d970b5b
AMBARI-21144 Create .md files to describe Log Search input configurations (mgergely)
Change-Id: I632de5e66607119548ba083fdfed31d39a4e085f
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f0d970b5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f0d970b5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f0d970b5
Branch: refs/heads/trunk
Commit: f0d970b5b93a40c2e0830d5eaf3039c81ada2290
Parents: 5845d54
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed May 31 09:10:00 2017 +0200
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Wed May 31 09:10:00 2017 +0200
----------------------------------------------------------------------
.../api/model/inputconfig/InputDescriptor.java | 2 -
.../inputconfig/impl/InputDescriptorImpl.java | 12 -
.../inputconfig/impl/MapDateDescriptorImpl.java | 2 +-
.../ambari-logsearch-logfeeder/README.md | 17 +-
.../ambari-logsearch-logfeeder/docs/filter.md | 59 ++++
.../ambari-logsearch-logfeeder/docs/input.md | 63 ++++
.../docs/inputConfig.md | 29 ++
.../docs/postMapValues.md | 61 ++++
.../logfeeder/input/AbstractInputFile.java | 9 +-
.../ambari/logfeeder/mapper/MapperDate.java | 2 +-
.../ambari/logfeeder/input/InputFileTest.java | 1 -
.../resources/samples/config/config_audit.json | 339 +++++++++----------
.../samples/config/config_service.json | 79 +++--
.../logsearch/model/common/LSServerInput.java | 8 -
.../logsearch/model/common/LSServerMapDate.java | 2 +-
.../logfeeder/shipper-conf/global.config.json | 5 +-
.../package/templates/global.config.json.j2 | 3 +-
17 files changed, 442 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
index c41da93..82e9504 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
@@ -38,8 +38,6 @@ public interface InputDescriptor {
Boolean isUseEventMd5AsId();
- String getStartPosition();
-
Boolean isCacheEnabled();
String getCacheKeyField();
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
index 94dcc2a..54b4b9b 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
@@ -53,10 +53,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
@Expose
@SerializedName("use_event_md5_as_id")
private Boolean useEventMd5AsId;
-
- @Expose
- @SerializedName("start_position")
- private String startPosition;
@Expose
@SerializedName("cache_enabled")
@@ -146,14 +142,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
this.useEventMd5AsId = useEventMd5AsId;
}
- public String getStartPosition() {
- return startPosition;
- }
-
- public void setStartPosition(String startPosition) {
- this.startPosition = startPosition;
- }
-
public Boolean isCacheEnabled() {
return cacheEnabled;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
index 9daad2b..2e54e7a 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
@@ -31,7 +31,7 @@ public class MapDateDescriptorImpl implements MapDateDescriptor {
}
@Expose
- @SerializedName("source_date_pattern")
+ @SerializedName("src_date_pattern")
private String sourceDatePattern;
@Expose
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/README.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
index d00eb57..c5c6848 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
@@ -17,9 +17,10 @@ limitations under the License.
{% endcomment %}
-->
-#LogSearch - LogFeeder:
+#Log Search - Log Feeder:
-Logfeeder is a tool that reads log, parses it and stores it in Apache Solr for analyising purpose.
+Log Feeder is a component of the Log Search service that reads logs, parses them and stores them in Apache Solr for the purpose
+of later analysis.
#Compilation
mvn clean compile package
@@ -34,3 +35,15 @@ mkdir /opt/logfeeder
cd /opt/logfeeder
tar xfz ~/logsearch-logfeeder.tar.gz
```
+
+#Input Configuration
+
+The configuration for the log feeder contains
+* description of the log files
+* description of the filters that parse the data of the log entries
+* description of the mappers that modify the parsed fields
+
+The element description can be found [here](docs/inputConfig.md)
+
+All these data are stored in json files, which should be named in the directory /etc/ambari-logsearch-logfeeder/conf, and the
+name of the files should be input.config-<service\_name>.json
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
new file mode 100644
index 0000000..42c3271
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
@@ -0,0 +1,59 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Filter
+
+The filter element in the [input configuration](inputConfig.md) contains a list of filter descriptions, each describing one filter
+applied on an input.
+
+The general elements in the json are the following:
+
+| Field | Description | Default |
+|-----------------------|-------------------------------------------------------------------------------------------------|-------------|
+| filter | The type of the filter, currently grok, keyvalue and json are supported | - |
+| conditions | The conditions of which input to filter | - |
+| sort\_order | Describes the order in which the filters should be applied | - |
+| source\_field | The source of the filter, must be set for keyvalue filters | log_message |
+| remove\_source\_field | Remove the source field after the filter is applied | false |
+| post\_map\_values | Mappings done after the filtering provided it's result, see [post map values](postMapValues.md) | - |
+| is\_enabled | A flag to show if the filter should be used | true |
+
+
+##Grok Filter
+
+Grok filters have the following additional parameters:
+
+| Field | Description | Default |
+|--------------------|------------------------------------------------------------------------------------------------------------|---------|
+| log4j\_format | The log4j pattern of the log, not used, it is only there for documentation | - |
+| multiline\_pattern | The grok pattern that shows that the line is not a log line on it's own but the part of a multi line entry | - |
+| message\_pattern | The grok pattern to use to parse the log entry | - |
+
+
+##Key-value Filter
+
+Key-value filters have the following additional parameters:
+
+| Field | Description | Default |
+|----------------|-------------------------------------------------------------------------------------------|---------|
+| field\_split | The string that splits the key-value pairs | "\t" |
+| value\_split | The string that separates keys from values | "=" |
+| value\_borders | The borders around the value, must be 2 characters long, first before it, second after it | - |
+
+If value\_borders is only used if it is specified, and value\_split is not.
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
new file mode 100644
index 0000000..e95401a
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
@@ -0,0 +1,63 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Input
+
+The input element in the [input configuration](inputConfig.md) contains a list of input descriptions, each describing one source
+of input.
+
+The general elements in the json are the following:
+
+| Field | Description | Default |
+|-----------------------------|-------------------------------------------------------------------------------------------------------|--------------|
+| type | The type of the input source, currently file and s3_file are supported | - |
+| rowtype | The type of the row, can be service / audit | - |
+| path | The path of the source, may contain '*' characters too | - |
+| add\_fields | The element contains field\_name: field\_value pairs which will be added to each rows data | - |
+| tail | The input should check for only the latest file matching the pattern, not all of them | true |
+| gen\_event\_md5 | Generate an event\_md5 field for each row by creating a hash of the row data | true |
+| use\_event\_md5\_as\_id | Generate an id for each row by creating a hash of the row data | false |
+| start\_position | Should the parsing start from the beginning | beginning |
+| cache\_enabled | Allows the input to use a cache to filter out duplications | true |
+| cache\_key\_field | Specifies the field for which to use the cache to find duplications of | log\_message |
+| cache\_last\_dedup\_enabled | Allow to filter out entries which are same as the most recent one irrelevant of it's time | false |
+| cache\_size | The number of entries to store in the cache | 100 |
+| cache\_dedup\_interval | The maximum interval in ms which may pass between two identical log messages to filter the latter out | 1000 |
+| is\_enabled | A flag to show if the input should be used | true |
+
+
+##File Input
+
+File inputs have the following parameters too:
+
+| Field | Description | Default |
+|--------------------------|--------------------------------------------------------------------|---------|
+| checkpoint\_interval\_ms | The time interval in ms when the checkpoint file should be updated | 5000 |
+| process\_file | Should the file be processed | true |
+| copy\_file | Should the file be copied \(only if not processed\) | false |
+
+
+##S3 File Input
+
+S3 file inputs have the following parameters in addition to the general file parameters:
+
+| Field | Description | Default |
+|-----------------|-----------------------------------------|---------|
+| s3\_access\_key | The access key used for AWS credentials | - |
+| s3\_secret\_key | The secret key used for AWS credentials | - |
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md
new file mode 100644
index 0000000..ae7ac05
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/inputConfig.md
@@ -0,0 +1,29 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Input Configuration
+
+The input configurations are stored in json files. Each of them are describing the processing of the log files of a service.
+
+The json contains two elements:
+
+| Field | Description |
+|--------|--------------------------------------------|
+| input | A list of [input descriptions](input.md) |
+| filter | A list of [filter descriptions](filter.md) |
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
new file mode 100644
index 0000000..aa381cd
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
@@ -0,0 +1,61 @@
+<!--
+{% comment %}
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to you under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+{% endcomment %}
+-->
+
+#Post Map Values
+
+The Post Map Values element in the [filter](filter.md) field names as keys, the values are lists of sets of post map values, each
+describing one mapping done on a field named before obtained after filtering.
+
+Currently there are four kind of mappings are supported:
+
+##Map Date
+
+The name of the mapping element should be map\_date. The value json element may contain the following parameters:
+
+| Field | Description |
+|-----------------------|--------------------------------------------------------------------------------------------------------|
+| src\_date\_pattern | If it is specified than the mapper converts from this format to the target, and also adds missing year |
+| target\_date\_pattern | If 'epoch' then the field is parsed as seconds from 1970, otherwise the content used as pattern |
+
+
+##Map Copy
+
+The name of the mapping element should be map\_copy. The value json element should contain the following parameter:
+
+| Field | Description |
+|------------|-------------------------------|
+| copy\_name | The name of the copied field |
+
+
+##Map Field Name
+
+The name of the mapping element should be map\_fieldname. The value json element should contain the following parameter:
+
+| Field | Description |
+|------------------|-------------------------------|
+| new\_field\_name | The name of the renamed field |
+
+##Map Field Value
+
+The name of the mapping element should be map\_fieldvalue. The value json element should contain the following parameter:
+
+| Field | Description |
+|-------------|--------------------------------------------------------------------|
+| pre\_value | The value that the field must match \(ignoring case\) to be mapped |
+| post\_value | The value to which the field is modified to |
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
index cfa1903..2bc4e68 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
@@ -45,7 +45,6 @@ public abstract class AbstractInputFile extends Input {
protected String base64FileKey;
protected boolean isReady;
- private boolean isStartFromBegining = true;
private String checkPointExtension;
private File checkPointFile;
@@ -82,12 +81,6 @@ public abstract class AbstractInputFile extends Input {
return;
}
- String startPosition = inputDescriptor.getStartPosition();
- if (StringUtils.isEmpty(startPosition) || startPosition.equalsIgnoreCase("beginning") ||
- startPosition.equalsIgnoreCase("begining") || !tail) {
- isStartFromBegining = true;
- }
-
setFilePath(logPath);
boolean isFileReady = isReady();
@@ -109,7 +102,7 @@ public abstract class AbstractInputFile extends Input {
br = openLogFile(logPathFile);
- boolean resume = isStartFromBegining;
+ boolean resume = true;
int resumeFromLineNumber = getResumeFromLineNumber();
if (resumeFromLineNumber > 0) {
resume = false;
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index 5d34c06..305688b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -38,7 +38,7 @@ public class MapperDate extends Mapper {
private SimpleDateFormat targetDateFormatter = null;
private boolean isEpoch = false;
- private SimpleDateFormat srcDateFormatter=null;
+ private SimpleDateFormat srcDateFormatter = null;
@Override
public boolean init(String inputDesc, String fieldName, String mapClassCode, MapFieldDescriptor mapFieldDescriptor) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
index 3a5f31e..efebc08 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
@@ -81,7 +81,6 @@ public class InputFileTest {
inputFileDescriptor.setSource("file");
inputFileDescriptor.setTail(true);
inputFileDescriptor.setGenEventMd5(true);
- inputFileDescriptor.setStartPosition("beginning");
inputFileDescriptor.setType("hdfs_datanode");
inputFileDescriptor.setRowtype("service");
inputFileDescriptor.setPath(path);
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
index b7d0a6e..05f3bff 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_audit.json
@@ -1,173 +1,172 @@
{
- "global": {
- "add_fields": {
- "cluster": "audit"
- },
- "source": "file",
- "tail": "true",
- "gen_event_md5": "true",
- "start_position": "beginning"
- },
- "input": [{
- "type": "hdfs_audit",
- "rowtype": "hdfs_audit",
- "path": "{path}/src/test/resources/samples/jsonlogs/audit_log.json"
- }],
- "filter": [{
- "filter": "json",
- "conditions": {
- "fields": {
- "type": [
- "hdfs_audit"
- ]
-
- }
- }
- }, {
- "filter": "keyvalue",
- "sort_order": 1,
- "conditions": {
- "fields": {
- "type": [
- "hdfs_audit"
- ]
-
- }
-
- },
- "source_field": "log_message",
- "value_split": "=",
- "field_split": "\t",
- "post_map_values": {
- "src": {
- "map_fieldname": {
- "new_fieldname": "resource"
- }
-
- },
- "ip": {
- "map_fieldname": {
- "new_fieldname": "cliIP"
- }
-
- },
- "allowed": [{
- "map_fieldvalue": {
- "pre_value": "true",
- "post_value": "1"
- }
-
- }, {
- "map_fieldvalue": {
- "pre_value": "false",
- "post_value": "0"
- }
-
- }, {
- "map_fieldname": {
- "new_fieldname": "result"
- }
-
- }
-
- ],
- "cmd": {
- "map_fieldname": {
- "new_fieldname": "action"
- }
-
- },
- "proto": {
- "map_fieldname": {
- "new_fieldname": "cliType"
- }
-
- },
- "callerContext": {
- "map_fieldname": {
- "new_fieldname": "req_caller_id"
- }
-
- }
-
- }
-
- }, {
- "filter": "grok",
- "sort_order": 2,
- "source_field": "ugi",
- "remove_source_field": "false",
- "conditions": {
- "fields": {
- "type": [
- "hdfs_audit"
- ]
-
- }
-
- },
- "message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
- "post_map_values": {
- "user": {
- "map_fieldname": {
- "new_fieldname": "reqUser"
- }
-
- },
- "x_user": {
- "map_fieldname": {
- "new_fieldname": "reqUser"
- }
-
- },
- "p_user": {
- "map_fieldname": {
- "new_fieldname": "reqUser"
- }
-
- },
- "k_user": {
- "map_fieldname": {
- "new_fieldname": "proxyUsers"
- }
-
- },
- "p_authType": {
- "map_fieldname": {
- "new_fieldname": "authType"
- }
-
- },
- "k_authType": {
- "map_fieldname": {
- "new_fieldname": "proxyAuthType"
- }
-
- }
-
- }
-
- }
-
- ],
-
- "output": [{
- "is_enabled": "true",
- "comment": "Output to file for audit logs",
- "destination": "solr",
- "url": "http://localhost:8983/solr/audit_logs",
- "collection": "audit_logs",
- "number_of_shards": "1",
- "splits_interval_mins": "100000",
- "conditions": {
- "fields": {
- "rowtype": [
- "hdfs_audit"
- ]
- }
- }
- }
-
- ]
+ "global": {
+ "add_fields": {
+ "cluster": "audit"
+ },
+ "source": "file",
+ "tail": "true",
+ "gen_event_md5": "true"
+ },
+ "input": [{
+ "type": "hdfs_audit",
+ "rowtype": "hdfs_audit",
+ "path": "{path}/src/test/resources/samples/jsonlogs/audit_log.json"
+ }],
+ "filter": [{
+ "filter": "json",
+ "conditions": {
+ "fields": {
+ "type": [
+ "hdfs_audit"
+ ]
+
+ }
+ }
+ }, {
+ "filter": "keyvalue",
+ "sort_order": 1,
+ "conditions": {
+ "fields": {
+ "type": [
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "source_field": "log_message",
+ "value_split": "=",
+ "field_split": "\t",
+ "post_map_values": {
+ "src": {
+ "map_fieldname": {
+ "new_fieldname": "resource"
+ }
+
+ },
+ "ip": {
+ "map_fieldname": {
+ "new_fieldname": "cliIP"
+ }
+
+ },
+ "allowed": [{
+ "map_fieldvalue": {
+ "pre_value": "true",
+ "post_value": "1"
+ }
+
+ }, {
+ "map_fieldvalue": {
+ "pre_value": "false",
+ "post_value": "0"
+ }
+
+ }, {
+ "map_fieldname": {
+ "new_fieldname": "result"
+ }
+
+ }
+
+ ],
+ "cmd": {
+ "map_fieldname": {
+ "new_fieldname": "action"
+ }
+
+ },
+ "proto": {
+ "map_fieldname": {
+ "new_fieldname": "cliType"
+ }
+
+ },
+ "callerContext": {
+ "map_fieldname": {
+ "new_fieldname": "req_caller_id"
+ }
+
+ }
+
+ }
+
+ }, {
+ "filter": "grok",
+ "sort_order": 2,
+ "source_field": "ugi",
+ "remove_source_field": "false",
+ "conditions": {
+ "fields": {
+ "type": [
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "message_pattern": "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+ "post_map_values": {
+ "user": {
+ "map_fieldname": {
+ "new_fieldname": "reqUser"
+ }
+
+ },
+ "x_user": {
+ "map_fieldname": {
+ "new_fieldname": "reqUser"
+ }
+
+ },
+ "p_user": {
+ "map_fieldname": {
+ "new_fieldname": "reqUser"
+ }
+
+ },
+ "k_user": {
+ "map_fieldname": {
+ "new_fieldname": "proxyUsers"
+ }
+
+ },
+ "p_authType": {
+ "map_fieldname": {
+ "new_fieldname": "authType"
+ }
+
+ },
+ "k_authType": {
+ "map_fieldname": {
+ "new_fieldname": "proxyAuthType"
+ }
+
+ }
+
+ }
+
+ }
+
+ ],
+
+ "output": [{
+ "is_enabled": "true",
+ "comment": "Output to file for audit logs",
+ "destination": "solr",
+ "url": "http://localhost:8983/solr/audit_logs",
+ "collection": "audit_logs",
+ "number_of_shards": "1",
+ "splits_interval_mins": "100000",
+ "conditions": {
+ "fields": {
+ "rowtype": [
+ "hdfs_audit"
+ ]
+ }
+ }
+ }
+
+ ]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
index 2e96fd7..0fff050 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/samples/config/config_service.json
@@ -1,43 +1,42 @@
{
- "global": {
- "add_fields": {
- "cluster": "cluster_name"
- },
- "source": "file",
- "tail": "true",
- "gen_event_md5": "true",
- "start_position": "beginning"
- },
- "input": [{
- "type": "logsearch",
- "rowtype": "service",
- "path": "{path}/src/test/resources/samples/jsonlogs/service_log.json"
- }],
- "filter": [{
- "filter": "json",
- "conditions": {
- "fields": {
- "type": [
- "logsearch"
- ]
- }
- }
- }],
- "output": [{
- "comment": "Output to solr for service records",
- "is_enabled": "true",
- "destination": "solr",
- "url": "http://localhost:8983/solr/hadoop_logs",
- "collection": "hadoop_logs",
- "number_of_shards": "1",
- "splits_interval_mins": "100000",
- "conditions": {
- "fields": {
- "rowtype": [
- "service"
- ]
- }
- }
- }]
+ "global": {
+ "add_fields": {
+ "cluster": "cluster_name"
+ },
+ "source": "file",
+ "tail": "true",
+ "gen_event_md5": "true"
+ },
+ "input": [{
+ "type": "logsearch",
+ "rowtype": "service",
+ "path": "{path}/src/test/resources/samples/jsonlogs/service_log.json"
+ }],
+ "filter": [{
+ "filter": "json",
+ "conditions": {
+ "fields": {
+ "type": [
+ "logsearch"
+ ]
+ }
+ }
+ }],
+ "output": [{
+ "comment": "Output to solr for service records",
+ "is_enabled": "true",
+ "destination": "solr",
+ "url": "http://localhost:8983/solr/hadoop_logs",
+ "collection": "hadoop_logs",
+ "number_of_shards": "1",
+ "splits_interval_mins": "100000",
+ "conditions": {
+ "fields": {
+ "rowtype": [
+ "service"
+ ]
+ }
+ }
+ }]
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
index e0f7564..6ef3d3f 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java
@@ -53,9 +53,6 @@ public abstract class LSServerInput {
@JsonProperty("use_event_md5_as_id")
private Boolean useEventMd5AsId;
- @JsonProperty("start_position")
- private String startPosition;
-
@JsonProperty("cache_enabled")
private Boolean cacheEnabled;
@@ -85,7 +82,6 @@ public abstract class LSServerInput {
this.tail = inputDescriptor.isTail();
this.genEventMd5 = inputDescriptor.isGenEventMd5();
this.useEventMd5AsId = inputDescriptor.isUseEventMd5AsId();
- this.startPosition = inputDescriptor.getStartPosition();
this.cacheEnabled = inputDescriptor.isCacheEnabled();
this.cacheKeyField = inputDescriptor.getCacheKeyField();
this.cacheLastDedupEnabled = inputDescriptor.getCacheLastDedupEnabled();
@@ -126,10 +122,6 @@ public abstract class LSServerInput {
return useEventMd5AsId;
}
- public String getStartPosition() {
- return startPosition;
- }
-
public Boolean getCacheEnabled() {
return cacheEnabled;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
index 3147402..dcb0393 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerMapDate.java
@@ -34,7 +34,7 @@ public class LSServerMapDate extends LSServerMapField {
return "map_date";
}
- @JsonProperty("source_date_pattern")
+ @JsonProperty("src_date_pattern")
private String sourceDatePattern;
@NotNull
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
index 7ac29dd..65da726 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/global.config.json
@@ -5,7 +5,6 @@
},
"source":"file",
"tail":"true",
- "gen_event_md5":"true",
- "start_position":"beginning"
+ "gen_event_md5":"true"
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/f0d970b5/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
index f337527..1b63e2a 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/global.config.json.j2
@@ -22,7 +22,6 @@
},
"source":"file",
"tail":"true",
- "gen_event_md5":"true",
- "start_position":"beginning"
+ "gen_event_md5":"true"
}
}
\ No newline at end of file