You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2017/11/13 16:57:40 UTC
[23/51] [abbrv] ambari git commit: AMBARI-22364. Log Feeder should be
able to run locally with dockerized dev env by default (oleewere)
AMBARI-22364. Log Feeder should be able to run locally with dockerized dev env by default (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/654404dc
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/654404dc
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/654404dc
Branch: refs/heads/branch-feature-AMBARI-21674
Commit: 654404dc8c545f47bafd8a446b99ecb251d83982
Parents: 063ba36
Author: Oliver Szabo <ol...@gmail.com>
Authored: Tue Nov 7 14:22:38 2017 +0100
Committer: Oliver Szabo <ol...@gmail.com>
Committed: Tue Nov 7 17:08:04 2017 +0100
----------------------------------------------------------------------
.../ambari-logsearch-logfeeder/README.md | 17 +++++-----
.../ambari-logsearch-logfeeder/pom.xml | 4 ++-
.../src/main/resources/log-samples/.gitignore | 4 +++
.../log-samples/logs/service_sample.txt | 3 ++
.../log-samples/shipper-conf/global.config.json | 10 ++++++
.../shipper-conf/input.config-sample.json | 31 ++++++++++++++++++
.../shipper-conf/output.config-sample.json | 34 ++++++++++++++++++++
.../src/main/resources/log4j.xml | 4 +--
.../src/main/resources/logfeeder.properties | 31 ++++++++++--------
9 files changed, 113 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/README.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/README.md b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
index 5a64daf..d2e55f0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/README.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/README.md
@@ -22,19 +22,18 @@ limitations under the License.
Log Feeder is a component of the Log Search service that reads logs, parses them and stores them in Apache Solr for the purpose
of later analysis.
-# Compilation
+## Start locally from maven / IDE
+
+First you need to start every required service (except logfeeder), go to `ambari-logsearch/docker` folder and run:
```bash
-mvn clean compile package
+docker-compose up -d zookeeper solr logsearch
```
-# Deploy
-## Copy to remote
-copy target/logsearch-logfeeder.tgz to host machine
-## Setup environment
+Secondly, if you are planning to run Log Feeder from an IDE, for running the LogFeeder main methoud, you will need to set the working directory to `ambari/ambari-logsearch/ambari-logsearch-logfeeder` and use `--monitor` as a command line argument.
+With Maven, you won't need these steps, just run this command from the ambari-logsearch-logfeeder folder:
+
```bash
-mkdir /opt/logfeeder
-cd /opt/logfeeder
-tar xfz ~/logsearch-logfeeder.tar.gz
+mvn clean package -DskipTests exec:java
```
# Input Configuration
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
index b1b6ece..6b7d94f 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml
@@ -220,7 +220,9 @@
</executions>
<configuration>
<mainClass>org.apache.ambari.logfeeder.LogFeeder</mainClass>
- <!-- <arguments> <argument></argument> </arguments> -->
+ <arguments>
+ <argument>--monitor</argument>
+ </arguments>
</configuration>
</plugin>
<!-- copy-dependencies -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore
new file mode 100644
index 0000000..dfb10d6
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/.gitignore
@@ -0,0 +1,4 @@
+logs/*.log
+shipper-conf/input.config-*.json
+!shipper-conf/input.config-sample.json
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt
new file mode 100644
index 0000000..21048ac
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/logs/service_sample.txt
@@ -0,0 +1,3 @@
+2016-07-13 10:45:49,640 [WARN] Sample log line 1 - warn level
+2016-07-13 10:45:49,640 [ERROR] Sample log line 2 - error level
+2016-07-13 10:45:50,351 [INFO] Sample log line 3 - info level
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json
new file mode 100644
index 0000000..6b8602c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/global.config.json
@@ -0,0 +1,10 @@
+{
+ "global":{
+ "add_fields":{
+ "cluster":"cl1"
+ },
+ "source":"file",
+ "tail":"true",
+ "gen_event_md5":"true"
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
new file mode 100644
index 0000000..4ab2eb2
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json
@@ -0,0 +1,31 @@
+{
+ "input": [
+ {
+ "type": "service_sample",
+ "rowtype": "service",
+ "path": "target/classes/log-samples/logs/service_sample.txt"
+ }
+ ],
+ "filter": [
+ {
+ "filter": "grok",
+ "conditions": {
+ "fields": {
+ "type": [
+ "service_sample"
+ ]
+ }
+ },
+ "log4j_format": "",
+ "multiline_pattern": "^(%{TIMESTAMP_ISO8601:logtime})",
+ "message_pattern": "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values": {
+ "logtime": {
+ "map_date": {
+ "date_pattern": "yyyy-MM-dd HH:mm:ss,SSS"
+ }
+ }
+ }
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json
new file mode 100644
index 0000000..94b44da
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/output.config-sample.json
@@ -0,0 +1,34 @@
+{
+ "output": [
+ {
+ "is_enabled": "true",
+ "comment": "Output to solr for service logs",
+ "destination": "solr",
+ "zk_connect_string": "localhost:2181",
+ "type": "service",
+ "skip_logtime": "true",
+ "conditions": {
+ "fields": {
+ "rowtype": [
+ "service"
+ ]
+ }
+ }
+ },
+ {
+ "comment": "Output to solr for audit records",
+ "is_enabled": "true",
+ "destination": "solr",
+ "zk_connect_string": "localhost:2181",
+ "type": "audit",
+ "skip_logtime": "true",
+ "conditions": {
+ "fields": {
+ "rowtype": [
+ "audit"
+ ]
+ }
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
index 8d5174e..eb20665 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml
@@ -28,7 +28,7 @@
</appender>
<appender name="daily_rolling_file" class="org.apache.log4j.DailyRollingFileAppender">
- <param name="file" value="logs/logsearch-logfeeder.log" />
+ <param name="file" value="target/logs/logsearch-logfeeder.log" />
<param name="datePattern" value="'.'yyyy-MM-dd" />
<param name="append" value="true" />
<layout class="org.apache.log4j.PatternLayout">
@@ -38,7 +38,7 @@
<appender name="rolling_file_json"
class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
- <param name="file" value="logs/logsearch-logfeeder.json" />
+ <param name="file" value="target/logs/logsearch-logfeeder.json" />
<param name="append" value="true" />
<param name="maxFileSize" value="10MB" />
<param name="maxBackupIndex" value="10" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/654404dc/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index 74ea0ef..115778b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -13,19 +13,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-logfeeder.checkpoint.folder=
+cluster.name=cl1
+logfeeder.checkpoint.folder=target/checkpoints
logfeeder.metrics.collector.hosts=
-
-#filter config
-logfeeder.log.filter.enable=false
+logfeeder.config.dir=target/classes/log-samples/shipper-conf/
+logfeeder.config.files=target/classes/log-samples/shipper-conf/global.config.json,\
+ target/classes/log-samples/shipper-conf/input.config-sample.json,\
+ target/classes/log-samples/shipper-conf/output.config-sample.json
+logfeeder.log.filter.enable=true
logfeeder.solr.config.interval=5
-logfeeder.solr.zk_connect_string=
-logfeeder.solr.url=
-
-logfeeder.solr.kerberos.enable=false
-logfeeder.solr.jaas.file=/usr/lib/ambari-logsearch-logfeeder/logsearch_solr_jaas.conf
-
-#logfeeder tmp dir
-logfeeder.tmp.dir=/tmp/$username/logfeeder/
-
logfeeder.solr.core.config.name=history
+logfeeder.solr.zk_connect_string=localhost:2181
+logfeeder.cache.enabled=true
+logfeeder.cache.size=100
+logfeeder.cache.key.field=log_message
+logfeeder.cache.dedup.interval=1000
+logfeeder.cache.last.dedup.enabled=true
+logsearch.config.zk_connect_string=localhost:2181
+logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN
+
+#logfeeder tmp dir
+logfeeder.tmp.dir=target/tmp