You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2016/09/07 20:42:02 UTC
[01/10] ambari git commit: AMBARI-18279. Use the PropertiesFile
python command to create property files instead of jinja templates (Miklos
Gergely via oleewere)
Repository: ambari
Updated Branches:
refs/heads/branch-dev-logsearch 1d76f8fec -> d89101eca
AMBARI-18279. Use the PropertiesFile python command to create property files instead of jinja templates (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0a30febb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0a30febb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0a30febb
Branch: refs/heads/branch-dev-logsearch
Commit: 0a30febbde25c4efb50b46fdbdb8aa12f07d44b2
Parents: 1d76f8f
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Sep 7 22:17:05 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Sep 7 22:17:05 2016 +0200
----------------------------------------------------------------------
.../configuration/logsearch-properties.xml | 6 +-
.../LOGSEARCH/0.5.0/package/scripts/params.py | 125 +++++++++----------
.../0.5.0/package/scripts/setup_logfeeder.py | 10 +-
.../0.5.0/package/scripts/setup_logsearch.py | 7 +-
.../package/templates/logfeeder.properties.j2 | 32 -----
.../package/templates/logsearch.properties.j2 | 72 -----------
.../stacks/2.4/LOGSEARCH/test_logfeeder.py | 18 ++-
.../stacks/2.4/LOGSEARCH/test_logsearch.py | 30 ++++-
.../test/python/stacks/2.4/configs/default.json | 13 +-
ambari-web/app/data/HDP2/site_properties.js | 6 +-
10 files changed, 120 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
index 56ff2d1..e521c8e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logsearch-properties.xml
@@ -133,7 +133,7 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>logsearch.external.auth.enabled</name>
+ <name>logsearch.auth.external_auth.enable</name>
<value>true</value>
<display-name>External authentication</display-name>
<description>Enable external authentication</description>
@@ -143,14 +143,14 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>logsearch.external.auth.host_url</name>
+ <name>logsearch.auth.external_auth.host_url</name>
<value>{ambari_server_auth_host_url}</value>
<display-name>External authentication url - host and port</display-name>
<description>The host and port of the external server used to authenticate</description>
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>logsearch.external.auth.login_url</name>
+ <name>logsearch.auth.external_auth.login_url</name>
<value>/api/v1/users/$USERNAME/privileges?fields=*</value>
<display-name>External authentication url - path and query</display-name>
<description>The path and query of the external server used to authenticate</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 5c3954f..8b0ce75 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -75,9 +75,9 @@ if 'metrics_collector_hosts' in config['clusterHostInfo']:
else:
metrics_collector_hosts = ''
-logsearch_solr_metrics_collector_hosts = format(config['configurations']['logsearch-properties']['logsearch.solr.metrics.collector.hosts'])
-
+#####################################
# Infra Solr configs
+#####################################
infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr')
infra_solr_instance_count = len(config['clusterHostInfo']['infra_solr_hosts'])
infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False)
@@ -108,57 +108,17 @@ if security_enabled:
#####################################
logsearch_dir = '/usr/lib/ambari-logsearch-portal'
-logsearch_collection_service_logs_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.service.logs.numshards']
-logsearch_collection_audit_logs_numshards_config = config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.numshards']
-
-if logsearch_collection_service_logs_numshards_config > 0:
- logsearch_collection_service_logs_numshards = str(logsearch_collection_service_logs_numshards_config)
-else:
- logsearch_collection_service_logs_numshards = format(str(infra_solr_instance_count))
-
-if logsearch_collection_audit_logs_numshards_config > 0:
- logsearch_collection_audit_logs_numshards = str(logsearch_collection_audit_logs_numshards_config)
-else:
- logsearch_collection_audit_logs_numshards = format(str(infra_solr_instance_count))
-
-logsearch_collection_service_logs_replication_factor = str(config['configurations']['logsearch-properties']['logsearch.collection.service.logs.replication.factor'])
-logsearch_collection_audit_logs_replication_factor = str(config['configurations']['logsearch-properties']['logsearch.collection.audit.logs.replication.factor'])
-
-logsearch_solr_collection_service_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.service.logs', 'hadoop_logs')
-logsearch_solr_collection_audit_logs = default('/configurations/logsearch-properties/logsearch.solr.collection.audit.logs','audit_logs')
-
logsearch_service_logs_max_retention = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_max_retention']
logsearch_service_logs_merge_factor = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_merge_factor']
-logsearch_service_logs_fields = config['configurations']['logsearch-properties']['logsearch.service.logs.fields']
-logsearch_service_logs_split_interval_mins = config['configurations']['logsearch-properties']['logsearch.service.logs.split.interval.mins']
logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention']
logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor']
-logsearch_audit_logs_split_interval_mins = config['configurations']['logsearch-properties']['logsearch.audit.logs.split.interval.mins']
-logsearch_logfeeder_include_default_level = default('/configurations/logsearch-properties/logsearch.logfeeder.include.default.level', 'fatal,error,warn')
-
-logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', zookeeper_quorum)
-logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', infra_solr_znode)
+logsearch_solr_audit_logs_zk_node = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_node', infra_solr_znode)
+logsearch_solr_audit_logs_zk_quorum = default('/configurations/logsearch-env/logsearch_solr_audit_logs_zk_quorum', zookeeper_quorum)
logsearch_solr_audit_logs_zk_node = format(logsearch_solr_audit_logs_zk_node)
logsearch_solr_audit_logs_zk_quorum = format(logsearch_solr_audit_logs_zk_quorum)
-# create custom properties - remove defaults
-logsearch_custom_properties = dict(config['configurations']['logsearch-properties'])
-logsearch_custom_properties.pop("logsearch.service.logs.fields", None)
-logsearch_custom_properties.pop("logsearch.audit.logs.split.interval.mins", None)
-logsearch_custom_properties.pop("logsearch.collection.service.logs.replication.factor", None)
-logsearch_custom_properties.pop("logsearch.solr.collection.service.logs", None)
-logsearch_custom_properties.pop("logsearch.solr.metrics.collector.hosts", None)
-logsearch_custom_properties.pop("logsearch.solr.collection.audit.logs", None)
-logsearch_custom_properties.pop("logsearch.logfeeder.include.default.level", None)
-logsearch_custom_properties.pop("logsearch.collection.audit.logs.replication.factor", None)
-logsearch_custom_properties.pop("logsearch.collection.service.logs.numshards", None)
-logsearch_custom_properties.pop("logsearch.service.logs.split.interval.mins", None)
-logsearch_custom_properties.pop("logsearch.collection.audit.logs.numshards", None)
-logsearch_custom_properties.pop("logsearch.external.auth.enabled", None)
-logsearch_custom_properties.pop("logsearch.external.auth.host_url", None)
-logsearch_custom_properties.pop("logsearch.external.auth.login_url", None)
# logsearch-env configs
logsearch_user = config['configurations']['logsearch-env']['logsearch_user']
@@ -215,9 +175,7 @@ hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs')
mapred_user = default('configurations/mapred-env/mapred_user', 'mapred')
yarn_user = default('configurations/yarn-env/yarn_user', 'yarn')
-#####################################
# Logsearch auth configs
-#####################################
logsearch_admin_credential_file = 'logsearch-admin.json'
logsearch_admin_username = default('/configurations/logsearch-admin-json/logsearch_admin_username', "admin")
@@ -236,9 +194,42 @@ if 'ambari_server_host' in config['clusterHostInfo']:
else:
ambari_server_auth_host_url = ''
-logsearch_auth_external_enabled = str(config['configurations']['logsearch-properties']['logsearch.external.auth.enabled']).lower()
-logsearch_auth_external_host_url = format(config['configurations']['logsearch-properties']['logsearch.external.auth.host_url'])
-logsearch_auth_external_login_url = config['configurations']['logsearch-properties']['logsearch.external.auth.login_url']
+# Logsearch propreties
+
+logsearch_properties = dict(config['configurations']['logsearch-properties'])
+
+logsearch_properties['logsearch.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
+logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_audit_logs_zk_quorum + logsearch_solr_audit_logs_zk_node
+
+logsearch_properties['logsearch.solr.collection.history'] = 'history'
+logsearch_properties['logsearch.solr.history.config.name'] = 'history'
+logsearch_properties['logsearch.collection.history.replication.factor'] = '1'
+
+logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts'])
+
+logsearch_properties['logsearch.solr.jmx.port'] = infra_solr_jmx_port
+
+logsearch_properties['logsearch.login.credentials.file'] = logsearch_admin_credential_file
+logsearch_properties['logsearch.auth.file.enable'] = 'true'
+logsearch_properties['logsearch.auth.ldap.enable'] = 'false'
+logsearch_properties['logsearch.auth.simple.enable'] = 'false'
+logsearch_properties['logsearch.roles.allowed'] = 'AMBARI.ADMINISTRATOR'
+logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url'])
+
+logsearch_properties['logsearch.protocol'] = logsearch_ui_protocol
+
+if security_enabled:
+ logfeeder_properties['logsearch.solr.kerberos.enable'] = 'true'
+ logfeeder_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file
+
+
+logsearch_solr_collection_service_logs = logsearch_properties['logsearch.solr.collection.service.logs']
+logsearch_service_logs_split_interval_mins = logsearch_properties['logsearch.service.logs.split.interval.mins']
+logsearch_collection_service_logs_numshards = logsearch_properties['logsearch.collection.service.logs.numshards']
+
+logsearch_solr_collection_audit_logs = logsearch_properties['logsearch.solr.collection.audit.logs']
+logsearch_audit_logs_split_interval_mins = logsearch_properties['logsearch.audit.logs.split.interval.mins']
+logsearch_collection_audit_logs_numshards = logsearch_properties['logsearch.collection.audit.logs.numshards']
#####################################
# Logfeeder configs
@@ -268,29 +259,31 @@ logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfe
logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password']
logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type']
-logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.checkpoint.folder',
- '/etc/ambari-logsearch-logfeeder/conf/checkpoints')
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra',
+ 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn',
+ 'zeppelin', 'zookeeper']
-logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower()
-logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5)
+logfeeder_config_file_names = \
+ ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper']
+default_config_files = ','.join(logfeeder_config_file_names)
-logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in
- logfeeder_supported_services]
+logfeeder_properties = dict(config['configurations']['logfeeder-properties'])
-default_config_files = ','.join(logfeeder_config_file_names)
+logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts'])
+logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files'])
+logfeeder_properties['logfeeder.solr.core.config.name'] = 'history'
+logfeeder_properties['logfeeder.solr.zk_connect_string'] = zookeeper_quorum + infra_solr_znode
-logfeeder_config_files = format(config['configurations']['logfeeder-properties']['logfeeder.config.files'])
-logfeeder_metrics_collector_hosts = format(config['configurations']['logfeeder-properties']['logfeeder.metrics.collector.hosts'])
+if security_enabled:
+ logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true'
+ logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file
-logfeeder_custom_properties = dict(config['configurations']['logfeeder-properties'])
-logfeeder_custom_properties.pop('logfeeder.config.files', None)
-logfeeder_custom_properties.pop('logfeeder.checkpoint.folder', None)
-logfeeder_custom_properties.pop('logfeeder.metrics.collector.hosts', None)
-logfeeder_custom_properties.pop('logfeeder.log.filter.enable', None)
-logfeeder_custom_properties.pop('logfeeder.solr.config.interval', None)
+logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder']
+
+#####################################
+# Smoke command
+#####################################
logsearch_server_hosts = config['clusterHostInfo']['logsearch_server_hosts']
logsearch_server_host = ""
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index d3851d3..688a9b0 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -20,12 +20,12 @@ limitations under the License.
from resource_management.core.resources.system import Directory, File
from resource_management.libraries.functions.format import format
from resource_management.core.source import InlineTemplate, Template
+from resource_management.libraries.resources.properties_file import PropertiesFile
def setup_logfeeder():
import params
- Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir,
- params.logfeeder_checkpoint_folder],
+ Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_checkpoint_folder],
mode=0755,
cd_access='a',
create_parents=True
@@ -43,9 +43,9 @@ def setup_logfeeder():
content=''
)
- File(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
- content=Template("logfeeder.properties.j2"),
- )
+ PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"),
+ properties = params.logfeeder_properties
+ )
File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"),
content=InlineTemplate(params.logfeeder_env_content),
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
index aa89609..8d84093 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logsearch.py
@@ -23,6 +23,7 @@ from resource_management.core.resources.system import Directory, Execute, File
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.decorator import retry
from resource_management.core.source import InlineTemplate, Template
+from resource_management.libraries.resources.properties_file import PropertiesFile
def setup_logsearch():
@@ -51,10 +52,8 @@ def setup_logsearch():
content=''
)
- File(format("{logsearch_server_conf}/logsearch.properties"),
- content=Template("logsearch.properties.j2"),
- owner=params.logsearch_user,
- group=params.user_group
+ PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"),
+ properties=params.logsearch_properties
)
File(format("{logsearch_server_conf}/HadoopServiceConfig.json"),
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
deleted file mode 100644
index 86f0be4..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logfeeder.properties.j2
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-logfeeder.checkpoint.folder={{logfeeder_checkpoint_folder}}
-logfeeder.metrics.collector.hosts={{logfeeder_metrics_collector_hosts}}
-logfeeder.config.files={{logfeeder_config_files}}
-logfeeder.log.filter.enable={{logfeeder_log_filter_enable}}
-logfeeder.solr.config.interval={{logfeeder_solr_config_interval}}
-logfeeder.solr.core.config.name=history
-logfeeder.solr.zk_connect_string={{zookeeper_quorum}}{{infra_solr_znode}}
-
-# Custom properties
-{% for key, value in logfeeder_custom_properties.items() %}
-{{key}}={{value}}
-{% endfor %}
-
-{% if security_enabled -%}
-logfeeder.solr.kerberos.enable=true
-logfeeder.solr.jaas.file={{logfeeder_jaas_file}}
-{% endif %}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
deleted file mode 100644
index 1bcee2d..0000000
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/logsearch.properties.j2
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-logsearch.solr.zk_connect_string={{zookeeper_quorum}}{{infra_solr_znode}}
-
-# Service Logs
-logsearch.solr.collection.service.logs={{logsearch_solr_collection_service_logs}}
-
-logsearch.service.logs.split.interval.mins={{logsearch_service_logs_split_interval_mins}}
-logsearch.collection.service.logs.numshards={{logsearch_collection_service_logs_numshards}}
-logsearch.collection.service.logs.replication.factor={{logsearch_collection_service_logs_replication_factor}}
-
-logsearch.service.logs.fields={{logsearch_service_logs_fields}}
-
-# Audit logs
-logsearch.solr.audit.logs.zk_connect_string={{logsearch_solr_audit_logs_zk_quorum}}{{logsearch_solr_audit_logs_zk_node}}
-logsearch.solr.collection.audit.logs={{logsearch_solr_collection_audit_logs}}
-logsearch.solr.audit.logs.url=
-
-logsearch.audit.logs.split.interval.mins={{logsearch_audit_logs_split_interval_mins}}
-logsearch.collection.audit.logs.numshards={{logsearch_collection_audit_logs_numshards}}
-logsearch.collection.audit.logs.replication.factor={{logsearch_collection_audit_logs_replication_factor}}
-
-# History logs
-logsearch.solr.collection.history=history
-logsearch.solr.history.config.name=history
-logsearch.collection.history.replication.factor=1
-
-# Metrics
-logsearch.solr.metrics.collector.hosts={{logsearch_solr_metrics_collector_hosts}}
-logsearch.solr.jmx.port={{infra_solr_jmx_port}}
-
-# Logfeeder Settings
-
-logsearch.logfeeder.include.default.level={{logsearch_logfeeder_include_default_level}}
-
-# logsearch-admin.json
-logsearch.auth.file.enable=true
-logsearch.login.credentials.file={{logsearch_admin_credential_file}}
-
-logsearch.auth.ldap.enable=false
-logsearch.auth.simple.enable=false
-
-logsearch.auth.external_auth.enable={{logsearch_auth_external_enabled}}
-logsearch.auth.external_auth.host_url={{logsearch_auth_external_host_url}}
-logsearch.auth.external_auth.login_url={{logsearch_auth_external_login_url}}
-logsearch.roles.allowed=AMBARI.ADMINISTRATOR
-
-logsearch.protocol={{logsearch_ui_protocol}}
-
-# Custom properties
-{% for key, value in logsearch_custom_properties.items() %}
-{{key}}={{value}}
-{% endfor %}
-
-{% if security_enabled -%}
-logsearch.solr.kerberos.enable=true
-logsearch.solr.jaas.file={{logsearch_jaas_file}}
-{% endif %}
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index f4dbd63..7f9acba 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -60,8 +60,13 @@ class TestLogFeeder(RMFTestCase):
mode=0644,
content=''
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
- content=Template('logfeeder.properties.j2')
+ self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties',
+ properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints',
+ 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-accumulo.json,input.config-ambari.json,input.config-ams.json,input.config-atlas.json,input.config-falcon.json,input.config-flume.json,input.config-hbase.json,input.config-hdfs.json,input.config-hive.json,input.config-hst.json,input.config-infra.json,input.config-kafka.json,input.config-knox.json,input.config-logsearch.json,input.config-nifi.json,input.config-oozie.json,input.config-ranger.json,input.config-spark.json,input.config-spark2.json,input.config-storm.json,input.config-yarn.json,input.config-zeppelin.json,input.config-zookeeper.json',
+ 'logfeeder.metrics.collector.hosts': '',
+ 'logfeeder.solr.core.config.name': 'history',
+ 'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr'
+ }
)
self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/logfeeder-env.sh',
mode=0755,
@@ -75,11 +80,12 @@ class TestLogFeeder(RMFTestCase):
encoding='utf-8'
)
- logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'infra', 'kafka',
- 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper']
+ logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst',
+ 'infra', 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm',
+ 'yarn', 'zeppelin', 'zookeeper']
- logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag
- in logfeeder_supported_services]
+ logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \
+ ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]
for file_name in logfeeder_config_file_names:
self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name,
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
index 5a2e6e8..0378a94 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py
@@ -70,10 +70,32 @@ class TestLogSearch(RMFTestCase):
mode = 0644,
content = ''
)
- self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
- owner = 'logsearch',
- group='hadoop',
- content = Template('logsearch.properties.j2')
+ self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-portal/conf/logsearch.properties',
+ properties = {'logsearch.audit.logs.split.interval.mins': '1',
+ 'logsearch.auth.external_auth.enabled': 'false',
+ 'logsearch.auth.external_auth.host_url': 'http://c6401.ambari.apache.org:8080',
+ 'logsearch.auth.external_auth.login_url': '/api/v1/users/$USERNAME/privileges?fields=*',
+ 'logsearch.auth.file.enable': 'true',
+ 'logsearch.auth.ldap.enable': 'false',
+ 'logsearch.auth.simple.enable': 'false',
+ 'logsearch.collection.audit.logs.numshards': '10',
+ 'logsearch.collection.audit.logs.replication.factor': '1',
+ 'logsearch.collection.history.replication.factor': '1',
+ 'logsearch.collection.service.logs.numshards': '10',
+ 'logsearch.collection.service.logs.replication.factor': '1',
+ 'logsearch.login.credentials.file': 'logsearch-admin.json',
+ 'logsearch.protocol': 'http',
+ 'logsearch.roles.allowed': 'AMBARI.ADMINISTRATOR',
+ 'logsearch.service.logs.split.interval.mins': '1',
+ 'logsearch.solr.audit.logs.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr',
+ 'logsearch.solr.collection.audit.logs': 'audit_logs',
+ 'logsearch.solr.collection.history': 'history',
+ 'logsearch.solr.collection.service.logs': 'hadoop_logs',
+ 'logsearch.solr.history.config.name': 'history',
+ 'logsearch.solr.metrics.collector.hosts': '',
+ 'logsearch.solr.jmx.port': '1',
+ 'logsearch.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr'
+ }
)
self.assertResourceCalled('File', '/etc/ambari-logsearch-portal/conf/HadoopServiceConfig.json',
owner = 'logsearch',
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-server/src/test/python/stacks/2.4/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json
index a016ce0..8ac6fa7 100644
--- a/ambari-server/src/test/python/stacks/2.4/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json
@@ -250,6 +250,7 @@
"infra_solr_datadir": "/opt/ambari_infra_solr/data",
"infra_solr_log_dir": "/var/log/ambari-infra-solr",
"infra_solr_client_log_dir": "/var/log/ambari-infra-solr-client",
+ "infra_solr_jmx_port" : "1",
"content": "# By default the script will use JAVA_HOME to determine which java\n# to use, but you can set a specific path for Solr to use without\n# affecting other Java applications on your server/workstation.\nSOLR_JAVA_HOME={{java64_home}}\n\n# Increase Java Min/Max Heap as needed to support your indexing / query needs\nSOLR_JAVA_MEM=\"-Xms{{solr_min_mem}} -Xmx{{solr_max_mem}}\"\n\n# Enable verbose GC logging\nGC_LOG_OPTS=\"-verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails \\\n-XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime\"\n\n# These GC settings have shown to work well for a number of common Solr workloads\nGC_TUNE=\"-XX:NewRatio=3 \\\n-XX:SurvivorRatio=4 \\\n-XX:TargetSurvivorRatio=90 \\\n-XX:MaxTenuringThreshold=8 \\\n-XX:+UseConcMarkSweepGC \\\n-XX:+UseParNewGC \\\n-XX:ConcGCThreads=4 -XX:ParallelGCThreads=4 \\\n-XX:+CMSScavengeBeforeRemark \\\n-XX:PretenureSizeThreshold=64m \\\n-XX:+UseCMSInitiatingOc
cupancyOnly \\\n-XX:CMSInitiatingOccupancyFraction=50 \\\n-XX:CMSMaxAbortablePrecleanTime=6000 \\\n-XX:+CMSParallelRemarkEnabled \\\n-XX:+ParallelRefProcEnabled\"\n\n# Set the ZooKeeper connection string if using an external ZooKeeper ensemble\n# e.g. host1:2181,host2:2181/chroot\n# Leave empty if not using SolrCloud\nZK_HOST=\"{{zookeeper_quorum}}{{solr_znode}}\"\n\n# Set the ZooKeeper client timeout (for SolrCloud mode)\nZK_CLIENT_TIMEOUT=\"60000\"\n\n# By default the start script uses \"localhost\"; override the hostname here\n# for production SolrCloud environments to control the hostname exposed to cluster state\n#SOLR_HOST=\"192.168.1.1\"\n\n# By default the start script uses UTC; override the timezone if needed\n#SOLR_TIMEZONE=\"UTC\"\n\n# Set to true to activate the JMX RMI connector to allow remote JMX client applications\n# to monitor the JVM hosting Solr; set to \"false\" to disable that behavior\n# (false is recommended in production environments)\nENABLE_REMOTE_JMX_OPTS
=\"false\"\n\n# The script will use SOLR_PORT+10000 for the RMI_PORT or you can set it here\n# RMI_PORT=18983\n\n# Anything you add to the SOLR_OPTS variable will be included in the java\n# start command line as-is, in ADDITION to other options. If you specify the\n# -a option on start script, those options will be appended as well. Examples:\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000\"\n#SOLR_OPTS=\"$SOLR_OPTS -Dsolr.clustering.enabled=true\"\n\n# Location where the bin/solr script will save PID files for running instances\n# If not set, the script will create PID files in $SOLR_TIP/bin\nSOLR_PID_DIR={{solr_piddir}}\n\n# Path to a directory where Solr creates index files, the specified directory\n# must contain a solr.xml; by default, Solr will use server/solr\nSOLR_HOME={{infra_solr_datadir}}\n\n# Solr provides a default Log4J configuration properties file in server/resources\n# however, you may want to cu
stomize the log settings and file appender location\n# so you can point the script to use a different log4j.properties file\nLOG4J_PROPS={{logsearch_solr_datadir}}/resources/log4j.properties\n\n# Location where Solr should write logs to; should agree with the file appender\n# settings in server/resources/log4j.properties\nSOLR_LOGS_DIR={{solr_log_dir}}\n\n# Sets the port Solr binds to, default is 8983\nSOLR_PORT={{solr_port}}\n\n# Be sure to update the paths to the correct keystore for your environment\n{% if logsearch_solr_ssl_enabled %}\nSOLR_SSL_KEY_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_KEY_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_TRUST_STORE={{logsearch_solr_keystore_location}}\nSOLR_SSL_TRUST_STORE_PASSWORD={{logsearch_solr_keystore_password}}\nSOLR_SSL_NEED_CLIENT_AUTH=false\nSOLR_SSL_WANT_CLIENT_AUTH=false\n{% endif %}\n\n# Uncomment to set a specific SSL port (-Djetty.ssl.port=N); if not set\n# and you are using SSL, then the start script
will use SOLR_PORT for the SSL port\n#SOLR_SSL_PORT="
},
"infra-solr-xml": {
@@ -280,17 +281,21 @@
"content": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n<!--\n Licensed to the Apache Software Foundation (ASF) under one or more\n contributor license agreements. See the NOTICE file distributed with\n this work for additional information regarding copyright ownership.\n The ASF licenses this file to You under the Apache License, Version 2.0\n (the \"License\"); you may not use this file except in compliance with\n the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n-->\n<!DOCTYPE log4j:configuration SYSTEM \"log4j.dtd\">\n<log4j:configuration xmlns:log4j=\"http://jakarta.
apache.org/log4j/\">\n <appender name=\"console\" class=\"org.apache.log4j.ConsoleAppender\">\n <param name=\"Target\" value=\"System.out\" />\n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n\n <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\"> \n <param name=\"file\" value=\"{{logsearch_log_dir}}/logsearch.log\" /> \n <param name=\"append\" value=\"true\" /> \n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\"> \n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"/> \n </layout> \n </appender> \n\n <appender name=\"performance_analyzer\" class=\"org.apache.log4j.RollingFileAppender\"
>\n <param name=\"file\" value=\"{{logsearch_log_dir}}/logsearch-performance.log\" />\n <param name=\"Threshold\" value=\"info\" />\n <param name=\"append\" value=\"true\" />\n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n \n <logger name=\"org.apache.ambari.logsearch.perfomance\" additivity=\"false\">\n <appender-ref ref=\"performance_analyzer\" />\n </logger>\n\n <category name=\"org.apache.ambari.logsearch\" additivity=\"false\">\n <priority value=\"info\" />\n <appender-ref ref=\"rolling_file\" />\n </category>\n\n <root>\n <priority value=\"warn\" />\n <appender-ref ref=\"rolling_file\" />\n </root>\
n</log4j:configuration>"
},
"logsearch-properties": {
+ "logsearch.solr.collection.service.logs" : "hadoop_logs",
+ "logsearch.solr.collection.audit.logs" : "audit_logs",
+ "logsearch.service.logs.split.interval.mins": "1",
+ "logsearch.audit.logs.split.interval.mins" : "1",
"logsearch.collection.service.logs.numshards": "10",
"logsearch.collection.service.logs.replication.factor": "1",
"logsearch.collection.audit.logs.numshards": "10",
"logsearch.collection.audit.logs.replication.factor": "1",
- "logsearch.app.max.memory": "1g",
"logsearch.solr.metrics.collector.hosts" : "{metrics_collector_hosts}",
- "logsearch.external.auth.enabled" : "false",
- "logsearch.external.auth.host_url" : "{ambari_server_auth_host_url}",
- "logsearch.external.auth.login_url" : "/api/v1/users/$USERNAME/privileges?fields=*"
+ "logsearch.auth.external_auth.enabled" : "false",
+ "logsearch.auth.external_auth.host_url" : "{ambari_server_auth_host_url}",
+ "logsearch.auth.external_auth.login_url" : "/api/v1/users/$USERNAME/privileges?fields=*"
},
"logfeeder-properties": {
+ "logfeeder.checkpoint.folder" : "/etc/ambari-logsearch-logfeeder/conf/checkpoints",
"logfeeder.metrics.collector.hosts" : "{metrics_collector_hosts}",
"logfeeder.config.files" : "{default_config_files}"
},
http://git-wip-us.apache.org/repos/asf/ambari/blob/0a30febb/ambari-web/app/data/HDP2/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2/site_properties.js b/ambari-web/app/data/HDP2/site_properties.js
index 1fad1d6..9311e75 100644
--- a/ambari-web/app/data/HDP2/site_properties.js
+++ b/ambari-web/app/data/HDP2/site_properties.js
@@ -2215,21 +2215,21 @@ var hdp2properties = [
"index": 6
},
{
- "name": "logsearch.external.auth.enabled",
+ "name": "logsearch.auth.external_auth.enable",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
"index": 7
},
{
- "name": "logsearch.external.auth.host_url",
+ "name": "logsearch.auth.external_auth.host_url",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
"index": 8
},
{
- "name": "logsearch.external.auth.login_url",
+ "name": "logsearch.auth.external_auth.login_url",
"serviceName": "LOGSEARCH",
"filename": "logsearch-properties.xml",
"category": "Advanced logsearch-properties",
[02/10] ambari git commit: AMBARI-18246. Clean up Log Feeder (Miklos
Gergely via oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
new file mode 100644
index 0000000..368a930
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputManagerTest.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.logfeeder.input;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.junit.Test;
+
+public class InputManagerTest {
+
+ @Test
+ public void testInputManager_addAndRemoveInputs() {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+ Input input4 = strictMock(Input.class);
+
+ expect(input3.getShortDescription()).andReturn("").times(2);
+ expect(input4.getShortDescription()).andReturn("").once();
+
+ replay(input1, input2, input3, input4);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.removeInput(input3);
+ manager.removeInput(input4);
+
+ verify(input1, input2, input3, input4);
+
+ List<Input> inputList = manager.getInputList();
+ assertEquals(inputList.size(), 2);
+ assertEquals(inputList.get(0), input1);
+ assertEquals(inputList.get(1), input2);
+ }
+
+ @Test
+ public void testInputManager_init() throws Exception {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ input1.init(); expectLastCall();
+ input2.init(); expectLastCall();
+ input3.init(); expectLastCall();
+
+ expect(input1.isTail()).andReturn(false);
+ expect(input2.isTail()).andReturn(false);
+ expect(input3.isTail()).andReturn(false);
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.init();
+
+ verify(input1, input2, input3);
+ }
+
+ @Test
+ public void testInputManager_monitor() throws Exception {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ expect(input1.isReady()).andReturn(true);
+ expect(input2.isReady()).andReturn(true);
+ expect(input3.isReady()).andReturn(false);
+
+ expect(input1.monitor()).andReturn(false);
+ expect(input2.monitor()).andReturn(false);
+ expect(input3.isTail()).andReturn(false);
+ expect(input3.getShortDescription()).andReturn("").once();
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.monitor();
+
+ verify(input1, input2, input3);
+ }
+
+
+ @Test
+ public void testInputManager_addMetricsContainers() throws Exception {
+ List<MetricData> metrics = new ArrayList<MetricData>();
+
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ input1.addMetricsContainers(metrics); expectLastCall();
+ input2.addMetricsContainers(metrics); expectLastCall();
+ input3.addMetricsContainers(metrics); expectLastCall();
+
+ expect(input1.isReady()).andReturn(true);
+ expect(input2.isReady()).andReturn(true);
+ expect(input3.isReady()).andReturn(false);
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.addMetricsContainers(metrics);
+
+ verify(input1, input2, input3);
+ }
+
+ @Test
+ public void testInputManager_logStat() throws Exception {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ input1.logStat(); expectLastCall();
+ input2.logStat(); expectLastCall();
+ input3.logStat(); expectLastCall();
+
+ expect(input1.isReady()).andReturn(true);
+ expect(input2.isReady()).andReturn(true);
+ expect(input3.isReady()).andReturn(false);
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.logStats();
+
+ verify(input1, input2, input3);
+ }
+
+ @Test
+ public void testInputManagr_waitOnAllInputs() throws Exception {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ Thread mockThread = strictMock(Thread.class);
+
+ expect(input1.getThread()).andReturn(null);
+ expect(input2.getThread()).andReturn(null);
+ expect(input3.getThread()).andReturn(mockThread);
+
+ mockThread.join(); expectLastCall();
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.waitOnAllInputs();
+
+ verify(input1, input2, input3);
+ }
+
+ @Test
+ public void testInputManager_checkInAll() throws Exception {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ input1.lastCheckIn(); expectLastCall();
+ input2.lastCheckIn(); expectLastCall();
+ input3.lastCheckIn(); expectLastCall();
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.checkInAll();
+
+ verify(input1, input2, input3);
+ }
+
+ @Test
+ public void testInputManager_close() throws Exception {
+ Input input1 = strictMock(Input.class);
+ Input input2 = strictMock(Input.class);
+ Input input3 = strictMock(Input.class);
+
+ input1.setDrain(true); expectLastCall();
+ input2.setDrain(true); expectLastCall();
+ input3.setDrain(true); expectLastCall();
+
+ expect(input1.isClosed()).andReturn(true);
+ expect(input2.isClosed()).andReturn(true);
+ expect(input3.isClosed()).andReturn(true);
+
+ replay(input1, input2, input3);
+
+ InputManager manager = new InputManager();
+ manager.add(input1);
+ manager.add(input2);
+ manager.add(input3);
+
+ manager.close();
+
+ verify(input1, input2, input3);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java
new file mode 100644
index 0000000..02ffd47
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.logfeeder.logconfig;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
+
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class LogConfigHandlerTest {
+
+ private static LogConfigFetcher mockFetcher;
+
+ private static final Map<String, Object> CONFIG_MAP = new HashMap<>();
+ static {
+ CONFIG_MAP.put("jsons",
+ "{'filter':{" +
+ "'configured_log_file':{" +
+ "'label':'configured_log_file'," +
+ "'hosts':[]," +
+ "'defaultLevels':['FATAL','ERROR','WARN','INFO']," +
+ "'overrideLevels':[]}," +
+ "'configured_log_file2':{" +
+ "'label':'configured_log_file2'," +
+ "'hosts':['host1']," +
+ "'defaultLevels':['FATAL','ERROR','WARN','INFO']," +
+ "'overrideLevels':['FATAL','ERROR','WARN','INFO','DEBUG','TRACE']," +
+ "'expiryTime':'3000-01-01T00:00:00.000Z'}," +
+ "'configured_log_file3':{" +
+ "'label':'configured_log_file3'," +
+ "'hosts':['host1']," +
+ "'defaultLevels':['FATAL','ERROR','WARN','INFO']," +
+ "'overrideLevels':['FATAL','ERROR','WARN','INFO','DEBUG','TRACE']," +
+ "'expiryTime':'1000-01-01T00:00:00.000Z'}" +
+ "}}");
+ }
+
+ @BeforeClass
+ public static void init() throws Exception {
+ mockFetcher = strictMock(LogConfigFetcher.class);
+ Field f = LogConfigFetcher.class.getDeclaredField("instance");
+ f.setAccessible(true);
+ f.set(null, mockFetcher);
+ expect(mockFetcher.getConfigDoc()).andReturn(CONFIG_MAP).anyTimes();
+ replay(mockFetcher);
+
+ LogFeederUtil.loadProperties("logfeeder.properties", null);
+ LogConfigHandler.handleConfig();
+ Thread.sleep(1000);
+ }
+
+ @Test
+ public void testLogConfigHandler_emptyDataAllowed() throws Exception {
+ assertTrue(FilterLogData.INSTANCE.isAllowed((String)null));
+ assertTrue(FilterLogData.INSTANCE.isAllowed(""));
+ assertTrue(FilterLogData.INSTANCE.isAllowed(Collections.<String, Object> emptyMap()));
+ }
+
+ @Test
+ public void testLogConfigHandler_notConfiguredLogAllowed() throws Exception {
+ assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'not_configured_log_file', 'level':'INFO'}"));
+ }
+
+ @Test
+ public void testLogConfigHandler_configuredDataAllow() throws Exception {
+ assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file', 'level':'INFO'}"));
+ }
+
+ @Test
+ public void testLogConfigHandler_configuredDataDontAllow() throws Exception {
+ assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file', 'level':'DEBUG'}"));
+ }
+
+ @Test
+ public void testLogConfigHandler_overridenConfiguredData() throws Exception {
+ assertTrue(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file2', 'level':'DEBUG'}"));
+ }
+
+ @Test
+ public void testLogConfigHandler_overridenConfiguredDataDifferentHost() throws Exception {
+ assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host2', 'type':'configured_log_file2', 'level':'DEBUG'}"));
+ }
+
+ @Test
+ public void testLogConfigHandler_overridenConfiguredDataExpired() throws Exception {
+ assertFalse(FilterLogData.INSTANCE.isAllowed("{'host':'host1', 'type':'configured_log_file3', 'level':'DEBUG'}"));
+ }
+
+ @AfterClass
+ public static void finish() {
+ verify(mockFetcher);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
index 301dea9..667c9ff 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -24,7 +24,6 @@ import java.util.Date;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.log4j.Logger;
import org.junit.Test;
@@ -61,7 +60,7 @@ public class MapperDateTest {
LOG.info("testMapperDate_pattern()");
Map<String, Object> mapConfigs = new HashMap<>();
- mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
+ mapConfigs.put("target_date_pattern", "yyyy-MM-dd HH:mm:ss.SSS");
MapperDate mapperDate = new MapperDate();
assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -70,7 +69,7 @@ public class MapperDateTest {
String dateString = "2016-04-08 15:55:23.548";
Object mappedValue = mapperDate.apply(jsonObj, dateString);
- Date d = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).parse(dateString);
+ Date d = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(dateString);
assertEquals("Value wasn't matched properly", d, mappedValue);
assertEquals("Value wasn't put into jsonObj", d, jsonObj.remove("someField"));
@@ -130,7 +129,7 @@ public class MapperDateTest {
LOG.info("testMapperDate_invalidDateStringValue()");
Map<String, Object> mapConfigs = new HashMap<>();
- mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
+ mapConfigs.put("target_date_pattern", "yyyy-MM-dd HH:mm:ss.SSS");
MapperDate mapperDate = new MapperDate();
assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs));
@@ -149,7 +148,7 @@ public class MapperDateTest {
String fieldName = "logtime";
Calendar currentCalendar = Calendar.getInstance();
Map<String, Object> mapConfigs = new HashMap<>();
- mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
+ mapConfigs.put("target_date_pattern", "yyyy-MM-dd HH:mm:ss.SSS");
String srcDatePattern ="MMM dd HH:mm:ss";
mapConfigs.put("src_date_pattern", srcDatePattern);
MapperDate mapperDate = new MapperDate();
@@ -160,7 +159,7 @@ public class MapperDateTest {
nextMonthCalendar.set(Calendar.MONTH, currentCalendar.get(Calendar.MONTH)+1 );
String inputDateStr = new SimpleDateFormat("MMM").format(nextMonthCalendar.getTime()) + " 01 12:01:45";
Object mappedValue = mapperDate.apply(jsonObj, inputDateStr);
- Date mappedDateValue = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).parse(mappedValue.toString());
+ Date mappedDateValue = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(mappedValue.toString());
String mappedDateValueStr = new SimpleDateFormat(srcDatePattern).format(mappedDateValue);
assertEquals(Date.class, mappedDateValue.getClass());
@@ -179,7 +178,7 @@ public class MapperDateTest {
String fieldName = "logtime";
Calendar currentCalendar = Calendar.getInstance();
Map<String, Object> mapConfigs = new HashMap<>();
- mapConfigs.put("target_date_pattern", LogFeederUtil.DATE_FORMAT);
+ mapConfigs.put("target_date_pattern", "yyyy-MM-dd HH:mm:ss.SSS");
String srcDatePattern ="MMM dd HH:mm:ss";
mapConfigs.put("src_date_pattern", srcDatePattern);
MapperDate mapperDate = new MapperDate();
@@ -187,7 +186,7 @@ public class MapperDateTest {
Map<String, Object> jsonObj = new HashMap<>();
String inputDateStr = new SimpleDateFormat("MMM").format(currentCalendar.getTime()) + " 01 12:01:45";
Object mappedValue = mapperDate.apply(jsonObj, inputDateStr);
- Date mappedDateValue = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).parse(mappedValue.toString());
+ Date mappedDateValue = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").parse(mappedValue.toString());
String mappedDateValueStr = new SimpleDateFormat(srcDatePattern).format(mappedDateValue);
assertEquals(Date.class, mappedDateValue.getClass());
int expectedLogYear = currentCalendar.get(Calendar.YEAR);
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
index 6edf766..8ecaad1 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
index df84b8e..fce4308 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java
new file mode 100644
index 0000000..8ee6d00
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/metrics/MetrcisManagerTest.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.logfeeder.metrics;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
+import org.easymock.Capture;
+import org.easymock.CaptureType;
+import org.easymock.EasyMock;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.List;
+import java.util.TreeMap;
+
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class MetrcisManagerTest {
+
+ private MetricsManager manager;
+ private LogFeederAMSClient mockClient;
+ private Capture<TimelineMetrics> capture;
+
+ @BeforeClass
+ public static void loadProperties() throws Exception {
+ LogFeederUtil.loadProperties("logfeeder.properties", null);
+ }
+
+ @Before
+ public void init() throws Exception {
+ manager = new MetricsManager();
+ manager.init();
+
+ mockClient = strictMock(LogFeederAMSClient.class);
+ Field f = MetricsManager.class.getDeclaredField("amsClient");
+ f.setAccessible(true);
+ f.set(manager, mockClient);
+
+ capture = EasyMock.newCapture(CaptureType.FIRST);
+ mockClient.emitMetrics(EasyMock.capture(capture));
+ EasyMock.expectLastCall().andReturn(true).once();
+
+ replay(mockClient);
+ }
+
+ @Test
+ public void testMetricManager_pointInTime() throws Exception {
+ MetricData metricCount1 = new MetricData("metric1", true);
+ metricCount1.value = 123;
+ metricCount1.prevPublishValue = 0;
+ metricCount1.publishCount = 0;
+
+ manager.useMetrics(Arrays.asList(metricCount1));
+
+ verify(mockClient);
+
+ TimelineMetrics metrics = capture.getValue();
+ List<TimelineMetric> metricList = metrics.getMetrics();
+ assertEquals(metricList.size(), 1);
+
+ TimelineMetric metric = metricList.get(0);
+ assertEquals(metric.getHostName(), "test_host_name");
+ assertEquals(metric.getAppId(), "logfeeder");
+ assertEquals(metric.getMetricName(), "metric1");
+ assertEquals(metric.getType(), "Long");
+
+ TreeMap<Long, Double> values = metric.getMetricValues();
+ assertEquals(values.size(), 1);
+ assertEquals(values.firstEntry().getValue(), Double.valueOf(123.0));
+ }
+
+ @Test
+ public void testMetricManager_notPointInTime() throws Exception {
+ MetricData metricCount1 = new MetricData("metric1", false);
+ metricCount1.value = 123;
+ metricCount1.prevPublishValue = 0;
+ metricCount1.publishCount = 0;
+
+ MetricData metricCount2 = new MetricData("metric1", false);
+ metricCount2.value = 123;
+ metricCount2.prevPublishValue = 100;
+ metricCount2.publishCount = 0;
+
+ MetricData metricCount3 = new MetricData("metric1", false); // not included due to decrease of count
+ metricCount3.value = 99;
+ metricCount3.prevPublishValue = 100;
+ metricCount3.publishCount = 1;
+
+ manager.useMetrics(Arrays.asList(metricCount1, metricCount2, metricCount3));
+
+ verify(mockClient);
+
+ TimelineMetrics metrics = capture.getValue();
+ List<TimelineMetric> metricList = metrics.getMetrics();
+ assertEquals(metricList.size(), 1);
+
+ TimelineMetric metric = metricList.get(0);
+ assertEquals(metric.getHostName(), "test_host_name");
+ assertEquals(metric.getAppId(), "logfeeder");
+ assertEquals(metric.getMetricName(), "metric1");
+ assertEquals(metric.getType(), "Long");
+
+ TreeMap<Long, Double> values = metric.getMetricValues();
+ assertEquals(values.size(), 1);
+ assertEquals(values.firstEntry().getValue(), Double.valueOf(146.0));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
index a7db3f8..38d4b8b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputKafkaTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -85,8 +85,7 @@ public class OutputKafkaTest {
EasyMock.replay(mockKafkaProducer);
for (int i = 0; i < 10; i++) {
- InputMarker inputMarker = new InputMarker();
- inputMarker.input = EasyMock.mock(Input.class);
+ InputMarker inputMarker = new InputMarker(EasyMock.mock(Input.class), null, 0);
outputKafka.write("value" + i, inputMarker);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
new file mode 100644
index 0000000..e103346
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java
@@ -0,0 +1,256 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.logfeeder.output;
+
+import static org.easymock.EasyMock.*;
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.junit.Test;
+
+public class OutputManagerTest {
+
+ @Test
+ public void testOutputManager_addAndRemoveOutputs() {
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+ Output output4 = strictMock(Output.class);
+
+ replay(output1, output2, output3, output4);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.retainUsedOutputs(Arrays.asList(output1, output2, output4));
+
+ verify(output1, output2, output3, output4);
+
+ List<Output> outputs = manager.getOutputs();
+ assertEquals(outputs.size(), 2);
+ assertEquals(outputs.get(0), output1);
+ assertEquals(outputs.get(1), output2);
+ }
+
+ @Test
+ public void testOutputManager_init() throws Exception {
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ output1.init(); expectLastCall();
+ output2.init(); expectLastCall();
+ output3.init(); expectLastCall();
+
+ replay(output1, output2, output3);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.init();
+
+ verify(output1, output2, output3);
+ }
+
+ @Test
+ public void testOutputManager_write() throws Exception {
+ Map<String, Object> jsonObj = new HashMap<>();
+ jsonObj.put("type", "testType");
+ jsonObj.put("path", "testPath");
+ jsonObj.put("host", "testHost");
+ jsonObj.put("ip", "testIp");
+ jsonObj.put("level", "testLevel");
+ jsonObj.put("id", "testId");
+
+ Input mockInput = strictMock(Input.class);
+ InputMarker inputMarker = new InputMarker(mockInput, null, 0);
+
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ expect(mockInput.getContextFields()).andReturn(Collections.<String, String> emptyMap());
+ expect(mockInput.isUseEventMD5()).andReturn(false);
+ expect(mockInput.isGenEventMD5()).andReturn(false);
+ expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
+
+ output1.write(jsonObj, inputMarker); expectLastCall();
+ output2.write(jsonObj, inputMarker); expectLastCall();
+ output3.write(jsonObj, inputMarker); expectLastCall();
+
+ replay(output1, output2, output3, mockInput);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.write(jsonObj, inputMarker);
+
+ verify(output1, output2, output3, mockInput);
+ }
+
+ @Test
+ public void testOutputManager_write2() throws Exception {
+ String jsonString = "{}";
+
+ Input mockInput = strictMock(Input.class);
+ InputMarker inputMarker = new InputMarker(mockInput, null, 0);
+
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
+
+ output1.write(jsonString, inputMarker); expectLastCall();
+ output2.write(jsonString, inputMarker); expectLastCall();
+ output3.write(jsonString, inputMarker); expectLastCall();
+
+ replay(output1, output2, output3, mockInput);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.write(jsonString, inputMarker);
+
+ verify(output1, output2, output3, mockInput);
+ }
+
+ @Test
+ public void testOutputManager_addMetricsContainers() throws Exception {
+ List<MetricData> metrics = new ArrayList<MetricData>();
+
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ output1.addMetricsContainers(metrics); expectLastCall();
+ output2.addMetricsContainers(metrics); expectLastCall();
+ output3.addMetricsContainers(metrics); expectLastCall();
+
+ replay(output1, output2, output3);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.addMetricsContainers(metrics);
+
+ verify(output1, output2, output3);
+ }
+
+ @Test
+ public void testOutputManager_logStat() throws Exception {
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ output1.logStat(); expectLastCall();
+ output2.logStat(); expectLastCall();
+ output3.logStat(); expectLastCall();
+
+ replay(output1, output2, output3);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.logStats();
+
+ verify(output1, output2, output3);
+ }
+
+ @Test
+ public void testOutputManager_copyFile() throws Exception {
+ File f = new File("");
+
+ Input mockInput = strictMock(Input.class);
+ InputMarker inputMarker = new InputMarker(mockInput, null, 0);
+
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3));
+
+ output1.copyFile(f, inputMarker); expectLastCall();
+ output2.copyFile(f, inputMarker); expectLastCall();
+ output3.copyFile(f, inputMarker); expectLastCall();
+
+ replay(output1, output2, output3, mockInput);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.copyFile(f, inputMarker);
+
+ verify(output1, output2, output3, mockInput);
+ }
+
+ @Test
+ public void testOutputManager_close() throws Exception {
+ Output output1 = strictMock(Output.class);
+ Output output2 = strictMock(Output.class);
+ Output output3 = strictMock(Output.class);
+
+ output1.setDrain(true); expectLastCall();
+ output2.setDrain(true); expectLastCall();
+ output3.setDrain(true); expectLastCall();
+
+ output1.close(); expectLastCall();
+ output2.close(); expectLastCall();
+ output3.close(); expectLastCall();
+
+ expect(output1.isClosed()).andReturn(true);
+ expect(output2.isClosed()).andReturn(true);
+ expect(output3.isClosed()).andReturn(true);
+
+ replay(output1, output2, output3);
+
+ OutputManager manager = new OutputManager();
+ manager.add(output1);
+ manager.add(output2);
+ manager.add(output3);
+
+ manager.close();
+
+ verify(output1, output2, output3);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
index 20a4f1f..1872135 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -69,15 +69,14 @@ public class OutputS3FileTest {
@Test
public void shouldSpoolLogEventToNewSpooler() throws Exception {
- InputMarker inputMarker = mock(InputMarker.class);
Input input = mock(Input.class);
- inputMarker.input = input;
+ InputMarker inputMarker = new InputMarker(input, null, 0);
expect(input.getFilePath()).andReturn("/var/log/hdfs-namenode.log");
expect(input.getStringValue(OutputS3File.INPUT_ATTRIBUTE_TYPE)).andReturn("hdfs-namenode");
final LogSpooler spooler = mock(LogSpooler.class);
spooler.add("log event block");
final S3Uploader s3Uploader = mock(S3Uploader.class);
- replay(input, inputMarker, spooler, s3Uploader);
+ replay(input, spooler, s3Uploader);
OutputS3File outputS3File = new OutputS3File() {
@Override
@@ -98,16 +97,15 @@ public class OutputS3FileTest {
@Test
public void shouldReuseSpoolerForSamePath() throws Exception {
- InputMarker inputMarker = mock(InputMarker.class);
Input input = mock(Input.class);
- inputMarker.input = input;
+ InputMarker inputMarker = new InputMarker(input, null, 0);
expect(input.getFilePath()).andReturn("/var/log/hdfs-namenode.log");
expect(input.getStringValue(OutputS3File.INPUT_ATTRIBUTE_TYPE)).andReturn("hdfs-namenode");
final LogSpooler spooler = mock(LogSpooler.class);
spooler.add("log event block1");
spooler.add("log event block2");
final S3Uploader s3Uploader = mock(S3Uploader.class);
- replay(input, inputMarker, spooler, s3Uploader);
+ replay(input, spooler, s3Uploader);
OutputS3File outputS3File = new OutputS3File() {
private boolean firstCallComplete;
@@ -169,16 +167,15 @@ public class OutputS3FileTest {
@Test
public void shouldUploadFileOnRollover() throws Exception {
- InputMarker inputMarker = mock(InputMarker.class);
Input input = mock(Input.class);
- inputMarker.input = input;
+ InputMarker inputMarker = new InputMarker(input, null, 0);
expect(input.getFilePath()).andReturn("/var/log/hdfs-namenode.log");
expect(input.getStringValue(OutputS3File.INPUT_ATTRIBUTE_TYPE)).andReturn("hdfs-namenode");
final LogSpooler spooler = mock(LogSpooler.class);
spooler.add("log event block1");
final S3Uploader s3Uploader = mock(S3Uploader.class);
s3Uploader.addFileForUpload("/var/ambari-logsearch/logfeeder/hdfs-namenode.log.gz");
- replay(input, inputMarker, spooler, s3Uploader);
+ replay(input, spooler, s3Uploader);
OutputS3File outputS3File = new OutputS3File() {
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
index 33bb33f..8985110 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputSolrTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -97,8 +97,7 @@ public class OutputSolrTest {
jsonObj.put("name" + ++count, "value" + ++count);
jsonObj.put("id", ++count);
- InputMarker inputMarker = new InputMarker();
- inputMarker.input = EasyMock.mock(Input.class);
+ InputMarker inputMarker = new InputMarker(EasyMock.mock(Input.class), null, 0);
outputSolr.write(jsonObj, inputMarker);
SolrInputDocument doc = new SolrInputDocument();
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3LogPathResolverTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3LogPathResolverTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3LogPathResolverTest.java
index cc6da56..d1376c4 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3LogPathResolverTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3LogPathResolverTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3UploaderTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3UploaderTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3UploaderTest.java
index c64e0c5..5477f5c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3UploaderTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/S3UploaderTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,7 +18,6 @@
package org.apache.ambari.logfeeder.output;
-import org.apache.ambari.logfeeder.util.S3Util;
import org.junit.Test;
import java.io.File;
@@ -46,22 +45,20 @@ public class S3UploaderTest {
Map<String, Object> configs = setupS3Configs();
S3OutputConfiguration s3OutputConfiguration = new S3OutputConfiguration(configs);
- S3Util s3Util = mock(S3Util.class);
- String s3Key = String.format("%s/%s/%s.%s", TEST_PATH, LOG_TYPE, fileName, GZ);
- s3Util.uploadFileTos3(TEST_BUCKET, s3Key, compressedFile, ACCESS_KEY_VALUE, SECRET_KEY_VALUE);
expect(compressedFile.delete()).andReturn(true);
expect(fileToUpload.delete()).andReturn(true);
- replay(fileToUpload, compressedFile, s3Util);
+ replay(fileToUpload, compressedFile);
- S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, s3Util, true, LOG_TYPE) {
+ S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, true, LOG_TYPE) {
@Override
protected File createCompressedFileForUpload(File fileToUpload, String compressionAlgo) {
return compressedFile;
}
+ protected void uploadFileToS3(String bucketName, String s3Key, File localFile, String accessKey, String secretKey) {
+ }
};
String resolvedPath = s3Uploader.uploadFile(fileToUpload, LOG_TYPE);
- verify(s3Util);
assertEquals("test_path/hdfs_namenode/hdfs_namenode.log.123343493473948.gz", resolvedPath);
}
@@ -74,18 +71,17 @@ public class S3UploaderTest {
Map<String, Object> configs = setupS3Configs();
S3OutputConfiguration s3OutputConfiguration = new S3OutputConfiguration(configs);
- S3Util s3Util = mock(S3Util.class);
- String s3Key = String.format("%s/%s/%s.%s", TEST_PATH, LOG_TYPE, fileName, GZ);
- s3Util.uploadFileTos3(TEST_BUCKET, s3Key, compressedFile, ACCESS_KEY_VALUE, SECRET_KEY_VALUE);
expect(compressedFile.delete()).andReturn(true);
expect(fileToUpload.delete()).andReturn(true);
- replay(fileToUpload, compressedFile, s3Util);
+ replay(fileToUpload, compressedFile);
- S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, s3Util, true, LOG_TYPE) {
+ S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, true, LOG_TYPE) {
@Override
protected File createCompressedFileForUpload(File fileToUpload, String compressionAlgo) {
return compressedFile;
}
+ protected void uploadFileToS3(String bucketName, String s3Key, File localFile, String accessKey, String secretKey) {
+ }
};
s3Uploader.uploadFile(fileToUpload, LOG_TYPE);
@@ -102,17 +98,16 @@ public class S3UploaderTest {
Map<String, Object> configs = setupS3Configs();
S3OutputConfiguration s3OutputConfiguration = new S3OutputConfiguration(configs);
- S3Util s3Util = mock(S3Util.class);
- String s3Key = String.format("%s/%s/%s.%s", TEST_PATH, LOG_TYPE, fileName, GZ);
- s3Util.uploadFileTos3(TEST_BUCKET, s3Key, compressedFile, ACCESS_KEY_VALUE, SECRET_KEY_VALUE);
expect(compressedFile.delete()).andReturn(true);
- replay(fileToUpload, compressedFile, s3Util);
+ replay(fileToUpload, compressedFile);
- S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, s3Util, false, LOG_TYPE) {
+ S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, false, LOG_TYPE) {
@Override
protected File createCompressedFileForUpload(File fileToUpload, String compressionAlgo) {
return compressedFile;
}
+ protected void uploadFileToS3(String bucketName, String s3Key, File localFile, String accessKey, String secretKey) {
+ }
};
s3Uploader.uploadFile(fileToUpload, LOG_TYPE);
@@ -131,22 +126,19 @@ public class S3UploaderTest {
S3OutputConfiguration s3OutputConfiguration = new S3OutputConfiguration(configs);
- S3Util s3Util = mock(S3Util.class);
- String s3Key = String.format("%s/%s/%s/%s.%s", "cl1", TEST_PATH, LOG_TYPE, fileName, GZ);
- s3Util.uploadFileTos3(TEST_BUCKET, s3Key, compressedFile, ACCESS_KEY_VALUE, SECRET_KEY_VALUE);
expect(compressedFile.delete()).andReturn(true);
expect(fileToUpload.delete()).andReturn(true);
- replay(fileToUpload, compressedFile, s3Util);
+ replay(fileToUpload, compressedFile);
- S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, s3Util, true, LOG_TYPE) {
+ S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, true, LOG_TYPE) {
@Override
protected File createCompressedFileForUpload(File fileToUpload, String compressionAlgo) {
return compressedFile;
}
+ protected void uploadFileToS3(String bucketName, String s3Key, File localFile, String accessKey, String secretKey) {
+ }
};
s3Uploader.uploadFile(fileToUpload, LOG_TYPE);
-
- verify(s3Util);
}
private Map<String, Object> setupS3Configs() {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerTest.java
index 7a47039..2cfe9ff 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/AWSUtilTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/AWSUtilTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/AWSUtilTest.java
deleted file mode 100644
index 6df2283..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/AWSUtilTest.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.util;
-
-import org.apache.ambari.logfeeder.util.AWSUtil;
-
-public class AWSUtilTest {
- public void testAWSUtil_getAwsUserName() throws Exception {
- String S3_ACCESS_KEY = "S3_ACCESS_KEY";
- String S3_SECRET_KEY = "S3_SECRET_KEY";
- AWSUtil.INSTANCE.getAwsUserName(S3_ACCESS_KEY, S3_SECRET_KEY);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/PlaceholderUtilTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/PlaceholderUtilTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/PlaceholderUtilTest.java
index 9789a14..43e03c7 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/PlaceholderUtilTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/PlaceholderUtilTest.java
@@ -1,6 +1,4 @@
-package org.apache.ambari.logfeeder.util;
-
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -17,6 +15,9 @@ package org.apache.ambari.logfeeder.util;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+package org.apache.ambari.logfeeder.util;
+
import java.util.HashMap;
import org.junit.Test;
@@ -26,19 +27,18 @@ import static org.junit.Assert.assertEquals;
public class PlaceholderUtilTest {
@Test
public void testPlaceholderUtil_replaceVariables() {
- HashMap<String, String> contextParam = new HashMap<String, String>();
String hostName = "host1";
String ip = "127.0.0.1";
String clusterName = "test-cluster";
+
+ HashMap<String, String> contextParam = new HashMap<String, String>();
contextParam.put("host", hostName);
contextParam.put("ip", ip);
contextParam.put("cluster", clusterName);
- String inputStr = "$CLUSTER/logfeeder/$HOST-$IP/logs";
- String resultStr = PlaceholderUtil.replaceVariables(inputStr, contextParam);
+
+ String resultStr = PlaceholderUtil.replaceVariables("$CLUSTER/logfeeder/$HOST-$IP/logs", contextParam);
String expectedStr = clusterName + "/logfeeder/" + hostName + "-" + ip + "/logs";
- assertEquals("Result string :" + resultStr
- + " is not equal to exptected string :" + expectedStr, resultStr,
- expectedStr);
+
+ assertEquals("Result string :" + resultStr + " is not equal to exptected string :" + expectedStr, resultStr, expectedStr);
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/S3UtilTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/S3UtilTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/S3UtilTest.java
index 84554b0..02918be 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/S3UtilTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/util/S3UtilTest.java
@@ -26,14 +26,14 @@ public class S3UtilTest {
public void testS3Util_pathToBucketName() throws Exception {
String s3Path = "s3://bucket_name/path/file.txt";
String expectedBucketName = "bucket_name";
- String actualBucketName = S3Util.INSTANCE.getBucketName(s3Path);
+ String actualBucketName = S3Util.getBucketName(s3Path);
assertEquals(expectedBucketName, actualBucketName);
}
public void testS3Util_pathToS3Key() throws Exception {
String s3Path = "s3://bucket_name/path/file.txt";
String expectedS3key = "path/file.txt";
- String actualS3key = S3Util.INSTANCE.getS3Key(s3Path);
+ String actualS3key = S3Util.getS3Key(s3Path);
assertEquals(expectedS3key, actualS3key);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
new file mode 100644
index 0000000..59020cc
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/resources/logfeeder.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+logfeeder.log.filter.enable=true
+logfeeder.solr.config.interval=5
+logfeeder.solr.zk_connect_string=some_connect_string
+logfeeder.metrics.collector.hosts=some_collector_host
+node.hostname=test_host_name
\ No newline at end of file
[10/10] ambari git commit: AMBARI-18310. Refactor logsearch portal
side code - Part 2 (oleewere)
Posted by ol...@apache.org.
AMBARI-18310. Refactor logsearch portal side code - Part 2 (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d89101ec
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d89101ec
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d89101ec
Branch: refs/heads/branch-dev-logsearch
Commit: d89101eca10f619d0be5fff89b5649365a878f91
Parents: 8399960
Author: oleewere <ol...@gmail.com>
Authored: Wed Sep 7 21:56:39 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Sep 7 22:32:52 2016 +0200
----------------------------------------------------------------------
ambari-logsearch/ambari-logsearch-it/pom.xml | 5 +
.../logsearch/steps/LogSearchApiSteps.java | 95 +++++++++
.../ambari/logsearch/steps/SolrSteps.java | 13 +-
.../logsearch/story/LogSearchApiQueryStory.java | 22 +++
.../ambari/logsearch/story/LogSearchStory.java | 4 +-
.../story/log_search_api_query_story.story | 17 ++
.../story/logfeeder_parsing_story.story | 16 +-
.../service-log-level-counts-values.json | 33 ++++
.../test-output/service-log-schema.json | 18 ++
.../ambari/logfeeder/output/OutputSolr.java | 9 +-
.../ambari-logsearch-portal/pom.xml | 6 +
.../org/apache/ambari/logsearch/LogSearch.java | 38 +++-
.../logsearch/common/ExternalServerClient.java | 16 +-
.../logsearch/conf/ApplicationConfig.java | 41 +---
.../ambari/logsearch/conf/AuthConfig.java | 12 +-
.../ambari/logsearch/conf/SecurityConfig.java | 112 +++++++++++
.../ambari/logsearch/dao/SolrDaoBase.java | 1 -
.../apache/ambari/logsearch/dao/UserDao.java | 13 +-
.../logsearch/manager/AuditLogsManager.java | 44 +++--
.../ambari/logsearch/manager/PublicManager.java | 6 +-
.../logsearch/manager/ServiceLogsManager.java | 142 ++++---------
.../logsearch/model/response/AuditLogData.java | 4 +-
.../ambari/logsearch/query/QueryGeneration.java | 48 +++--
.../query/SearchCriteriaConstants.java | 78 ++++++++
.../AbstractCommonAuditLogRequestConverter.java | 16 +-
.../AbstractCommonSearchRequestConverter.java | 10 +-
...bstractCommonServiceLogRequestConverter.java | 39 ++--
.../query/converter/AbstractConverterAware.java | 47 +++++
.../converter/AnyGraphRequestConverter.java | 12 +-
.../AuditBarGraphRequestConverter.java | 2 +-
.../converter/AuditLogRequestConverter.java | 2 +-
.../BaseServiceLogRequestConverter.java | 7 +-
.../FieldAuditLogRequestConverter.java | 2 +-
.../FieldBarGraphRequestConverter.java | 4 +-
.../converter/LogFileRequestConverter.java | 6 +-
.../converter/LogFileTailRequestConverter.java | 8 +-
.../ServiceAnyGraphRequestConverter.java | 12 +-
.../ServiceExtremeDatesRequestConverter.java | 3 +-
.../converter/ServiceGraphRequestConverter.java | 6 +-
.../ServiceLogExportRequestConverter.java | 9 +-
.../ServiceLogFileRequestConverter.java | 4 +-
.../converter/ServiceLogRequestConverter.java | 14 +-
.../ServiceLogTruncatedRequestConverter.java | 10 +-
.../converter/SimpleQueryRequestConverter.java | 11 +-
.../converter/UserConfigRequestConverter.java | 10 +-
.../converter/UserExportRequestConverter.java | 3 +-
.../query/model/AnyGraphSearchCriteria.java | 56 +++++-
.../model/AuditBarGraphSearchCriteria.java | 12 +-
.../query/model/AuditLogSearchCriteria.java | 12 +-
.../query/model/CommonSearchCriteria.java | 93 +++++----
.../model/CommonServiceLogSearchCriteria.java | 96 +++++++++
.../model/FieldAuditBarGraphSearchCriteria.java | 11 +-
.../model/FieldAuditLogSearchCriteria.java | 11 +-
.../query/model/LogFileSearchCriteria.java | 29 ++-
.../query/model/LogFileTailSearchCriteria.java | 13 +-
.../logsearch/query/model/SearchCriteria.java | 60 +++---
.../model/ServiceAnyGraphSearchCriteria.java | 37 +++-
.../model/ServiceExtremeDatesCriteria.java | 11 +-
.../query/model/ServiceGraphSearchCriteria.java | 11 +-
.../model/ServiceLogExportSearchCriteria.java | 19 +-
.../model/ServiceLogFileSearchCriteria.java | 21 +-
.../query/model/ServiceLogSearchCriteria.java | 44 ++++-
.../ServiceLogTruncatedSearchCriteria.java | 27 ++-
.../query/model/UserConfigSearchCriteria.java | 27 ++-
.../query/model/UserExportSearchCriteria.java | 10 +-
.../logsearch/rest/AuditLogsResource.java | 2 +-
.../logsearch/rest/ServiceLogsResource.java | 7 +-
.../logsearch/solr/metrics/SolrAmsClient.java | 67 -------
.../logsearch/solr/metrics/SolrJmxAdapter.java | 134 -------------
.../solr/metrics/SolrMetricsLoader.java | 198 -------------------
.../logsearch/solr/model/SolrAuditLogData.java | 6 +-
.../LogsearchLogoutSuccessHandler.java | 2 +-
.../LogsearchAuthenticationEntryPoint.java | 2 +-
.../LogsearchKRBAuthenticationFilter.java | 1 +
...LogsearchSecurityContextFormationFilter.java | 1 +
...rchUsernamePasswordAuthenticationFilter.java | 2 +
.../web/listeners/SpringEventListener.java | 32 ---
...LogsearchAbstractAuthenticationProvider.java | 10 -
.../LogsearchAuthenticationProvider.java | 13 +-
...rchExternalServerAuthenticationProvider.java | 12 +-
.../LogsearchFileAuthenticationProvider.java | 11 +-
.../LogsearchLdapAuthenticationProvider.java | 22 ++-
.../LogsearchLdapBindAuthenticator.java | 64 +-----
.../LogsearchSimpleAuthenticationProvider.java | 13 +-
.../src/main/resources/log4j.xml | 6 +-
.../META-INF/security-applicationContext.xml | 83 --------
.../src/main/webapp/WEB-INF/web.xml | 62 ------
.../test-config/logfeeder/logfeeder.properties | 3 +-
.../shipper-conf/input.config-hdfs.json | 172 ++++++++++++++++
.../docker/test-logs/hdfs-audit/hdfs-audit.log | 4 +
90 files changed, 1410 insertions(+), 1149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/pom.xml b/ambari-logsearch/ambari-logsearch-it/pom.xml
index d918c8a..373068e 100644
--- a/ambari-logsearch/ambari-logsearch-it/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-it/pom.xml
@@ -82,6 +82,11 @@
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.10</version>
</dependency>
+ <dependency>
+ <groupId>com.flipkart.zjsonpatch</groupId>
+ <artifactId>zjsonpatch</artifactId>
+ <version>0.2.4</version>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java
new file mode 100644
index 0000000..bcfc4db
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/LogSearchApiSteps.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.steps;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.flipkart.zjsonpatch.JsonDiff;
+import com.google.common.io.Resources;
+import org.apache.ambari.logsearch.domain.StoryDataRegistry;
+import org.glassfish.jersey.client.JerseyClient;
+import org.glassfish.jersey.client.JerseyClientBuilder;
+import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature;
+import org.jbehave.core.annotations.Named;
+import org.jbehave.core.annotations.Then;
+import org.jbehave.core.annotations.When;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.client.Invocation;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.core.MediaType;
+import java.io.File;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class LogSearchApiSteps {
+
+ private static Logger LOG = LoggerFactory.getLogger(LogSearchApiSteps.class);
+
+ private String response;
+
+ @When("LogSearch api query sent: <apiQuery>")
+ public void sendApiQuery(@Named("apiQuery") String apiQuery) {
+ JerseyClient jerseyClient = JerseyClientBuilder.createClient();
+ HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder()
+ .credentials("admin", "admin")
+ .build();
+ jerseyClient.register(authFeature);
+
+ String logsearchUrl = String.format("http://%s:%d%s",
+ StoryDataRegistry.INSTANCE.getDockerHost(),
+ StoryDataRegistry.INSTANCE.getLogsearchPort(),
+ apiQuery);
+
+ LOG.info("Url: {}", logsearchUrl);
+
+ WebTarget target = jerseyClient.target(logsearchUrl);
+ Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE);
+ response = invocationBuilder.get().readEntity(String.class);
+ }
+
+
+ @Then("The api query result is <jsonResult>")
+ public void verifyRestApiCall(@Named("jsonResult") String jsonResult) throws IOException, URISyntaxException {
+ ObjectMapper mapper = new ObjectMapper();
+ Path jsonFilePath = new File(Resources.getResource("test-output/" + jsonResult).toURI()).toPath();
+ String jsonExpected = new String(Files.readAllBytes(jsonFilePath));
+
+ JsonNode expected = mapper.readTree(jsonExpected);
+ JsonNode result = mapper.readTree(response);
+ JsonNode patch = JsonDiff.asJson(expected, result);
+ List<Object> diffObjects = mapper.convertValue(patch, List.class);
+ assertDiffs(diffObjects, expected);
+
+ }
+
+ private void assertDiffs(List<Object> diffObjects, JsonNode expected) {
+ for (Object diffObj : diffObjects) {
+ String path = ((Map<String, String>) diffObj).get("path");
+ Assert.assertTrue(expected.at(path).isMissingNode());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
index e1e5b99..7c72ca7 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/steps/SolrSteps.java
@@ -22,25 +22,24 @@ import org.apache.ambari.logsearch.domain.StoryDataRegistry;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocumentList;
-import org.jbehave.core.annotations.Given;
+import org.jbehave.core.annotations.Named;
import org.jbehave.core.annotations.Then;
-import org.jbehave.core.annotations.When;
-import org.jbehave.core.model.Story;
import org.junit.Assert;
import java.io.IOException;
public class SolrSteps {
- @Then("the number of $component docs is: $docSize")
- public void numberOfDocsForComponent(String component, int docSize) throws IOException, SolrServerException {
+ @Then("the number of <component> docs is: <docSize>")
+ public void numberOfDocsForComponent(@Named("component") String component, @Named("docSize") int docSize)
+ throws IOException, SolrServerException, InterruptedException {
SolrClient solrClient = StoryDataRegistry.INSTANCE.getCloudSolrClient();
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery(String.format("type:%s", component));
+ solrQuery.setStart(0);
+ solrQuery.setRows(20);
QueryResponse queryResponse = solrClient.query(StoryDataRegistry.INSTANCE.getServiceLogsCollection(), solrQuery);
SolrDocumentList list = queryResponse.getResults();
Assert.assertEquals(docSize, list.size());
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchApiQueryStory.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchApiQueryStory.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchApiQueryStory.java
new file mode 100644
index 0000000..45455bf
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchApiQueryStory.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.story;
+
+public class LogSearchApiQueryStory extends LogSearchStory {
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
index f85eb08..ce6b9cb 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/story/LogSearchStory.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.story;
+import org.apache.ambari.logsearch.steps.LogSearchApiSteps;
import org.apache.ambari.logsearch.steps.SolrSteps;
import org.apache.ambari.logsearch.steps.LogSearchDockerSteps;
import org.jbehave.core.configuration.Configuration;
@@ -47,7 +48,8 @@ abstract public class LogSearchStory extends JUnitStory {
public InjectableStepsFactory stepsFactory() {
return new InstanceStepsFactory(configuration(),
new LogSearchDockerSteps(),
- new SolrSteps());
+ new SolrSteps(),
+ new LogSearchApiSteps());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/log_search_api_query_story.story
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/log_search_api_query_story.story b/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/log_search_api_query_story.story
new file mode 100644
index 0000000..5abe8b4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/log_search_api_query_story.story
@@ -0,0 +1,17 @@
+Meta:
+
+Narrative:
+As a user
+I want to perform queries against Log Search api
+So that I can validate the json outputs
+
+Scenario: scenario description
+
+Given logsearch docker container
+When LogSearch api query sent: <apiQuery>
+Then The api query result is <jsonResult>
+
+Examples:
+|apiQuery|jsonResult|
+|/api/v1/service/logs/schema/fields|service-log-schema.json|
+|/api/v1/service/logs/levels/counts/namevalues?page=0&pageSize=25&startIndex=0&q=*%3A*|service-log-level-counts-values.json|
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story b/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
index a40d04a..388e624 100644
--- a/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/org/apache/ambari/logsearch/story/logfeeder_parsing_story.story
@@ -5,14 +5,16 @@ As a user
I want to start logsearch/logfeeder/solr components in a docker container with test logs
So that I can parse and store the logs into Solr
-Scenario: Logsearch logs are stored into Solr.
+Scenario: Number of logs for components
Given logsearch docker container
When logfeeder started (parse logs & send data to solr)
-Then the number of logsearch_app docs is: 1
+Then the number of <component> docs is: <docSize>
-Scenario: Zookeeper logs are stored into Solr.
-
-Given logsearch docker container
-When logfeeder started (parse logs & send data to solr)
-Then the number of zookeeper docs is: 3
+Examples:
+|component|docSize|
+|logsearch_app|1|
+|zookeeper|3|
+|hst_agent|4|
+|secure_log|11|
+|system_message|17|
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-level-counts-values.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-level-counts-values.json b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-level-counts-values.json
new file mode 100644
index 0000000..0ef05e8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-level-counts-values.json
@@ -0,0 +1,33 @@
+{
+ "vNameValues": [
+ {
+ "name": "FATAL",
+ "value": "0"
+ },
+ {
+ "name": "ERROR",
+ "value": "0"
+ },
+ {
+ "name": "WARN",
+ "value": "3"
+ },
+ {
+ "name": "INFO",
+ "value": "4"
+ },
+ {
+ "name": "DEBUG",
+ "value": "1"
+ },
+ {
+ "name": "TRACE",
+ "value": "0"
+ },
+ {
+ "name": "UNKNOWN",
+ "value": "28"
+ }
+ ],
+ "listSize": 7
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json
new file mode 100644
index 0000000..9efa58b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-it/src/test/resources/test-output/service-log-schema.json
@@ -0,0 +1,18 @@
+{
+ "log_message": "Log Message",
+ "bundle_id": "Bundle Id",
+ "case_id": "Case Id",
+ "cluster": "Cluster",
+ "event_count": "Event Count",
+ "file": "File",
+ "host": "Host",
+ "id": "Id",
+ "ip": "IP",
+ "level": "Level",
+ "logfile_line_number": "Logfile Line Number",
+ "logger_name": "Logger Name",
+ "method": "method",
+ "path": "Path",
+ "text": "Text",
+ "type": "Type"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index 47f139d..53cb6c0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -77,17 +77,17 @@ public class OutputSolr extends Output {
private BlockingQueue<OutputData> outgoingBuffer = null;
private List<SolrWorkerThread> workerThreadList = new ArrayList<>();
-
+
@Override
protected String getStatMetricName() {
return "output.solr.write_logs";
}
-
+
@Override
protected String getWriteBytesMetricName() {
return "output.solr.write_bytes";
}
-
+
@Override
public void init() throws Exception {
super.init();
@@ -263,6 +263,9 @@ public class OutputSolr extends Output {
private void useActualDateIfNeeded(Map<String, Object> jsonObj) {
if (skipLogtime) {
jsonObj.put("logtime", DateUtil.getActualDateStr());
+ if (jsonObj.get("evtTime") != null) {
+ jsonObj.put("evtTime", DateUtil.getActualDateStr());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml
index ebba82c..a1d1bfc 100755
--- a/ambari-logsearch/ambari-logsearch-portal/pom.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml
@@ -36,6 +36,7 @@
<jersey.version>2.23.2</jersey.version>
<jetty-version>9.2.11.v20150529</jetty-version>
<swagger.version>1.5.8</swagger.version>
+ <spring-data-solr.version>2.0.2.RELEASE</spring-data-solr.version>
</properties>
<profiles>
<!-- Dev Profile Start -->
@@ -751,5 +752,10 @@
<artifactId>swagger-ui</artifactId>
<version>2.1.0</version>
</dependency>
+ <dependency>
+ <groupId>org.springframework.data</groupId>
+ <artifactId>spring-data-solr</artifactId>
+ <version>${spring-data-solr.version}</version>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
index 27b943a..75d8be3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/LogSearch.java
@@ -24,11 +24,11 @@ import java.net.ServerSocket;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.util.EnumSet;
-import org.apache.ambari.logsearch.common.ConfigHelper;
import org.apache.ambari.logsearch.common.ManageStartEndTime;
import org.apache.ambari.logsearch.common.PropertiesHelper;
-import org.apache.ambari.logsearch.solr.metrics.SolrMetricsLoader;
+import org.apache.ambari.logsearch.conf.ApplicationConfig;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.eclipse.jetty.server.Connector;
@@ -40,10 +40,18 @@ import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
import org.eclipse.jetty.server.handler.HandlerList;
import org.eclipse.jetty.server.handler.ResourceHandler;
+import org.eclipse.jetty.servlet.FilterHolder;
import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
import org.eclipse.jetty.util.resource.Resource;
import org.eclipse.jetty.util.ssl.SslContextFactory;
import org.eclipse.jetty.webapp.WebAppContext;
+import org.springframework.web.context.ContextLoaderListener;
+import org.springframework.web.context.request.RequestContextListener;
+import org.springframework.web.context.support.AnnotationConfigWebApplicationContext;
+import org.springframework.web.filter.DelegatingFilterProxy;
+
+import javax.servlet.DispatcherType;
public class LogSearch {
private static final Logger logger = Logger.getLogger(LogSearch.class);
@@ -65,6 +73,7 @@ public class LogSearch {
private static final String WEB_RESOURCE_FOLDER = "webapps/app";
private static final String ROOT_CONTEXT = "/";
+ private static final Integer SESSION_TIMEOUT = 30;
public static void main(String[] argv) {
@@ -75,7 +84,6 @@ public class LogSearch {
} catch (Throwable e) {
logger.error("Error running logsearch server", e);
}
- SolrMetricsLoader.startSolrMetricsLoaderTasks();
}
public void run(String[] argv) throws Exception {
@@ -150,6 +158,21 @@ public class LogSearch {
context.setBaseResource(Resource.newResource(webResourceBase));
context.setContextPath(ROOT_CONTEXT);
context.setParentLoaderPriority(true);
+
+ // Configure Spring
+ context.addEventListener(new ContextLoaderListener());
+ context.addEventListener(new RequestContextListener());
+ context.addFilter(new FilterHolder(new DelegatingFilterProxy("springSecurityFilterChain")), "/*", EnumSet.allOf(DispatcherType.class));
+ context.setInitParameter("contextClass", AnnotationConfigWebApplicationContext.class.getName());
+ context.setInitParameter("contextConfigLocation", ApplicationConfig.class.getName());
+
+ // Configure Jersey
+ ServletHolder jerseyServlet = context.addServlet(org.glassfish.jersey.servlet.ServletContainer.class, "/api/v1/*");
+ jerseyServlet.setInitOrder(1);
+ jerseyServlet.setInitParameter("jersey.config.server.provider.packages","org.apache.ambari.logsearch.rest,io.swagger.jaxrs.listing");
+
+ context.getSessionHandler().getSessionManager().setMaxInactiveInterval(SESSION_TIMEOUT);
+
return context;
}
@@ -167,18 +190,19 @@ public class LogSearch {
private URI findWebResourceBase() {
URL fileCompleteUrl = Thread.currentThread().getContextClassLoader()
.getResource(WEB_RESOURCE_FOLDER);
+ String errorMessage = "Web Resource Folder " + WEB_RESOURCE_FOLDER+ " not found in classpath";
if (fileCompleteUrl != null) {
try {
return fileCompleteUrl.toURI().normalize();
} catch (URISyntaxException e) {
- logger.error("Web Resource Folder " + WEB_RESOURCE_FOLDER+ " not found in classpath", e);
+ logger.error(errorMessage, e);
System.exit(1);
}
- }else{
- logger.error("Web Resource Folder " + WEB_RESOURCE_FOLDER+ " not found in classpath");
+ } else {
+ logger.error(errorMessage);
System.exit(1);
}
- return null;
+ throw new IllegalStateException(errorMessage);
}
private void checkPort(int port) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
index edc78d1..2590dfe 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java
@@ -21,7 +21,6 @@ package org.apache.ambari.logsearch.common;
import java.util.List;
import java.util.Map;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
@@ -29,7 +28,6 @@ import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import org.apache.ambari.logsearch.conf.AuthConfig;
-import org.apache.ambari.logsearch.web.security.LogsearchAbstractAuthenticationProvider;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.glassfish.jersey.client.JerseyClient;
@@ -55,23 +53,18 @@ public class ExternalServerClient {
@Inject
private AuthConfig authConfig;
- private boolean enableLog = false;// default
-
/**
* Send GET request to an external server
*/
- @SuppressWarnings({ "unchecked", "rawtypes" })
public Object sendGETRequest(String url, Class klass, MultivaluedMap<String, String> queryParam,
String username, String password)
throws Exception {
url = authConfig.getExternalAuthHostUrl() + url;
JerseyClient client = localJerseyClient.get();
- HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder().build();
-
+ HttpAuthenticationFeature authFeature = HttpAuthenticationFeature.basicBuilder()
+ .credentials(username, password)
+ .build();
client.register(authFeature);
- if (enableLog) {
- client.register(LoggingFilter.class);
- }
WebTarget target = client.target(url);
LOG.debug("URL: " + url);
@@ -80,9 +73,6 @@ public class ExternalServerClient {
LOG.debug(
String.format("Query parameter: name - %s ; value - %s ;" + entry.getKey(), StringUtils.join(entry.getValue(),',')));
}
- target
- .property(HttpAuthenticationFeature.HTTP_AUTHENTICATION_BASIC_USERNAME, username)
- .property(HttpAuthenticationFeature.HTTP_AUTHENTICATION_BASIC_PASSWORD, password);
Invocation.Builder invocationBuilder = target.request(MediaType.APPLICATION_JSON_TYPE);
try {
return invocationBuilder.get().readEntity(klass);
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
index 72ea942..b279a83 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java
@@ -18,38 +18,16 @@
*/
package org.apache.ambari.logsearch.conf;
-import org.apache.ambari.logsearch.query.converter.AnyGraphRequestConverter;
-import org.apache.ambari.logsearch.query.converter.AuditBarGraphRequestConverter;
-import org.apache.ambari.logsearch.query.converter.AuditLogRequestConverter;
-import org.apache.ambari.logsearch.query.converter.BaseAuditLogRequestConverter;
-import org.apache.ambari.logsearch.query.converter.BaseServiceLogRequestConverter;
-import org.apache.ambari.logsearch.query.converter.FieldAuditLogRequestConverter;
-import org.apache.ambari.logsearch.query.converter.FieldBarGraphRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceAnyGraphRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceExtremeDatesRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceGraphRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceLogExportRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceLogFileRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceLogRequestConverter;
-import org.apache.ambari.logsearch.query.converter.ServiceLogTruncatedRequestConverter;
-import org.apache.ambari.logsearch.query.converter.SimpleQueryRequestConverter;
-import org.apache.ambari.logsearch.query.converter.UserExportRequestConverter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
-import org.springframework.context.annotation.ImportResource;
import org.springframework.context.annotation.PropertySource;
import org.springframework.context.support.ConversionServiceFactoryBean;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
-import org.springframework.core.convert.converter.Converter;
-
-import java.util.HashSet;
-import java.util.Set;
@Configuration
@ComponentScan("org.apache.ambari.logsearch")
@PropertySource(value = {"classpath:default.properties", "classpath:logsearch.properties"})
-@ImportResource("META-INF/security-applicationContext.xml")
public class ApplicationConfig {
@Bean
@@ -60,24 +38,7 @@ public class ApplicationConfig {
@Bean(name="conversionService")
public ConversionServiceFactoryBean conversionServiceFactoryBean() {
ConversionServiceFactoryBean conversionServiceFactoryBean = new ConversionServiceFactoryBean();
- Set<Converter> converters = new HashSet<>();
- converters.add(new AnyGraphRequestConverter());
- converters.add(new AuditBarGraphRequestConverter());
- converters.add(new AuditLogRequestConverter());
- converters.add(new BaseAuditLogRequestConverter());
- converters.add(new BaseServiceLogRequestConverter());
- converters.add(new FieldAuditLogRequestConverter());
- converters.add(new FieldBarGraphRequestConverter());
- converters.add(new SimpleQueryRequestConverter());
- converters.add(new UserExportRequestConverter());
- converters.add(new ServiceAnyGraphRequestConverter());
- converters.add(new ServiceExtremeDatesRequestConverter());
- converters.add(new ServiceGraphRequestConverter());
- converters.add(new ServiceLogExportRequestConverter());
- converters.add(new ServiceLogFileRequestConverter());
- converters.add(new ServiceLogRequestConverter());
- converters.add(new ServiceLogTruncatedRequestConverter());
- conversionServiceFactoryBean.setConverters(converters);
+ conversionServiceFactoryBean.afterPropertiesSet();
return conversionServiceFactoryBean;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java
index 3398a83..46d07bd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/AuthConfig.java
@@ -30,9 +30,11 @@ public class AuthConfig {
boolean authLdapEnabled;
@Value("${logsearch.auth.simple.enable:false}")
boolean authSimpleEnabled;
+ @Value("${logsearch.auth.external_auth.enable:false}")
+ boolean authExternalEnabled;
@Value("${logsearch.auth.external_auth.host_url:'http://ip:port'}")
private String externalAuthHostUrl;
- @Value("${logsearch.auth.login_url:/api/v1/users/$USERNAME/privileges?fields=*}")
+ @Value("${logsearch.auth.external_auth.login_url:/api/v1/users/$USERNAME/privileges?fields=*}")
private String externalAuthLoginUrl;
@Value("${logsearch.login.credentials.file:user_pass.json}")
private String credentialsFile;
@@ -84,4 +86,12 @@ public class AuthConfig {
public void setExternalAuthLoginUrl(String externalAuthLoginUrl) {
this.externalAuthLoginUrl = externalAuthLoginUrl;
}
+
+ public boolean isAuthExternalEnabled() {
+ return authExternalEnabled;
+ }
+
+ public void setAuthExternalEnabled(boolean authExternalEnabled) {
+ this.authExternalEnabled = authExternalEnabled;
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
new file mode 100644
index 0000000..d3db110
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SecurityConfig.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.conf;
+
+import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler;
+import org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler;
+import org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler;
+import org.apache.ambari.logsearch.web.filters.LogsearchAuthenticationEntryPoint;
+import org.apache.ambari.logsearch.web.filters.LogsearchKRBAuthenticationFilter;
+import org.apache.ambari.logsearch.web.filters.LogsearchSecurityContextFormationFilter;
+import org.apache.ambari.logsearch.web.filters.LogsearchUsernamePasswordAuthenticationFilter;
+import org.apache.ambari.logsearch.web.security.LogsearchAuthenticationProvider;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.security.config.annotation.web.builders.HttpSecurity;
+import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity;
+import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter;
+import org.springframework.security.config.http.SessionCreationPolicy;
+import org.springframework.security.web.access.intercept.FilterSecurityInterceptor;
+import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
+
+@Configuration
+@EnableWebSecurity
+public class SecurityConfig extends WebSecurityConfigurerAdapter {
+
+ @Override
+ protected void configure(HttpSecurity http) throws Exception {
+ http
+ .csrf().disable()
+ .sessionManagement()
+ .sessionFixation()
+ .newSession()
+ .sessionCreationPolicy(SessionCreationPolicy.ALWAYS)
+ .and()
+ .authorizeRequests()
+ .antMatchers("/login.html").permitAll()
+ .antMatchers("/styles/**").permitAll()
+ .antMatchers("/fonts/**").permitAll()
+ .antMatchers("/fonts/**").permitAll()
+ .antMatchers("/scripts/**").permitAll()
+ .antMatchers("/libs/**").permitAll()
+ .antMatchers("/images/**").permitAll()
+ .antMatchers("/templates/**").permitAll()
+ .antMatchers("/favicon.ico").permitAll()
+ .antMatchers("/api/v1/public/**").permitAll()
+ .antMatchers("/api/v1/swagger.json").permitAll()
+ .antMatchers("/**").authenticated()
+ .and()
+ .authenticationProvider(logsearchAuthenticationProvider())
+ .formLogin()
+ .loginPage("/login.html")
+ .and()
+ .httpBasic()
+ .authenticationEntryPoint(logsearchAuthenticationEntryPoint())
+ .and()
+ .addFilterBefore(logsearchUsernamePasswordAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class)
+ .addFilterBefore(new LogsearchKRBAuthenticationFilter(), UsernamePasswordAuthenticationFilter.class)
+ .addFilterAfter(securityContextFormationFilter(), FilterSecurityInterceptor.class)
+ .logout()
+ .logoutUrl("/logout.html")
+ .deleteCookies("JSESSIONID")
+ .logoutSuccessHandler(new LogsearchLogoutSuccessHandler());
+ }
+
+ @Bean
+ public LogsearchSecurityContextFormationFilter securityContextFormationFilter() {
+ return new LogsearchSecurityContextFormationFilter();
+ }
+
+ @Bean
+ public LogsearchKRBAuthenticationFilter logsearchKRBAuthenticationFilter() {
+ return new LogsearchKRBAuthenticationFilter();
+ }
+
+ @Bean
+ public LogsearchAuthenticationProvider logsearchAuthenticationProvider() {
+ return new LogsearchAuthenticationProvider();
+ }
+
+ @Bean
+ public LogsearchAuthenticationEntryPoint logsearchAuthenticationEntryPoint() {
+ LogsearchAuthenticationEntryPoint entryPoint = new LogsearchAuthenticationEntryPoint("/login.html");
+ entryPoint.setForceHttps(false);
+ return entryPoint;
+ }
+
+ @Bean
+ public LogsearchUsernamePasswordAuthenticationFilter logsearchUsernamePasswordAuthenticationFilter() throws Exception {
+ LogsearchUsernamePasswordAuthenticationFilter filter = new LogsearchUsernamePasswordAuthenticationFilter();
+ filter.setAuthenticationSuccessHandler(new LogsearchAuthSuccessHandler());
+ filter.setAuthenticationFailureHandler(new LogsearchAuthFailureHandler());
+ filter.setAuthenticationManager(authenticationManagerBean());
+ return filter;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
index b325171..ac7f56f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java
@@ -29,7 +29,6 @@ import java.util.List;
import org.apache.ambari.logsearch.common.ConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchContext;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.common.PropertiesHelper;
import org.apache.ambari.logsearch.conf.SolrKerberosConfig;
import org.apache.ambari.logsearch.conf.SolrUserConfig;
import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
index a04dee4..4ca9df6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java
@@ -52,9 +52,6 @@ public class UserDao {
private static final String NAME = "name";
@Inject
- private LogsearchFileAuthenticationProvider fileAuthenticationProvider;
-
- @Inject
private AuthConfig authConfig;
private ArrayList<HashMap<String, String>> userList = null;
@@ -62,7 +59,7 @@ public class UserDao {
@SuppressWarnings("unchecked")
@PostConstruct
public void initialization() {
- if (fileAuthenticationProvider.isEnable()) {
+ if (authConfig.isAuthFileEnabled()) {
try {
String userPassJsonFileName = authConfig.getCredentialsFile();
logger.info("USER PASS JSON file NAME:" + userPassJsonFileName);
@@ -151,12 +148,4 @@ public class UserDao {
}
return isUpdated;
}
-
- public String encryptPassword(String username, String password) {
- if (!StringUtils.isEmpty(username)) {
- username = username.toLowerCase();
- }
- String saltEncodedpasswd = CommonUtil.encryptPassword(password, username);
- return saltEncodedpasswd;
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
index 7affc5a..c64cf71 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java
@@ -49,6 +49,12 @@ import org.apache.ambari.logsearch.model.response.LogData;
import org.apache.ambari.logsearch.model.response.LogSearchResponse;
import org.apache.ambari.logsearch.model.response.NameValueData;
import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
+import org.apache.ambari.logsearch.query.model.AnyGraphSearchCriteria;
+import org.apache.ambari.logsearch.query.model.AuditBarGraphSearchCriteria;
+import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
+import org.apache.ambari.logsearch.query.model.FieldAuditBarGraphSearchCriteria;
+import org.apache.ambari.logsearch.query.model.FieldAuditLogSearchCriteria;
+import org.apache.ambari.logsearch.query.model.UserExportSearchCriteria;
import org.apache.ambari.logsearch.solr.model.SolrAuditLogData;
import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData;
import org.apache.ambari.logsearch.util.BizUtil;
@@ -82,7 +88,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
private SolrAuditLogConfig solrAuditLogConfig;
public AuditLogResponse getLogs(AuditLogSearchCriteria searchCriteria) {
- Boolean isLastPage = (Boolean) searchCriteria.getParamValue("isLastPage");
+ Boolean isLastPage = searchCriteria.isLastPage();
if (isLastPage) {
SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
LogSearchResponse logResponse = getLastPage(searchCriteria, LogSearchConstants.AUDIT_EVTTIME, auditSolrDao, lastPageQuery);
@@ -95,7 +101,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
return getLogAsPaginationProvided(solrQuery, auditSolrDao);
}
- private List<LogData> getComponents(SearchCriteria searchCriteria) {
+ private List<LogData> getComponents(CommonSearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
List<LogData> docList = new ArrayList<>();
try {
@@ -133,7 +139,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
}
- public GroupListResponse getAuditComponents(SearchCriteria searchCriteria) {
+ public GroupListResponse getAuditComponents(CommonSearchCriteria searchCriteria) {
GroupListResponse componentResponse = new GroupListResponse();
List<LogData> docList = getComponents(searchCriteria);
componentResponse.setGroupList(docList);
@@ -141,13 +147,13 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
@SuppressWarnings("unchecked")
- public BarGraphDataListResponse getAuditBarGraphData(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphSearchCriteria searchCriteria) {
BarGraphDataListResponse dataList = new BarGraphDataListResponse();
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- String from = getFrom((String) searchCriteria.getParamValue("startTime"));
- String to = getTo((String) searchCriteria.getParamValue("endTime"));
- String unit = getUnit((String) searchCriteria.getParamValue("unit"));
+ String from = getFrom(searchCriteria.getStartTime());
+ String to = getTo(searchCriteria.getEndTime());
+ String unit = getUnit(searchCriteria.getUnit());
List<BarGraphData> histogramData = new ArrayList<BarGraphData>();
String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT,
@@ -225,7 +231,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
}
- public BarGraphDataListResponse topTenUsers(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse topTenUsers(FieldAuditBarGraphSearchCriteria searchCriteria) {
String jsonUserQuery =
"{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}";
@@ -262,7 +268,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
}
- public BarGraphDataListResponse topTenResources(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse topTenResources(FieldAuditLogSearchCriteria searchCriteria) {
String jsonUserQuery =
"{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}";
@@ -294,11 +300,11 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
@SuppressWarnings("unchecked")
- public BarGraphDataListResponse getRequestUserLineGraph(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse getRequestUserLineGraph(FieldAuditBarGraphSearchCriteria searchCriteria) {
- String from = getFrom((String) searchCriteria.getParamValue("startTime"));
- String to = getTo((String) searchCriteria.getParamValue("endTime"));
- String unit = getUnit((String) searchCriteria.getParamValue("unit"));
+ String from = getFrom(searchCriteria.getStartTime());
+ String to = getTo(searchCriteria.getEndTime());
+ String unit = getUnit(searchCriteria.getUnit());
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
@@ -355,7 +361,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
- public BarGraphDataListResponse getAnyGraphData(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse getAnyGraphData(AnyGraphSearchCriteria searchCriteria) {
searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME);
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery);
@@ -396,13 +402,13 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
}
@SuppressWarnings({"unchecked"})
- public Response exportUserTableToTextFile(SearchCriteria searchCriteria) {
+ public Response exportUserTableToTextFile(UserExportSearchCriteria searchCriteria) {
String jsonUserQuery =
"{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}";
SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria);
- String startTime = (String) searchCriteria.getParamValue("startTime");
- String endTime = (String) searchCriteria.getParamValue("endTime");
+ String startTime = searchCriteria.getStartTime();
+ String endTime = searchCriteria.getEndTime();
startTime = startTime == null ? "" : startTime;
endTime = endTime == null ? "" : "_" + endTime;
@@ -410,7 +416,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
SolrUtil.setJSONFacet(solrQuery, jsonUserQuery);
SolrUtil.setRowCount(solrQuery, 0);
- String dataFormat = (String) searchCriteria.getParamValue("format");
+ String dataFormat = searchCriteria.getFormat();
FileOutputStream fis = null;
try {
QueryResponse queryResponse = auditSolrDao.process(solrQuery);
@@ -543,7 +549,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp
return fieldWithBlank;
}
- public BarGraphDataListResponse getServiceLoad(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse getServiceLoad(CommonSearchCriteria searchCriteria) {
BarGraphDataListResponse dataList = new BarGraphDataListResponse();
Collection<BarGraphData> vaDatas = new ArrayList<BarGraphData>();
dataList.setGraphData(vaDatas);
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
index 23f62aa..3ac2be6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java
@@ -22,9 +22,9 @@ package org.apache.ambari.logsearch.manager;
import java.util.ArrayList;
import java.util.List;
+import org.apache.ambari.logsearch.conf.AuthConfig;
import org.apache.ambari.logsearch.model.response.NameValueData;
import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
-import org.apache.ambari.logsearch.web.security.LogsearchSimpleAuthenticationProvider;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
@@ -33,14 +33,14 @@ import javax.inject.Inject;
public class PublicManager extends JsonManagerBase {
@Inject
- private LogsearchSimpleAuthenticationProvider simpleAuthenticationProvider;
+ private AuthConfig authConfig;
public String getGeneralConfig() {
NameValueDataListResponse nameValueList = new NameValueDataListResponse();
List<NameValueData> nameValues = new ArrayList<>();
NameValueData nameValue = new NameValueData();
nameValue.setName("simpleAuth");
- nameValue.setValue("" + simpleAuthenticationProvider.isEnable());
+ nameValue.setValue("" + authConfig.isAuthSimpleEnabled());
nameValues.add(nameValue);
nameValueList.setvNameValues(nameValues);
return convertObjToString(nameValueList);
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
index c4d14a9..02309fc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java
@@ -43,7 +43,6 @@ import javax.ws.rs.core.Response;
import org.apache.ambari.logsearch.common.ConfigHelper;
import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.MessageEnums;
-import org.apache.ambari.logsearch.common.PropertiesHelper;
import org.apache.ambari.logsearch.conf.SolrServiceLogConfig;
import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao;
import org.apache.ambari.logsearch.graph.GraphDataGenerator;
@@ -64,6 +63,13 @@ import org.apache.ambari.logsearch.model.response.NodeListResponse;
import org.apache.ambari.logsearch.model.response.ServiceLogData;
import org.apache.ambari.logsearch.model.response.ServiceLogResponse;
import org.apache.ambari.logsearch.query.QueryGenerationBase;
+import org.apache.ambari.logsearch.query.model.CommonServiceLogSearchCriteria;
+import org.apache.ambari.logsearch.query.model.ServiceAnyGraphSearchCriteria;
+import org.apache.ambari.logsearch.query.model.ServiceGraphSearchCriteria;
+import org.apache.ambari.logsearch.query.model.ServiceLogExportSearchCriteria;
+import org.apache.ambari.logsearch.query.model.ServiceLogFileSearchCriteria;
+import org.apache.ambari.logsearch.query.model.ServiceLogSearchCriteria;
+import org.apache.ambari.logsearch.query.model.ServiceLogTruncatedSearchCriteria;
import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData;
import org.apache.ambari.logsearch.solr.model.SolrHostLogData;
import org.apache.ambari.logsearch.solr.model.SolrServiceLogData;
@@ -83,7 +89,6 @@ import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.client.solrj.response.FacetField.Count;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.client.solrj.response.RangeFacet;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
@@ -114,10 +119,10 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
@Inject
private SolrServiceLogConfig solrServiceLogConfig;
- public ServiceLogResponse searchLogs(SearchCriteria searchCriteria) {
- String keyword = (String) searchCriteria.getParamValue("keyword");
- String logId = (String) searchCriteria.getParamValue("sourceLogId");
- Boolean isLastPage = (Boolean) searchCriteria.getParamValue("isLastPage");
+ public ServiceLogResponse searchLogs(ServiceLogSearchCriteria searchCriteria) {
+ String keyword = searchCriteria.getKeyword();
+ String logId = searchCriteria.getSourceLogId();
+ Boolean isLastPage = searchCriteria.isLastPage();
if (!StringUtils.isBlank(keyword)) {
try {
@@ -203,7 +208,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
return getFields(LogSearchConstants.SOLR_COMPONENT, SolrComponentTypeLogData.class);
}
- public GraphDataListResponse getAggregatedInfo(SearchCriteria searchCriteria) {
+ public GraphDataListResponse getAggregatedInfo(CommonServiceLogSearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
String hierarchy = "host,type,level";
GraphDataListResponse graphInfo = new GraphDataListResponse();
@@ -401,7 +406,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
return extensionTree;
}
- public NodeListResponse getTreeExtension(SearchCriteria searchCriteria) {
+ public NodeListResponse getTreeExtension(ServiceLogFileSearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
solrQuery.setParam("event", "/getTreeExtension");
@@ -464,7 +469,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
return list;
}
- public NodeListResponse getHostListByComponent(SearchCriteria searchCriteria) {
+ public NodeListResponse getHostListByComponent(ServiceLogFileSearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
solrQuery.setParam("event", "/service/hosts/components");
@@ -524,7 +529,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
}
- public NameValueDataListResponse getLogsLevelCount(SearchCriteria sc) {
+ public NameValueDataListResponse getLogsLevelCount(ServiceLogFileSearchCriteria sc) {
NameValueDataListResponse nameValueList = new NameValueDataListResponse();
SolrQuery query = queryGenerator.commonServiceFilterQuery(sc);
query.setParam("event", "/service/logs/levels/counts/namevalues");
@@ -584,11 +589,11 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
return list;
}
- public LogListResponse getPageByKeyword(SearchCriteria searchCriteria)
+ public LogListResponse getPageByKeyword(ServiceLogSearchCriteria searchCriteria)
throws SolrServerException {
String defaultChoice = "0";
- String key = (String) searchCriteria.getParamValue("keyword");
+ String key = (String) searchCriteria.getKeyword();
if(StringUtils.isBlank(key)){
throw RESTErrorUtil.createRESTException("Keyword was not given",
MessageEnums.DATA_NOT_FOUND);
@@ -603,7 +608,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
keyword = "*" + keyword + "*";
- String keyType = (String) searchCriteria.getParamValue("keywordType");
+ String keyType = (String) searchCriteria.getKeywordType();
QueryResponse queryResponse = null;
if (!defaultChoice.equals(keyType)) {
@@ -689,9 +694,8 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
// Keyword Sequence Number Calculation
- String endTime = (String) searchCriteria.getParamValue("to");
- String startTime = (String) searchCriteria
- .getParamValue("from");
+ String endTime = searchCriteria.getTo();
+ String startTime = searchCriteria.getFrom();
SolrQuery logTimeThroughRangeQuery = queryGenerator
.commonServiceFilterQuery(searchCriteria);
logTimeThroughRangeQuery.remove("start");
@@ -809,8 +813,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
rangeLogQuery.remove("rows");
logIdQuery.setStart(start);
logIdQuery.setRows(searchCriteria.getMaxRows());
- LogListResponse logResponse = getLogAsPaginationProvided(logIdQuery, serviceLogsSolrDao);
- return logResponse;
+ return getLogAsPaginationProvided(logIdQuery, serviceLogsSolrDao);
} catch (Exception e) {
//do nothing
@@ -899,9 +902,8 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
// Keyword LogTime Calculation
- String endTime = (String) searchCriteria.getParamValue("to");
- String startTime = (String) searchCriteria
- .getParamValue("from");
+ String endTime = (String) searchCriteria.getTo();
+ String startTime = searchCriteria.getFrom();
SolrQuery logTimeThroughRangeQuery = queryGenerator
.commonServiceFilterQuery(searchCriteria);
logTimeThroughRangeQuery.remove("start");
@@ -1034,7 +1036,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
MessageEnums.ERROR_SYSTEM);
}
- private LogSearchResponse getPageByLogId(SearchCriteria searchCriteria) {
+ private LogSearchResponse getPageByLogId(ServiceLogSearchCriteria searchCriteria) {
LogSearchResponse logResponse = new ServiceLogResponse();
String endLogTime = (String) searchCriteria.getParamValue("to");
if(StringUtils.isBlank(endLogTime)){
@@ -1135,68 +1137,14 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
@SuppressWarnings("unchecked")
- public List<NameValueData> getHistogramCounts(SolrQuery solrQuery,
- String from, String to, String unit) {
- List<NameValueData> logsCounts = new ArrayList<>();
- try {
-
- SolrUtil.setFacetRange(solrQuery, LogSearchConstants.LOGTIME,
- from, to, unit);
-
- List<RangeFacet.Count> logLevelCounts = null;
-
- QueryResponse response = serviceLogsSolrDao.process(solrQuery);
- if(response == null){
- return logsCounts;
- }
- @SuppressWarnings("rawtypes")
- List<RangeFacet> rangeFacetList = response.getFacetRanges();
- if (rangeFacetList == null) {
- return logsCounts;
-
- }
-
- @SuppressWarnings("rawtypes")
- RangeFacet rangeFacet=rangeFacetList.get(0);
- if (rangeFacet == null) {
- return logsCounts;
- }
- logLevelCounts = rangeFacet.getCounts();
-
- if(logLevelCounts == null){
- return logsCounts;
- }
- for (RangeFacet.Count logCount : logLevelCounts) {
- NameValueData nameValue = new NameValueData();
- nameValue.setName(logCount.getValue());
- nameValue.setValue("" + logCount.getCount());
- logsCounts.add(nameValue);
- }
- } catch (SolrException | SolrServerException | IOException e) {
- logger.error("Error during solrQuery=" + solrQuery, e);
- }
- return logsCounts;
- }
-
- public List<Count> getFacetCountsByDate(SolrQuery solrQuery,
- String facetField) throws SolrServerException, IOException,
- SolrException {
-
- QueryResponse response = serviceLogsSolrDao.process(solrQuery);
-
- FacetField field = response.getFacetDate(facetField);
- return field.getValues();
- }
-
- @SuppressWarnings("unchecked")
- public BarGraphDataListResponse getHistogramData(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse getHistogramData(ServiceGraphSearchCriteria searchCriteria) {
String deafalutValue = "0";
BarGraphDataListResponse dataList = new BarGraphDataListResponse();
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
solrQuery.set("event", "/audit/logs/histogram");
- String from = getFrom((String) searchCriteria.getParamValue("from"));
- String to = getTo((String) searchCriteria.getParamValue("to"));
- String unit = getUnit((String) searchCriteria.getParamValue("unit"));
+ String from = getFrom(searchCriteria.getFrom());
+ String to = getTo(searchCriteria.getTo());
+ String unit = getUnit(searchCriteria.getUnit());
List<BarGraphData> histogramData = new ArrayList<>();
@@ -1267,16 +1215,6 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
}
- public void arrangeLevel(String level,
- List<BarGraphData> histogramDataLocal,
- List<BarGraphData> histogramData) {
- for (BarGraphData histData : histogramData) {
- if (histData != null && level.equals(histData.getName())) {
- histogramDataLocal.add(histData);
- }
- }
- }
-
public String cancelFindRequestByDate(String uniqueId) {
if (StringUtils.isEmpty(uniqueId)) {
logger.error("Unique id is Empty");
@@ -1305,13 +1243,13 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
return true;
}
- public Response exportToTextFile(SearchCriteria searchCriteria) {
+ public Response exportToTextFile(ServiceLogExportSearchCriteria searchCriteria) {
String defaultFormat = "text";
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
- String from = (String) searchCriteria.getParamValue("from");
- String to = (String) searchCriteria.getParamValue("to");
- String utcOffset = (String) searchCriteria.getParamValue("utcOffset");
- String format = (String) searchCriteria.getParamValue("format");
+ String from = searchCriteria.getFrom();
+ String to = searchCriteria.getTo();
+ String utcOffset = searchCriteria.getUtcOffset();
+ String format = searchCriteria.getFormat();
format = defaultFormat.equalsIgnoreCase(format) && format != null ? ".txt"
: ".json";
@@ -1449,7 +1387,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
}
- public NodeListResponse getComponentListWithLevelCounts(SearchCriteria searchCriteria) {
+ public NodeListResponse getComponentListWithLevelCounts(ServiceLogFileSearchCriteria searchCriteria) {
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
solrQuery.setParam("event", "/service/logs/components/levels/counts");
@@ -1670,7 +1608,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
}
- public BarGraphDataListResponse getAnyGraphData(SearchCriteria searchCriteria) {
+ public BarGraphDataListResponse getAnyGraphData(ServiceAnyGraphSearchCriteria searchCriteria) {
searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME);
SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria);
BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, serviceLogsSolrDao, solrQuery);
@@ -1681,7 +1619,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
- public ServiceLogResponse getAfterBeforeLogs(SearchCriteria searchCriteria) {
+ public ServiceLogResponse getAfterBeforeLogs(ServiceLogTruncatedSearchCriteria searchCriteria) {
ServiceLogResponse logResponse = new ServiceLogResponse();
List<SolrServiceLogData> docList = null;
String id = (String) searchCriteria
@@ -1692,11 +1630,11 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
}
String maxRows = "";
- maxRows = (String) searchCriteria.getParamValue("numberRows");
+ maxRows = searchCriteria.getNumberRows();
if (StringUtils.isBlank(maxRows)){
maxRows = ""+maxRows;
}
- String scrollType = (String) searchCriteria.getParamValue("scrollType");
+ String scrollType = searchCriteria.getScrollType();
if(StringUtils.isBlank(scrollType)){
scrollType = "";
}
@@ -1803,7 +1741,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
searchCriteria.addParam(LogSearchConstants.SORT, sortOrder);
queryGenerator.setMultipleSortOrder(solrQuery, searchCriteria);
- return (ServiceLogResponse) getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao);
+ return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao);
}
private ServiceLogResponse whenScrollDown(SearchCriteria searchCriteria,
@@ -1835,7 +1773,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL
searchCriteria.addParam(LogSearchConstants.SORT, sortOrder);
queryGenerator.setMultipleSortOrder(solrQuery, searchCriteria);
- return (ServiceLogResponse) getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao);
+ return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao);
}
@Scheduled(cron = "${logsearch.solr.warming.cron}")
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/response/AuditLogData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/response/AuditLogData.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/response/AuditLogData.java
index 6df56a1..41eca1e 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/response/AuditLogData.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/response/AuditLogData.java
@@ -98,9 +98,9 @@ public interface AuditLogData extends CommonLogData {
void setRepo(String repo);
@JsonProperty("repoType")
- String getRepoType();
+ Integer getRepoType();
- void setRepoType(String repoType);
+ void setRepoType(Integer repoType);
@JsonProperty("reqData")
String getRequestData();
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
index 10224bc..9ea29c2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java
@@ -30,6 +30,9 @@ import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.common.PropertiesHelper;
import org.apache.ambari.logsearch.conf.SolrAuditLogConfig;
import org.apache.ambari.logsearch.conf.SolrServiceLogConfig;
+import org.apache.ambari.logsearch.query.model.AuditLogSearchCriteria;
+import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
+import org.apache.ambari.logsearch.query.model.CommonServiceLogSearchCriteria;
import org.apache.ambari.logsearch.query.model.SearchCriteria;
import org.apache.ambari.logsearch.dao.SolrDaoBase;
import org.apache.ambari.logsearch.manager.ManagerBase.LogType;
@@ -59,32 +62,33 @@ public class QueryGeneration extends QueryGenerationBase {
@Inject
private SolrAuditLogConfig solrAuditLogConfig;
- public SolrQuery commonServiceFilterQuery(SearchCriteria searchCriteria) {
+ public SolrQuery commonServiceFilterQuery(CommonServiceLogSearchCriteria searchCriteria) {
LogType logType = LogType.SERVICE;
SolrQuery solrQuery = new SolrQuery();
- String treeParams = (String) searchCriteria.getParamValue("treeParams");
- String givenQuery = (String) searchCriteria.getParamValue("q");
- String level = (String) searchCriteria.getParamValue("level");
- String startTime = (String) searchCriteria.getParamValue("from");
- String endTime = (String) searchCriteria.getParamValue("to");
- String iMessage = (String) searchCriteria.getParamValue("iMessage");
- String eMessage = (String) searchCriteria.getParamValue("eMessage");
+ String advQuery = (String) searchCriteria.getParamValue("advanceSearch"); // TODO: check these are used from the UI or not
String gEmessage = (String) searchCriteria.getParamValue("gEMessage");
- String selectedComp = (String) searchCriteria.getParamValue("selectComp");
- String bundleId = (String) searchCriteria.getParamValue(LogSearchConstants.BUNDLE_ID);
String globalExcludeComp = (String) searchCriteria.getParamValue("gMustNot");
String unselectedComp = (String) searchCriteria.getParamValue("unselectComp");
- String urlHostName = (String) searchCriteria.getParamValue("host_name");
- String urlComponentName = (String) searchCriteria.getParamValue("component_name");
- String file_name = (String) searchCriteria.getParamValue("file_name");
- String advQuery = (String) searchCriteria.getParamValue("advanceSearch");
+
+ String treeParams = searchCriteria.getTreeParams();
+ String givenQuery = (String) searchCriteria.getParamValue("q");
+ String level = searchCriteria.getLevel();
+ String startTime = searchCriteria.getFrom();
+ String endTime = searchCriteria.getTo();
+ String iMessage = searchCriteria.getIncludeMessage();
+ String eMessage = searchCriteria.getExcludeMessage();
+ String selectedComp = searchCriteria.getSelectComp();
+ String bundleId = searchCriteria.getBundleId();
+ String urlHostName = searchCriteria.getHostName();
+ String urlComponentName = searchCriteria.getComponentName();
+ String file_name = searchCriteria.getFileName();
// build advance query
if (!StringUtils.isBlank(advQuery)) {
String advQueryParameters[] = advQuery.split(Pattern.quote("}{"));
SolrQuery advSolrQuery = new SolrQuery();
for (String queryParam : advQueryParameters) {
String params[] = queryParam.split(Pattern.quote("="));
- if (params != null && params.length > 1)
+ if (params.length > 1)
advSolrQuery.setParam(params[0], params[1]);
}
setFilterClauseWithFieldName(advSolrQuery, level, LogSearchConstants.SOLR_LEVEL, "", Condition.OR);
@@ -226,18 +230,20 @@ public class QueryGeneration extends QueryGenerationBase {
}
}
- public SolrQuery commonAuditFilterQuery(SearchCriteria searchCriteria) {
+ public SolrQuery commonAuditFilterQuery(CommonSearchCriteria searchCriteria) {
LogType logType = LogType.AUDIT;
SolrQuery solrQuery = new SolrQuery();
solrQuery.setQuery("*:*");
- String startTime = (String) searchCriteria.getParamValue("startTime");
- String endTime = (String) searchCriteria.getParamValue("endTime");
- String selectedComp = (String) searchCriteria.getParamValue("includeString");
+
+ String globalExcludeComp = (String) searchCriteria.getParamValue("gMustNot"); // TODO: check this are used from UI or not
+ String unselectedComp = (String) searchCriteria.getParamValue("unselectComp");
+
+ String startTime = searchCriteria.getStartTime();
+ String endTime = searchCriteria.getEndTime();
+ String selectedComp = searchCriteria.getMustBe();
setFilterClauseWithFieldName(solrQuery, selectedComp, LogSearchConstants.AUDIT_COMPONENT, LogSearchConstants.NO_OPERATOR, Condition.OR);
- String globalExcludeComp = (String) searchCriteria.getParamValue("gMustNot");
setUserSpecificFilter(searchCriteria, solrQuery, LogSearchConstants.INCLUDE_QUERY, LogSearchConstants.INCLUDE_QUERY, logType);
setUserSpecificFilter(searchCriteria, solrQuery, LogSearchConstants.EXCLUDE_QUERY, LogSearchConstants.EXCLUDE_QUERY, logType);
- String unselectedComp = (String) searchCriteria.getParamValue("unselectComp");
setFilterClauseWithFieldName(solrQuery, globalExcludeComp, LogSearchConstants.AUDIT_COMPONENT, LogSearchConstants.MINUS_OPERATOR, Condition.AND);
setFilterClauseWithFieldName(solrQuery, unselectedComp, LogSearchConstants.AUDIT_COMPONENT, LogSearchConstants.MINUS_OPERATOR, Condition.AND);
setSingleRangeFilter(solrQuery, LogSearchConstants.AUDIT_EVTTIME, startTime, endTime);
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/SearchCriteriaConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/SearchCriteriaConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/SearchCriteriaConstants.java
new file mode 100644
index 0000000..c67d3cd
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/SearchCriteriaConstants.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.query;
+
+public class SearchCriteriaConstants {
+
+ private SearchCriteriaConstants() {
+ }
+
+ public static final String PARAM_FIELD = "field";
+ public static final String PARAM_UNIT = "unit";
+ public static final String PARAM_INCLUDE_MESSAGE = "iMessage";
+ public static final String PARAM_EXCLUDE_MESSAGE = "eMessage";
+ public static final String PARAM_MUST_BE_STRING = "includeString";
+ public static final String PARAM_MUST_NOT_STRING = "unselectComp";
+ public static final String PARAM_EXCLUDE_QUERY = "excludeQuery";
+ public static final String PARAM_INCLUDE_QUERY = "includeQuery";
+ public static final String PARAM_START_TIME = "startTime";
+ public static final String PARAM_END_TIME = "endTime";
+
+ public static final String PARAM_IS_LAST_PAGE = "isLastPage";
+
+ public static final String PARAM_GLOBAL_START_TIME = "globalStartTime";
+ public static final String PARAM_GLOBAL_END_TIME = "globalEndTime";
+
+ public static final String PARAM_X_AXIS = "xAxis";
+ public static final String PARAM_Y_AXIS = "yAxis";
+ public static final String PARAM_STACK_BY = "stackBy";
+ public static final String PARAM_FROM = "from";
+ public static final String PARAM_TO = "to";
+
+ public static final String PARAM_LOG_FILE_COMPONENT = "component";
+ public static final String PARAM_LOG_FILE_HOST = "host";
+ public static final String PARAM_LOG_FILE_TYPE = "logType";
+ public static final String PARAM_LOG_TAIL_SIZE = "tailSize";
+
+ public static final String PARAM_COMPONENT_NAME = "component_name";
+ public static final String PARAM_HOST_NAME = "host_name";
+ public static final String PARAM_FILE_NAME = "file_name";
+ public static final String PARAM_BUNDLE_ID = "bundle_id";
+ public static final String PARAM_SELECT_COMP = "selectComp";
+ public static final String PARAM_LEVEL = "level";
+ public static final String PARAM_TREE_PARAMS = "treeParams";
+
+ public static final String PARAM_HOST_LOG_FILE = "hostLogFile";
+ public static final String PARAM_COMPONENT_LOG_FILE = "compLogFile";
+ public static final String PARAM_ID = "id";
+ public static final String PARAM_SCROLL_TYPE = "scrollType";
+ public static final String PARAM_NUMBER_ROWS = "numberRows";
+
+ public static final String PARAM_FORMAT = "format";
+ public static final String PARAM_UTC_OFFSET = "utcOffset";
+ public static final String PARAM_KEYWORD = "keyword";
+ public static final String PARAM_SOURCE_LOG_ID = "sourceLogId";
+ public static final String PARAM_KEYWORD_TYPE = "keywordType";
+ public static final String PARAM_TOKEN = "token";
+
+ public static final String PARAM_USER_NAME = "username";
+ public static final String PARAM_FILTER_NAME = "filtername";
+ public static final String PARAM_ROW_TYPE = "rowtype";
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonAuditLogRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonAuditLogRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonAuditLogRequestConverter.java
index 798bd47..e25eb51 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonAuditLogRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonAuditLogRequestConverter.java
@@ -32,15 +32,15 @@ public abstract class AbstractCommonAuditLogRequestConverter<SOURCE extends Base
RESULT criteria = createCriteria(request);
criteria.addParam("q", request.getQuery());
criteria.addParam("columnQuery", StringEscapeUtils.unescapeXml(request.getColumnQuery()));
- criteria.addParam("iMessage", StringEscapeUtils.unescapeXml(request.getiMessage()));
criteria.addParam("gEMessage", StringEscapeUtils.unescapeXml(request.getgEMessage()));
- criteria.addParam("eMessage", StringEscapeUtils.unescapeXml(request.getgEMessage()));
- criteria.addParam("includeString", request.getMustBe());
- criteria.addParam("unselectComp", request.getMustNot());
- criteria.addParam("excludeQuery", StringEscapeUtils.unescapeXml(request.getExcludeQuery()));
- criteria.addParam("includeQuery", StringEscapeUtils.unescapeXml(request.getIncludeQuery()));
- criteria.addParam("startTime", request.getFrom());
- criteria.addParam("endTime", request.getTo());
+ criteria.setIncludeMessage(StringEscapeUtils.unescapeXml(request.getiMessage()));
+ criteria.setExcludeMessage(StringEscapeUtils.unescapeXml(request.getgEMessage()));
+ criteria.setMustBe(request.getMustBe());
+ criteria.setMustNot(request.getMustNot());
+ criteria.setExcludeQuery(StringEscapeUtils.unescapeXml(request.getExcludeQuery()));
+ criteria.setIncludeQuery(StringEscapeUtils.unescapeXml(request.getIncludeQuery()));
+ criteria.setStartTime(request.getFrom());
+ criteria.setEndTime(request.getTo());
return criteria;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonSearchRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonSearchRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonSearchRequestConverter.java
index 3fd07e2..ea2c28a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonSearchRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonSearchRequestConverter.java
@@ -21,10 +21,12 @@ package org.apache.ambari.logsearch.query.converter;
import org.apache.ambari.logsearch.model.request.impl.CommonSearchRequest;
import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
import org.apache.commons.lang.StringUtils;
-import org.springframework.core.convert.converter.Converter;
+
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_GLOBAL_END_TIME;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_GLOBAL_START_TIME;
public abstract class AbstractCommonSearchRequestConverter<SOURCE extends CommonSearchRequest, RESULT extends CommonSearchCriteria>
- implements Converter<SOURCE, RESULT> {
+ extends AbstractConverterAware<SOURCE, RESULT> {
@Override
public RESULT convert(SOURCE source) {
@@ -43,11 +45,11 @@ public abstract class AbstractCommonSearchRequestConverter<SOURCE extends Common
criteria.setSortType(request.getSortType());
if (StringUtils.isNotEmpty(request.getStartTime())){
criteria.setGlobalStartTime(request.getStartTime());
- criteria.getUrlParamMap().put("globalStartTime", request.getStartTime());
+ criteria.getUrlParamMap().put(PARAM_GLOBAL_START_TIME, request.getStartTime());
}
if (StringUtils.isNotEmpty(request.getEndTime())){
criteria.setGlobalEndTime(request.getEndTime());
- criteria.getUrlParamMap().put("globalEndTime", request.getEndTime());
+ criteria.getUrlParamMap().put(PARAM_GLOBAL_END_TIME, request.getEndTime());
}
}
}
[05/10] ambari git commit: AMBARI-18246. Clean up Log Feeder (Miklos
Gergely via oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/LogsearchReaderFactory.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/LogsearchReaderFactory.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/LogsearchReaderFactory.java
index c57c028..5fc2e14 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/LogsearchReaderFactory.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/LogsearchReaderFactory.java
@@ -27,17 +27,15 @@ import org.apache.log4j.Logger;
public enum LogsearchReaderFactory {
INSTANCE;
- private static Logger logger = Logger
- .getLogger(LogsearchReaderFactory.class);
+ private static final Logger LOG = Logger.getLogger(LogsearchReaderFactory.class);
public Reader getReader(File file) throws FileNotFoundException {
- logger.debug("Inside reader factory for file:" + file);
+ LOG.debug("Inside reader factory for file:" + file);
if (GZIPReader.isValidFile(file.getAbsolutePath())) {
- logger.info("Reading file " + file + " as gzip file");
+ LOG.info("Reading file " + file + " as gzip file");
return new GZIPReader(file.getAbsolutePath());
} else {
return new FileReader(file);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
deleted file mode 100644
index ae0cfc0..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.logconfig;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.TimeZone;
-
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logfeeder.util.SolrUtil;
-import org.apache.ambari.logfeeder.view.VLogfeederFilter;
-import org.apache.ambari.logfeeder.view.VLogfeederFilterWrapper;
-import org.apache.log4j.Logger;
-
-public class FetchConfigFromSolr extends Thread {
- private static Logger logger = Logger.getLogger(FetchConfigFromSolr.class);
- private static VLogfeederFilterWrapper logfeederFilterWrapper = null;
- private static int solrConfigInterval = 5;// 5 sec;
- private static long delay;
- private static String endTimeDateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSS";//2016-04-05T04:30:00.000Z
- private static String sysTimeZone = "GMT";
-
- FetchConfigFromSolr(boolean isDaemon) {
- this.setName(this.getClass().getSimpleName());
- this.setDaemon(isDaemon);
- }
-
- @Override
- public void run() {
- String zkConnectString = LogFeederUtil.getStringProperty("logfeeder.solr.zk_connect_string");
- String solrUrl = LogFeederUtil.getStringProperty("logfeeder.solr.url");
- if ((zkConnectString == null || zkConnectString.trim().length() == 0 )
- && (solrUrl == null || solrUrl.trim().length() == 0)) {
- logger.warn("Neither Solr ZK Connect String nor solr Uril for UserConfig/History is set." +
- "Won't look for level configuration from Solr.");
- return;
- }
- solrConfigInterval = LogFeederUtil.getIntProperty("logfeeder.solr.config.interval", solrConfigInterval);
- delay = 1000 * solrConfigInterval;
- do {
- logger.debug("Updating config from solr after every " + solrConfigInterval + " sec.");
- pullConfigFromSolr();
- try {
- Thread.sleep(delay);
- } catch (InterruptedException e) {
- logger.error(e.getLocalizedMessage(), e.getCause());
- }
- } while (true);
- }
-
- private synchronized void pullConfigFromSolr() {
- SolrUtil solrUtil = SolrUtil.getInstance();
- if(solrUtil!=null){
- HashMap<String, Object> configDocMap = solrUtil.getConfigDoc();
- if (configDocMap != null) {
- String configJson = (String) configDocMap.get(LogFeederConstants.VALUES);
- if (configJson != null) {
- logfeederFilterWrapper = LogFeederUtil.getGson().fromJson(configJson, VLogfeederFilterWrapper.class);
- }
- }
- }
- }
-
- private static boolean isFilterExpired(VLogfeederFilter logfeederFilter) {
- boolean isFilterExpired = false;// default is false
- if (logfeederFilter != null) {
- Date filterEndDate = parseFilterExpireDate(logfeederFilter);
- if (filterEndDate != null) {
- Date currentDate = getCurrentDate();
- if (currentDate.compareTo(filterEndDate) >= 0) {
- logger.debug("Filter for Component :" + logfeederFilter.getLabel() + " and Hosts :"
- + listToStr(logfeederFilter.getHosts()) + "Filter is expired because of filter endTime : "
- + dateToStr(filterEndDate) + " is older than currentTime :" + dateToStr(currentDate));
- isFilterExpired = true;
- }
- }
- }
- return isFilterExpired;
- }
-
- private static String dateToStr(Date date) {
- if (date == null) {
- return "";
- }
- SimpleDateFormat formatter = new SimpleDateFormat(endTimeDateFormat);
- TimeZone timeZone = TimeZone.getTimeZone(sysTimeZone);
- formatter.setTimeZone(timeZone);
- return formatter.format(date);
- }
-
- private static Date parseFilterExpireDate(VLogfeederFilter vLogfeederFilter) {
- String expiryTime = vLogfeederFilter.getExpiryTime();
- if (expiryTime != null && !expiryTime.isEmpty()) {
- SimpleDateFormat formatter = new SimpleDateFormat(endTimeDateFormat);
- TimeZone timeZone = TimeZone.getTimeZone(sysTimeZone);
- formatter.setTimeZone(timeZone);
- try {
- return formatter.parse(expiryTime);
- } catch (ParseException e) {
- logger.error("Filter have invalid ExpiryTime : " + expiryTime + " for component :" + vLogfeederFilter.getLabel()
- + " and hosts :" + listToStr(vLogfeederFilter.getHosts()));
- }
- }
- return null;
- }
-
- public static List<String> getAllowedLevels(String hostName, VLogfeederFilter componentFilter) {
- String componentName = componentFilter.getLabel();
- List<String> hosts = componentFilter.getHosts();
- List<String> defaultLevels = componentFilter.getDefaultLevels();
- List<String> overrideLevels = componentFilter.getOverrideLevels();
- String expiryTime=componentFilter.getExpiryTime();
- //check is user override or not
- if ((expiryTime != null && !expiryTime.isEmpty())
- || (overrideLevels != null && !overrideLevels.isEmpty())
- || (hosts != null && !hosts.isEmpty())) {
- if (hosts == null || hosts.isEmpty()) {
- // hosts list is empty or null consider it apply on all hosts
- hosts.add(LogFeederConstants.ALL);
- }
- if (LogFeederUtil.isListContains(hosts, hostName, false)) {
- if (isFilterExpired(componentFilter)) {
- logger.debug("Filter for component " + componentName + " and host :"
- + hostName + " is expired at " + componentFilter.getExpiryTime());
- return defaultLevels;
- } else {
- return overrideLevels;
- }
- }
- }
- return defaultLevels;
- }
-
- public static boolean isFilterAvailable() {
- return logfeederFilterWrapper != null;
- }
-
- public static VLogfeederFilter findComponentFilter(String componentName) {
- if (logfeederFilterWrapper != null) {
- HashMap<String, VLogfeederFilter> filter = logfeederFilterWrapper.getFilter();
- if (filter != null) {
- VLogfeederFilter componentFilter = filter.get(componentName);
- if (componentFilter != null) {
- return componentFilter;
- }
- }
- }
- logger.trace("Filter is not there for component :" + componentName);
- return null;
- }
-
-
- public static Date getCurrentDate() {
- TimeZone.setDefault(TimeZone.getTimeZone(sysTimeZone));
- Date date = new Date();
- return date;
- }
-
- public static String listToStr(List<String> strList) {
- StringBuilder out = new StringBuilder("[");
- if (strList != null) {
- int counter = 0;
- for (Object o : strList) {
- if (counter > 0) {
- out.append(",");
- }
- out.append(o.toString());
- counter++;
- }
- }
- out.append("]");
- return out.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FilterLogData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FilterLogData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FilterLogData.java
new file mode 100644
index 0000000..801a289
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FilterLogData.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.logconfig;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+
+/**
+ * Read configuration from solr and filter the log
+ */
+public enum FilterLogData {
+ INSTANCE;
+
+ private static final Logger LOG = Logger.getLogger(FilterLogData.class);
+
+ private static final boolean DEFAULT_VALUE = true;
+
+ public boolean isAllowed(String jsonBlock) {
+ if (StringUtils.isEmpty(jsonBlock)) {
+ return DEFAULT_VALUE;
+ }
+ Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
+ return isAllowed(jsonObj);
+ }
+
+ public boolean isAllowed(Map<String, Object> jsonObj) {
+ boolean isAllowed = applyFilter(jsonObj);
+ if (!isAllowed) {
+ LOG.trace("Filter block the content :" + LogFeederUtil.getGson().toJson(jsonObj));
+ }
+ return isAllowed;
+ }
+
+
+ private boolean applyFilter(Map<String, Object> jsonObj) {
+ if (MapUtils.isEmpty(jsonObj)) {
+ LOG.warn("Output jsonobj is empty");
+ return DEFAULT_VALUE;
+ }
+
+ String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST);
+ String componentName = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
+ String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
+ if (StringUtils.isNotBlank(hostName) && StringUtils.isNotBlank(componentName) && StringUtils.isNotBlank(level)) {
+ LogFeederFilter componentFilter = LogConfigHandler.findComponentFilter(componentName);
+ if (componentFilter == null) {
+ return DEFAULT_VALUE;
+ }
+ List<String> allowedLevels = LogConfigHandler.getAllowedLevels(hostName, componentFilter);
+ if (CollectionUtils.isEmpty(allowedLevels)) {
+ allowedLevels.add(LogFeederConstants.ALL);
+ }
+ return LogFeederUtil.isListContains(allowedLevels, level, false);
+ }
+ else {
+ return DEFAULT_VALUE;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigFetcher.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigFetcher.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigFetcher.java
new file mode 100644
index 0000000..12c744c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigFetcher.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.logconfig;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.SolrRequest.METHOD;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+
+public class LogConfigFetcher {
+ private static final Logger LOG = Logger.getLogger(LogConfigFetcher.class);
+
+ private static LogConfigFetcher instance;
+ public synchronized static LogConfigFetcher getInstance() {
+ if (instance == null) {
+ try {
+ instance = new LogConfigFetcher();
+ } catch (Exception e) {
+ String logMessageKey = LogConfigFetcher.class.getSimpleName() + "_SOLR_UTIL";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Error constructing solrUtil", e, LOG, Level.WARN);
+ }
+ }
+ return instance;
+ }
+
+ private SolrClient solrClient;
+
+ private String solrDetail = "";
+
+ public LogConfigFetcher() throws Exception {
+ String url = LogFeederUtil.getStringProperty("logfeeder.solr.url");
+ String zkConnectString = LogFeederUtil.getStringProperty("logfeeder.solr.zk_connect_string");
+ String collection = LogFeederUtil.getStringProperty("logfeeder.solr.core.config.name", "history");
+ connectToSolr(url, zkConnectString, collection);
+ }
+
+ private SolrClient connectToSolr(String url, String zkConnectString, String collection) throws Exception {
+ solrDetail = "zkConnectString=" + zkConnectString + ", collection=" + collection + ", url=" + url;
+
+ LOG.info("connectToSolr() " + solrDetail);
+ if (StringUtils.isEmpty(collection)) {
+ throw new Exception("For solr, collection name is mandatory. " + solrDetail);
+ }
+
+ if (StringUtils.isEmpty(zkConnectString) && StringUtils.isBlank(url))
+ throw new Exception("Both zkConnectString and URL are empty. zkConnectString=" + zkConnectString + ", collection=" +
+ collection + ", url=" + url);
+
+ if (StringUtils.isNotEmpty(zkConnectString)) {
+ solrDetail = "zkConnectString=" + zkConnectString + ", collection=" + collection;
+ LOG.info("Using zookeepr. " + solrDetail);
+ CloudSolrClient solrClouldClient = new CloudSolrClient(zkConnectString);
+ solrClouldClient.setDefaultCollection(collection);
+ solrClient = solrClouldClient;
+ checkSolrStatus(3 * 60 * 1000);
+ } else {
+ solrDetail = "collection=" + collection + ", url=" + url;
+ String collectionURL = url + "/" + collection;
+ LOG.info("Connecting to solr : " + collectionURL);
+ solrClient = new HttpSolrClient(collectionURL);
+ }
+ return solrClient;
+ }
+
+ private boolean checkSolrStatus(int waitDurationMS) {
+ boolean status = false;
+ try {
+ long beginTimeMS = System.currentTimeMillis();
+ long waitIntervalMS = 2000;
+ int pingCount = 0;
+ while (true) {
+ pingCount++;
+ CollectionAdminResponse response = null;
+ try {
+ CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
+ response = colListReq.process(solrClient);
+ } catch (Exception ex) {
+ LOG.error("Con't connect to Solr. solrDetail=" + solrDetail, ex);
+ }
+ if (response != null && response.getStatus() == 0) {
+ LOG.info("Solr getCollections() is success. solr=" + solrDetail);
+ status = true;
+ break;
+ }
+ if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
+ LOG.error("Solr is not reachable even after " + (System.currentTimeMillis() - beginTimeMS)
+ + " ms. If you are using alias, then you might have to restart LogSearch after Solr is up and running. solr="
+ + solrDetail + ", response=" + response);
+ break;
+ } else {
+ LOG.warn("Solr is not reachable yet. getCollections() attempt count=" + pingCount + ". Will sleep for " +
+ waitIntervalMS + " ms and try again." + " solr=" + solrDetail + ", response=" + response);
+ }
+ Thread.sleep(waitIntervalMS);
+ }
+ } catch (Throwable t) {
+ LOG.error("Seems Solr is not up. solrDetail=" + solrDetail, t);
+ }
+ return status;
+ }
+
+ public Map<String, Object> getConfigDoc() {
+ HashMap<String, Object> configMap = new HashMap<String, Object>();
+ SolrQuery solrQuery = new SolrQuery();
+ solrQuery.setQuery("*:*");
+ String fq = LogFeederConstants.ROW_TYPE + ":" + LogFeederConstants.LOGFEEDER_FILTER_NAME;
+ solrQuery.setFilterQueries(fq);
+ try {
+ QueryResponse response = process(solrQuery);
+ if (response != null) {
+ SolrDocumentList documentList = response.getResults();
+ if (CollectionUtils.isNotEmpty(documentList)) {
+ SolrDocument configDoc = documentList.get(0);
+ String configJson = LogFeederUtil.getGson().toJson(configDoc);
+ configMap = (HashMap<String, Object>) LogFeederUtil.toJSONObject(configJson);
+ }
+ }
+ } catch (Exception e) {
+ String logMessageKey = this.getClass().getSimpleName() + "_FETCH_FILTER_CONFIG_ERROR";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Error getting filter config from solr", e, LOG, Level.ERROR);
+ }
+ return configMap;
+ }
+
+ private QueryResponse process(SolrQuery solrQuery) throws SolrServerException, IOException, SolrException {
+ if (solrClient != null) {
+ QueryResponse queryResponse = solrClient.query(solrQuery, METHOD.POST);
+ return queryResponse;
+ } else {
+ LOG.error("solrClient can't be null");
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandler.java
new file mode 100644
index 0000000..4f52b0b
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandler.java
@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.logconfig;
+
+import java.text.DateFormat;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+
+public class LogConfigHandler extends Thread {
+ private static final Logger LOG = Logger.getLogger(LogConfigHandler.class);
+
+ private static final int DEFAULT_SOLR_CONFIG_INTERVAL = 5;
+ private static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS";
+ private static final String TIMEZONE = "GMT";
+
+ static {
+ TimeZone.setDefault(TimeZone.getTimeZone(TIMEZONE));
+ }
+
+ private static ThreadLocal<DateFormat> formatter = new ThreadLocal<DateFormat>() {
+ protected DateFormat initialValue() {
+ SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_FORMAT);
+ dateFormat.setTimeZone(TimeZone.getTimeZone(TIMEZONE));
+ return dateFormat;
+ }
+ };
+
+ private static LogFeederFilterWrapper logFeederFilterWrapper;
+
+ private static boolean running = false;
+
+ public static void handleConfig() {
+ boolean filterEnable = LogFeederUtil.getBooleanProperty("logfeeder.log.filter.enable", false);
+ if (!filterEnable) {
+ LOG.info("Logfeeder filter Scheduler is disabled.");
+ return;
+ }
+ if (!running) {
+ new LogConfigHandler().start();
+ running = true;
+ LOG.info("Logfeeder Filter Thread started!");
+ } else {
+ LOG.warn("Logfeeder Filter Thread is already running.");
+ }
+ }
+
+ private LogConfigHandler() {
+ setName(getClass().getSimpleName());
+ setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ String zkConnectString = LogFeederUtil.getStringProperty("logfeeder.solr.zk_connect_string");
+ String solrUrl = LogFeederUtil.getStringProperty("logfeeder.solr.url");
+ if (StringUtils.isBlank(zkConnectString) && StringUtils.isBlank(solrUrl)) {
+ LOG.warn("Neither Solr ZK Connect String nor solr Url for UserConfig/History is set." +
+ "Won't look for level configuration from Solr.");
+ return;
+ }
+
+ int solrConfigInterval = LogFeederUtil.getIntProperty("logfeeder.solr.config.interval", DEFAULT_SOLR_CONFIG_INTERVAL);
+ do {
+ LOG.debug("Updating config from solr after every " + solrConfigInterval + " sec.");
+ fetchConfig();
+ try {
+ Thread.sleep(1000 * solrConfigInterval);
+ } catch (InterruptedException e) {
+ LOG.error(e.getLocalizedMessage(), e.getCause());
+ }
+ } while (true);
+ }
+
+ private synchronized void fetchConfig() {
+ LogConfigFetcher fetcher = LogConfigFetcher.getInstance();
+ if (fetcher != null) {
+ Map<String, Object> configDocMap = fetcher.getConfigDoc();
+ String configJson = (String) configDocMap.get(LogFeederConstants.VALUES);
+ if (configJson != null) {
+ logFeederFilterWrapper = LogFeederUtil.getGson().fromJson(configJson, LogFeederFilterWrapper.class);
+ }
+ }
+ }
+
+ public static boolean isFilterAvailable() {
+ return logFeederFilterWrapper != null;
+ }
+
+ public static List<String> getAllowedLevels(String hostName, LogFeederFilter componentFilter) {
+ String componentName = componentFilter.getLabel();
+ List<String> hosts = componentFilter.getHosts();
+ List<String> defaultLevels = componentFilter.getDefaultLevels();
+ List<String> overrideLevels = componentFilter.getOverrideLevels();
+ String expiryTime = componentFilter.getExpiryTime();
+
+ // check is user override or not
+ if (StringUtils.isNotEmpty(expiryTime) || CollectionUtils.isNotEmpty(overrideLevels) || CollectionUtils.isNotEmpty(hosts)) {
+ if (CollectionUtils.isEmpty(hosts)) { // hosts list is empty or null consider it apply on all hosts
+ hosts.add(LogFeederConstants.ALL);
+ }
+
+ if (LogFeederUtil.isListContains(hosts, hostName, false)) {
+ if (isFilterExpired(componentFilter)) {
+ LOG.debug("Filter for component " + componentName + " and host :" + hostName + " is expired at " +
+ componentFilter.getExpiryTime());
+ return defaultLevels;
+ } else {
+ return overrideLevels;
+ }
+ }
+ }
+ return defaultLevels;
+ }
+
+ private static boolean isFilterExpired(LogFeederFilter logfeederFilter) {
+ if (logfeederFilter == null)
+ return false;
+
+ Date filterEndDate = parseFilterExpireDate(logfeederFilter);
+ if (filterEndDate == null) {
+ return false;
+ }
+
+ Date currentDate = new Date();
+ if (!currentDate.before(filterEndDate)) {
+ LOG.debug("Filter for Component :" + logfeederFilter.getLabel() + " and Hosts : [" +
+ StringUtils.join(logfeederFilter.getHosts(), ',') + "] is expired because of filter endTime : " +
+ formatter.get().format(filterEndDate) + " is older than currentTime :" + formatter.get().format(currentDate));
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ private static Date parseFilterExpireDate(LogFeederFilter vLogfeederFilter) {
+ String expiryTime = vLogfeederFilter.getExpiryTime();
+ if (StringUtils.isNotEmpty(expiryTime)) {
+ try {
+ return formatter.get().parse(expiryTime);
+ } catch (ParseException e) {
+ LOG.error("Filter have invalid ExpiryTime : " + expiryTime + " for component :" + vLogfeederFilter.getLabel()
+ + " and hosts : [" + StringUtils.join(vLogfeederFilter.getHosts(), ',') + "]");
+ }
+ }
+ return null;
+ }
+
+ public static LogFeederFilter findComponentFilter(String componentName) {
+ if (logFeederFilterWrapper != null) {
+ HashMap<String, LogFeederFilter> filter = logFeederFilterWrapper.getFilter();
+ if (filter != null) {
+ LogFeederFilter componentFilter = filter.get(componentName);
+ if (componentFilter != null) {
+ return componentFilter;
+ }
+ }
+ }
+ LOG.trace("Filter is not there for component :" + componentName);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
deleted file mode 100644
index 09673a0..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederConstants.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.logconfig;
-
-public class LogFeederConstants {
-
- public static final String ALL = "all";
- public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config";
- public static final String LOG_LEVEL_UNKNOWN = "UNKNOWN";
- // solr fields
- public static final String SOLR_LEVEL = "level";
- public static final String SOLR_COMPONENT = "type";
- public static final String SOLR_HOST = "host";
-
- // UserConfig Constants History
- public static final String VALUES = "jsons";
- public static final String ROW_TYPE = "rowtype";
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilter.java
new file mode 100644
index 0000000..60c8ae8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilter.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.logconfig;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LogFeederFilter {
+
+ private String label;
+ private List<String> hosts;
+ private List<String> defaultLevels;
+ private List<String> overrideLevels;
+ private String expiryTime;
+
+ public LogFeederFilter() {
+ hosts = new ArrayList<String>();
+ defaultLevels = new ArrayList<String>();
+ overrideLevels = new ArrayList<String>();
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ public List<String> getHosts() {
+ return hosts;
+ }
+
+ public void setHosts(List<String> hosts) {
+ this.hosts = hosts;
+ }
+
+ public List<String> getDefaultLevels() {
+ return defaultLevels;
+ }
+
+ public void setDefaultLevels(List<String> defaultLevels) {
+ this.defaultLevels = defaultLevels;
+ }
+
+ public List<String> getOverrideLevels() {
+ return overrideLevels;
+ }
+
+ public void setOverrideLevels(List<String> overrideLevels) {
+ this.overrideLevels = overrideLevels;
+ }
+
+ public String getExpiryTime() {
+ return expiryTime;
+ }
+
+ public void setExpiryTime(String expiryTime) {
+ this.expiryTime = expiryTime;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilterWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilterWrapper.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilterWrapper.java
new file mode 100644
index 0000000..9199cd3
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogFeederFilterWrapper.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.logconfig;
+
+import java.util.HashMap;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class LogFeederFilterWrapper {
+
+ private HashMap<String, LogFeederFilter> filter;
+ private String id;
+
+ public HashMap<String, LogFeederFilter> getFilter() {
+ return filter;
+ }
+
+ public void setFilter(HashMap<String, LogFeederFilter> filter) {
+ this.filter = filter;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
deleted file mode 100644
index bc807193..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.logconfig;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.log4j.Logger;
-
-public enum LogfeederScheduler {
-
- INSTANCE;
-
- private Logger logger = Logger.getLogger(LogfeederScheduler.class);
-
- private static boolean running = false;
-
- public synchronized void start() {
- boolean filterEnable = LogFeederUtil.getBooleanProperty("logfeeder.log.filter.enable", false);
- if (!filterEnable) {
- logger.info("Logfeeder filter Scheduler is disabled.");
- return;
- }
- if (!running) {
- for (Thread thread : getThreadList()) {
- thread.start();
- }
- running = true;
- logger.info("Logfeeder Scheduler started!");
- } else {
- logger.warn("Logfeeder Scheduler is already running.");
- }
- }
-
- private List<Thread> getThreadList() {
- List<Thread> tasks = new ArrayList<Thread>();
- Thread configMonitor = new FetchConfigFromSolr(true);
- tasks.add(configMonitor);
- return tasks;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
deleted file mode 100644
index b5e4eb3..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.logconfig.filter;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.ambari.logfeeder.logconfig.FetchConfigFromSolr;
-import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logfeeder.view.VLogfeederFilter;
-import org.apache.log4j.Logger;
-
-class ApplyLogFilter extends DefaultDataFilter {
-
- private static Logger logger = Logger.getLogger(ApplyLogFilter.class);
-
- @Override
- public boolean applyFilter(Map<String, Object> jsonObj, boolean defaultValue) {
- if (isEmpty(jsonObj)) {
- logger.warn("Output jsonobj is empty");
- return defaultValue;
- }
- String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST);
- if (isNotEmpty(hostName)) {
- String componentName = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT);
- if (isNotEmpty(componentName)) {
- String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL);
- if (isNotEmpty(level)) {
- VLogfeederFilter componentFilter = FetchConfigFromSolr.findComponentFilter(componentName);
- if (componentFilter == null) {
- return defaultValue;
- }
- List<String> allowedLevels = FetchConfigFromSolr.getAllowedLevels(
- hostName, componentFilter);
- if (allowedLevels == null || allowedLevels.isEmpty()) {
- allowedLevels.add(LogFeederConstants.ALL);
- }
- return LogFeederUtil.isListContains(allowedLevels, level, false);
- }
- }
- }
- return defaultValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
deleted file mode 100644
index 04d2ca4..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.logconfig.filter;
-
-import java.util.Map;
-
-/**
- * Default filter to allow everything
- */
-class DefaultDataFilter {
- public boolean applyFilter(Map<String, Object> outputJsonObj, boolean defaultValue) {
- return defaultValue;
- }
-
- protected boolean isEmpty(Map<String, Object> map) {
- if (map == null || map.isEmpty()) {
- return true;
- }
- return false;
- }
-
- protected boolean isEmpty(String str) {
- if (str == null || str.trim().isEmpty()) {
- return true;
- }
- return false;
- }
-
- protected boolean isNotEmpty(String str) {
- return !isEmpty(str);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
deleted file mode 100644
index 3a8eae9..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.logconfig.filter;
-
-import java.util.Map;
-
-import org.apache.ambari.logfeeder.logconfig.filter.ApplyLogFilter;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.log4j.Logger;
-
-/**
- * Read configuration from solr and filter the log
- */
-public enum FilterLogData {
- INSTANCE;
- private ApplyLogFilter applyLogFilter = new ApplyLogFilter();
- private static Logger logger = Logger.getLogger(FilterLogData.class);
- // by default allow every log
- boolean defaultValue = true;
-
- public boolean isAllowed(String jsonBlock) {
- if (jsonBlock == null || jsonBlock.isEmpty()) {
- return defaultValue;
- }
- Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock);
- return isAllowed(jsonObj);
- }
-
- public boolean isAllowed(Map<String, Object> jsonObj) {
- boolean isAllowed = applyLogFilter.applyFilter(jsonObj, defaultValue);
- if (!isAllowed) {
- logger.trace("Filter block the content :" + LogFeederUtil.getGson().toJson(jsonObj));
- }
- return isAllowed;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/Mapper.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/Mapper.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/Mapper.java
index 906dd25..96709c0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/Mapper.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/Mapper.java
@@ -26,22 +26,18 @@ public abstract class Mapper {
protected String fieldName;
private String mapClassCode;
- public boolean init(String inputDesc, String fieldName,
- String mapClassCode, Object mapConfigs) {
+ public abstract boolean init(String inputDesc, String fieldName, String mapClassCode, Object mapConfigs);
+
+ protected void init(String inputDesc, String fieldName, String mapClassCode) {
this.inputDesc = inputDesc;
this.fieldName = fieldName;
this.mapClassCode = mapClassCode;
- return true;
}
- public Object apply(Map<String, Object> jsonObj, Object value) {
- return value;
- }
+ public abstract Object apply(Map<String, Object> jsonObj, Object value);
@Override
public String toString() {
- return "mapClass=" + mapClassCode + ", input=" + inputDesc
- + ", fieldName=" + fieldName;
+ return "mapClass=" + mapClassCode + ", input=" + inputDesc + ", fieldName=" + fieldName;
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
index 6dbf8be..eb3ae01 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperDate.java
@@ -31,31 +31,29 @@ import org.apache.log4j.Level;
import org.apache.log4j.Logger;
public class MapperDate extends Mapper {
- private static final Logger logger = Logger.getLogger(MapperDate.class);
+ private static final Logger LOG = Logger.getLogger(MapperDate.class);
private SimpleDateFormat targetDateFormatter = null;
private boolean isEpoch = false;
private SimpleDateFormat srcDateFormatter=null;
@Override
- public boolean init(String inputDesc, String fieldName,
- String mapClassCode, Object mapConfigs) {
- super.init(inputDesc, fieldName, mapClassCode, mapConfigs);
+ public boolean init(String inputDesc, String fieldName, String mapClassCode, Object mapConfigs) {
+ init(inputDesc, fieldName, mapClassCode);
if (!(mapConfigs instanceof Map)) {
- logger.fatal("Can't initialize object. mapConfigs class is not of type Map. "
- + mapConfigs.getClass().getName()
- + ", map="
- + this.toString());
+ LOG.fatal("Can't initialize object. mapConfigs class is not of type Map. " + mapConfigs.getClass().getName() +
+ ", map=" + this);
return false;
}
+
@SuppressWarnings("unchecked")
Map<String, Object> mapObjects = (Map<String, Object>) mapConfigs;
String targetDateFormat = (String) mapObjects.get("target_date_pattern");
String srcDateFormat = (String) mapObjects.get("src_date_pattern");
if (StringUtils.isEmpty(targetDateFormat)) {
- logger.fatal("Date format for map is empty. " + this.toString());
+ LOG.fatal("Date format for map is empty. " + this);
} else {
- logger.info("Date mapper format is " + targetDateFormat);
+ LOG.info("Date mapper format is " + targetDateFormat);
if (targetDateFormat.equalsIgnoreCase("epoch")) {
isEpoch = true;
@@ -68,8 +66,7 @@ public class MapperDate extends Mapper {
}
return true;
} catch (Throwable ex) {
- logger.fatal("Error creating date format. format="
- + targetDateFormat + ". " + this.toString());
+ LOG.fatal("Error creating date format. format=" + targetDateFormat + ". " + this.toString());
}
}
}
@@ -84,7 +81,7 @@ public class MapperDate extends Mapper {
long ms = Long.parseLong(value.toString()) * 1000;
value = new Date(ms);
} else if (targetDateFormatter != null) {
- if(srcDateFormatter!=null){
+ if (srcDateFormatter != null) {
Date srcDate = srcDateFormatter.parse(value.toString());
//set year in src_date when src_date does not have year component
if (!srcDateFormatter.toPattern().contains("yy")) {
@@ -108,12 +105,9 @@ public class MapperDate extends Mapper {
}
jsonObj.put(fieldName, value);
} catch (Throwable t) {
- LogFeederUtil.logErrorMessageByInterval(this.getClass()
- .getSimpleName() + ":apply",
- "Error applying date transformation. isEpoch="
- + isEpoch + ", targetateFormat=" + (targetDateFormatter!=null ?targetDateFormatter.toPattern():"")
- + ", value=" + value + ". " + this.toString(),
- t, logger, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(this.getClass().getSimpleName() + ":apply", "Error applying date transformation." +
+ " isEpoch=" + isEpoch + ", targetateFormat=" + (targetDateFormatter!=null ?targetDateFormatter.toPattern():"")
+ + ", value=" + value + ". " + this.toString(), t, LOG, Level.ERROR);
}
}
return value;
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldName.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldName.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldName.java
index c692a9d..9b6e83c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldName.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldName.java
@@ -30,24 +30,23 @@ import org.apache.log4j.Logger;
* Overrides the value for the field
*/
public class MapperFieldName extends Mapper {
- private static final Logger logger = Logger.getLogger(MapperFieldName.class);
+ private static final Logger LOG = Logger.getLogger(MapperFieldName.class);
private String newValue = null;
@Override
- public boolean init(String inputDesc, String fieldName,
- String mapClassCode, Object mapConfigs) {
- super.init(inputDesc, fieldName, mapClassCode, mapConfigs);
+ public boolean init(String inputDesc, String fieldName, String mapClassCode, Object mapConfigs) {
+ init(inputDesc, fieldName, mapClassCode);
if (!(mapConfigs instanceof Map)) {
- logger.fatal("Can't initialize object. mapConfigs class is not of type Map. "
- + mapConfigs.getClass().getName());
+ LOG.fatal("Can't initialize object. mapConfigs class is not of type Map. " + mapConfigs.getClass().getName());
return false;
}
+
@SuppressWarnings("unchecked")
Map<String, Object> mapObjects = (Map<String, Object>) mapConfigs;
newValue = (String) mapObjects.get("new_fieldname");
if (StringUtils.isEmpty(newValue)) {
- logger.fatal("Map field value is empty.");
+ LOG.fatal("Map field value is empty.");
return false;
}
return true;
@@ -59,12 +58,9 @@ public class MapperFieldName extends Mapper {
jsonObj.remove(fieldName);
jsonObj.put(newValue, value);
} else {
- LogFeederUtil.logErrorMessageByInterval(this.getClass()
- .getSimpleName() + ":apply",
- "New fieldName is null, so transformation is not applied. "
- + this.toString(), null, logger, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(this.getClass().getSimpleName() + ":apply",
+ "New fieldName is null, so transformation is not applied. " + this.toString(), null, LOG, Level.ERROR);
}
return value;
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldValue.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldValue.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldValue.java
index e618261..87cda65 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldValue.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/mapper/MapperFieldValue.java
@@ -30,25 +30,25 @@ import org.apache.log4j.Logger;
* Overrides the value for the field
*/
public class MapperFieldValue extends Mapper {
- private Logger logger = Logger.getLogger(MapperFieldValue.class);
+ private static final Logger LOG = Logger.getLogger(MapperFieldValue.class);
+
private String prevValue = null;
private String newValue = null;
@Override
- public boolean init(String inputDesc, String fieldName,
- String mapClassCode, Object mapConfigs) {
- super.init(inputDesc, fieldName, mapClassCode, mapConfigs);
+ public boolean init(String inputDesc, String fieldName, String mapClassCode, Object mapConfigs) {
+ init(inputDesc, fieldName, mapClassCode);
if (!(mapConfigs instanceof Map)) {
- logger.fatal("Can't initialize object. mapConfigs class is not of type Map. "
- + mapConfigs.getClass().getName());
+ LOG.fatal("Can't initialize object. mapConfigs class is not of type Map. " + mapConfigs.getClass().getName());
return false;
}
+
@SuppressWarnings("unchecked")
Map<String, Object> mapObjects = (Map<String, Object>) mapConfigs;
prevValue = (String) mapObjects.get("pre_value");
newValue = (String) mapObjects.get("post_value");
if (StringUtils.isEmpty(newValue)) {
- logger.fatal("Map field value is empty.");
+ LOG.fatal("Map field value is empty.");
return false;
}
return true;
@@ -56,20 +56,15 @@ public class MapperFieldValue extends Mapper {
@Override
public Object apply(Map<String, Object> jsonObj, Object value) {
- if (newValue != null) {
- if (prevValue != null) {
- if (prevValue.equalsIgnoreCase(value.toString())) {
- value = newValue;
- jsonObj.put(fieldName, value);
- }
+ if (newValue != null && prevValue != null) {
+ if (prevValue.equalsIgnoreCase(value.toString())) {
+ value = newValue;
+ jsonObj.put(fieldName, value);
}
} else {
- LogFeederUtil.logErrorMessageByInterval(
- this.getClass().getSimpleName() + ":apply",
- "New value is null, so transformation is not applied. "
- + this.toString(), null, logger, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(this.getClass().getSimpleName() + ":apply",
+ "New value is null, so transformation is not applied. " + this.toString(), null, LOG, Level.ERROR);
}
return value;
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
index 0a0f4e9..32dfef2 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/LogFeederAMSClient.java
@@ -20,26 +20,26 @@
package org.apache.ambari.logfeeder.metrics;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
import org.apache.log4j.Logger;
// TODO: Refactor for failover
public class LogFeederAMSClient extends AbstractTimelineMetricsSink {
- private static final Logger logger = Logger.getLogger(LogFeederAMSClient.class);
+ private static final Logger LOG = Logger.getLogger(LogFeederAMSClient.class);
private String collectorHosts = null;
public LogFeederAMSClient() {
- collectorHosts = LogFeederUtil
- .getStringProperty("logfeeder.metrics.collector.hosts");
- if (collectorHosts != null && collectorHosts.trim().length() == 0) {
+ collectorHosts = LogFeederUtil.getStringProperty("logfeeder.metrics.collector.hosts");
+ if (StringUtils.isBlank(collectorHosts)) {
collectorHosts = null;
}
if (collectorHosts != null) {
collectorHosts = collectorHosts.trim();
}
- logger.info("AMS collector URL=" + collectorHosts);
+ LOG.info("AMS collector URL=" + collectorHosts);
}
@Override
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricCount.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricCount.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricCount.java
deleted file mode 100644
index abb84c7..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricCount.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.metrics;
-
-public class MetricCount {
- public String metricsName = null;
- public boolean isPointInTime = false;
-
- public long count = 0;
- public long prevLogCount = 0;
- public long prevLogMS = System.currentTimeMillis();
- public long prevPublishCount = 0;
- public int publishCount = 0; // Count of published metrics. Used for first time sending metrics
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricData.java
new file mode 100644
index 0000000..e7f5d37
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricData.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.metrics;
+
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
+
+public class MetricData {
+ public final String metricsName;
+ public final boolean isPointInTime;
+
+ public MetricData(String metricsName, boolean isPointInTime) {
+ this.metricsName = metricsName;
+ this.isPointInTime = isPointInTime;
+ }
+
+ public long value = 0;
+ public long prevPublishValue = 0;
+
+ public long prevLogValue = 0;
+ public long prevLogTime = System.currentTimeMillis();
+
+ public int publishCount = 0; // Number of times the metric was published so far
+
+ @Override
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
new file mode 100644
index 0000000..942c0b4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsManager.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.metrics;
+
+import java.net.InetAddress;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.TreeMap;
+
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
+import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
+import org.apache.log4j.Logger;
+
+public class MetricsManager {
+ private static final Logger LOG = Logger.getLogger(MetricsManager.class);
+
+ private boolean isMetricsEnabled = false;
+ private String nodeHostName = null;
+ private String appId = "logfeeder";
+
+ private long lastPublishTimeMS = 0; // Let's do the first publish immediately
+ private long lastFailedPublishTimeMS = System.currentTimeMillis(); // Reset the clock
+
+ private int publishIntervalMS = 60 * 1000;
+ private int maxMetricsBuffer = 60 * 60 * 1000; // If AMS is down, we should not keep the metrics in memory forever
+ private HashMap<String, TimelineMetric> metricsMap = new HashMap<String, TimelineMetric>();
+ private LogFeederAMSClient amsClient = null;
+
+ public void init() {
+ LOG.info("Initializing MetricsManager()");
+ amsClient = new LogFeederAMSClient();
+
+ if (amsClient.getCollectorUri(null) != null) {
+ findNodeHostName();
+ if (nodeHostName == null) {
+ isMetricsEnabled = false;
+ LOG.error("Failed getting hostname for node. Disabling publishing LogFeeder metrics");
+ } else {
+ isMetricsEnabled = true;
+ LOG.info("LogFeeder Metrics is enabled. Metrics host=" + amsClient.getCollectorUri(null));
+ }
+ } else {
+ LOG.info("LogFeeder Metrics publish is disabled");
+ }
+ }
+
+ private void findNodeHostName() {
+ nodeHostName = LogFeederUtil.getStringProperty("node.hostname");
+ if (nodeHostName == null) {
+ try {
+ nodeHostName = InetAddress.getLocalHost().getHostName();
+ } catch (Throwable e) {
+ LOG.warn("Error getting hostname using InetAddress.getLocalHost().getHostName()", e);
+ }
+ }
+ if (nodeHostName == null) {
+ try {
+ nodeHostName = InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (Throwable e) {
+ LOG.warn("Error getting hostname using InetAddress.getLocalHost().getCanonicalHostName()", e);
+ }
+ }
+ }
+
+ public boolean isMetricsEnabled() {
+ return isMetricsEnabled;
+ }
+
+ public synchronized void useMetrics(List<MetricData> metricsList) {
+ if (!isMetricsEnabled) {
+ return;
+ }
+ LOG.info("useMetrics() metrics.size=" + metricsList.size());
+ long currMS = System.currentTimeMillis();
+
+ gatherMetrics(metricsList, currMS);
+ publishMetrics(currMS);
+ }
+
+ private void gatherMetrics(List<MetricData> metricsList, long currMS) {
+ Long currMSLong = new Long(currMS);
+ for (MetricData metric : metricsList) {
+ if (metric.metricsName == null) {
+ LOG.debug("metric.metricsName is null");
+ continue;
+ }
+ long currCount = metric.value;
+ if (!metric.isPointInTime && metric.publishCount > 0 && currCount <= metric.prevPublishValue) {
+ LOG.debug("Nothing changed. " + metric.metricsName + ", currCount=" + currCount + ", prevPublishCount=" +
+ metric.prevPublishValue);
+ continue;
+ }
+ metric.publishCount++;
+
+ LOG.debug("Ensuring metrics=" + metric.metricsName);
+ TimelineMetric timelineMetric = metricsMap.get(metric.metricsName);
+ if (timelineMetric == null) {
+ LOG.debug("Creating new metric obbject for " + metric.metricsName);
+ timelineMetric = new TimelineMetric();
+ timelineMetric.setMetricName(metric.metricsName);
+ timelineMetric.setHostName(nodeHostName);
+ timelineMetric.setAppId(appId);
+ timelineMetric.setStartTime(currMS);
+ timelineMetric.setType("Long");
+ timelineMetric.setMetricValues(new TreeMap<Long, Double>());
+
+ metricsMap.put(metric.metricsName, timelineMetric);
+ }
+
+ LOG.debug("Adding metrics=" + metric.metricsName);
+ if (metric.isPointInTime) {
+ timelineMetric.getMetricValues().put(currMSLong, new Double(currCount));
+ } else {
+ Double value = timelineMetric.getMetricValues().get(currMSLong);
+ if (value == null) {
+ value = new Double(0);
+ }
+ value += (currCount - metric.prevPublishValue);
+ timelineMetric.getMetricValues().put(currMSLong, value);
+ metric.prevPublishValue = currCount;
+ }
+ }
+ }
+
+ private void publishMetrics(long currMS) {
+ if (!metricsMap.isEmpty() && currMS - lastPublishTimeMS > publishIntervalMS) {
+ try {
+ TimelineMetrics timelineMetrics = new TimelineMetrics();
+ timelineMetrics.setMetrics(new ArrayList<TimelineMetric>(metricsMap.values()));
+ amsClient.emitMetrics(timelineMetrics);
+
+ LOG.info("Published " + timelineMetrics.getMetrics().size() + " metrics to AMS");
+ metricsMap.clear();
+ lastPublishTimeMS = currMS;
+ } catch (Throwable t) {
+ LOG.warn("Error sending metrics to AMS.", t);
+ if (currMS - lastFailedPublishTimeMS > maxMetricsBuffer) {
+ LOG.error("AMS was not sent for last " + maxMetricsBuffer / 1000 +
+ " seconds. Purging it and will start rebuilding it again");
+ metricsMap.clear();
+ lastFailedPublishTimeMS = currMS;
+ }
+ }
+ } else {
+ LOG.info("Not publishing metrics. metrics.size()=" + metricsMap.size() + ", lastPublished=" +
+ (currMS - lastPublishTimeMS) / 1000 + " seconds ago, intervalConfigured=" + publishIntervalMS / 1000);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsMgr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsMgr.java
deleted file mode 100644
index 33397c7..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/metrics/MetricsMgr.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.metrics;
-
-import java.net.InetAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.TreeMap;
-
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.apache.log4j.Logger;
-
-public class MetricsMgr {
- private static final Logger logger = Logger.getLogger(MetricsMgr.class);
-
- private boolean isMetricsEnabled = false;
- private String nodeHostName = null;
- private String appId = "logfeeder";
-
- private long lastPublishTimeMS = 0; // Let's do the first publish immediately
- private long lastFailedPublishTimeMS = System.currentTimeMillis(); // Reset the clock
-
- private int publishIntervalMS = 60 * 1000;
- private int maxMetricsBuffer = 60 * 60 * 1000; // If AMS is down, we should not keep
- // the metrics in memory forever
- private HashMap<String, TimelineMetric> metricsMap = new HashMap<String, TimelineMetric>();
- private LogFeederAMSClient amsClient = null;
-
- public void init() {
- logger.info("Initializing MetricsMgr()");
- amsClient = new LogFeederAMSClient();
-
- if (amsClient.getCollectorUri(null) != null) {
- nodeHostName = LogFeederUtil.getStringProperty("node.hostname");
- if (nodeHostName == null) {
- try {
- nodeHostName = InetAddress.getLocalHost().getHostName();
- } catch (Throwable e) {
- logger.warn(
- "Error getting hostname using InetAddress.getLocalHost().getHostName()",
- e);
- }
- if (nodeHostName == null) {
- try {
- nodeHostName = InetAddress.getLocalHost()
- .getCanonicalHostName();
- } catch (Throwable e) {
- logger.warn(
- "Error getting hostname using InetAddress.getLocalHost().getCanonicalHostName()",
- e);
- }
- }
- }
- if (nodeHostName == null) {
- isMetricsEnabled = false;
- logger.error("Failed getting hostname for node. Disabling publishing LogFeeder metrics");
- } else {
- isMetricsEnabled = true;
- logger.info("LogFeeder Metrics is enabled. Metrics host="
- + amsClient.getCollectorUri(null));
- }
- } else {
- logger.info("LogFeeder Metrics publish is disabled");
- }
- }
-
- public boolean isMetricsEnabled() {
- return isMetricsEnabled;
- }
-
- synchronized public void useMetrics(List<MetricCount> metricsList) {
- if (!isMetricsEnabled) {
- return;
- }
- logger.info("useMetrics() metrics.size=" + metricsList.size());
- long currMS = System.currentTimeMillis();
- Long currMSLong = new Long(currMS);
- for (MetricCount metric : metricsList) {
- if (metric.metricsName == null) {
- logger.debug("metric.metricsName is null");
- // Metrics is not meant to be published
- continue;
- }
- long currCount = metric.count;
- if (!metric.isPointInTime && metric.publishCount > 0
- && currCount <= metric.prevPublishCount) {
- // No new data added, so let's ignore it
- logger.debug("Nothing changed. " + metric.metricsName
- + ", currCount=" + currCount + ", prevPublishCount="
- + metric.prevPublishCount);
- continue;
- }
- metric.publishCount++;
-
- TimelineMetric timelineMetric = metricsMap.get(metric.metricsName);
- if (timelineMetric == null) {
- logger.debug("Creating new metric obbject for "
- + metric.metricsName);
- // First time for this metric
- timelineMetric = new TimelineMetric();
- timelineMetric.setMetricName(metric.metricsName);
- timelineMetric.setHostName(nodeHostName);
- timelineMetric.setAppId(appId);
- timelineMetric.setStartTime(currMS);
- timelineMetric.setType("Long");
- timelineMetric.setMetricValues(new TreeMap<Long, Double>());
-
- metricsMap.put(metric.metricsName, timelineMetric);
- }
- logger.debug("Adding metrics=" + metric.metricsName);
- if (metric.isPointInTime) {
- timelineMetric.getMetricValues().put(currMSLong,
- new Double(currCount));
- } else {
- Double value = timelineMetric.getMetricValues().get(currMSLong);
- if (value == null) {
- value = new Double(0);
- }
- value += (currCount - metric.prevPublishCount);
- timelineMetric.getMetricValues().put(currMSLong, value);
- metric.prevPublishCount = currCount;
- }
- }
-
- if (metricsMap.size() > 0
- && currMS - lastPublishTimeMS > publishIntervalMS) {
- try {
- // Time to publish
- TimelineMetrics timelineMetrics = new TimelineMetrics();
- List<TimelineMetric> timeLineMetricList = new ArrayList<TimelineMetric>();
- timeLineMetricList.addAll(metricsMap.values());
- timelineMetrics.setMetrics(timeLineMetricList);
- amsClient.emitMetrics(timelineMetrics);
- logger.info("Published " + timeLineMetricList.size()
- + " metrics to AMS");
- metricsMap.clear();
- timeLineMetricList.clear();
- lastPublishTimeMS = currMS;
- } catch (Throwable t) {
- logger.warn("Error sending metrics to AMS.", t);
- if (currMS - lastFailedPublishTimeMS > maxMetricsBuffer) {
- logger.error("AMS was not sent for last "
- + maxMetricsBuffer
- / 1000
- + " seconds. Purging it and will start rebuilding it again");
- metricsMap.clear();
- lastFailedPublishTimeMS = currMS;
- }
- }
- } else {
- logger.info("Not publishing metrics. metrics.size()="
- + metricsMap.size() + ", lastPublished="
- + (currMS - lastPublishTimeMS) / 1000
- + " seconds ago, intervalConfigured=" + publishIntervalMS
- / 1000);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
index 6f84251..bc6a553 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java
@@ -26,16 +26,19 @@ import java.util.Map.Entry;
import org.apache.ambari.logfeeder.common.ConfigBlock;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
+import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.log4j.Logger;
public abstract class Output extends ConfigBlock {
- private static final Logger logger = Logger.getLogger(Output.class);
+ private static final Logger LOG = Logger.getLogger(Output.class);
private String destination = null;
- protected MetricCount writeBytesMetric = new MetricCount();
+ protected MetricData writeBytesMetric = new MetricData(getWriteBytesMetricName(), false);
+ protected String getWriteBytesMetricName() {
+ return null;
+ }
@Override
public String getShortDescription() {
@@ -67,7 +70,7 @@ public abstract class Output extends ConfigBlock {
* Extend this method to clean up
*/
public void close() {
- logger.info("Calling base close()." + getShortDescription());
+ LOG.info("Calling base close()." + getShortDescription());
isClosed = true;
}
@@ -91,7 +94,7 @@ public abstract class Output extends ConfigBlock {
}
@Override
- public void addMetricsContainers(List<MetricCount> metricsList) {
+ public void addMetricsContainers(List<MetricData> metricsList) {
super.addMetricsContainers(metricsList);
metricsList.add(writeBytesMetric);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputData.java
index 4a408f9..c46086e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputData.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputData.java
@@ -27,18 +27,16 @@ import org.apache.ambari.logfeeder.input.InputMarker;
* This contains the output json object and InputMarker.
*/
public class OutputData {
- Map<String, Object> jsonObj;
- InputMarker inputMarker;
+ public final Map<String, Object> jsonObj;
+ public final InputMarker inputMarker;
public OutputData(Map<String, Object> jsonObj, InputMarker inputMarker) {
- super();
this.jsonObj = jsonObj;
this.inputMarker = inputMarker;
}
@Override
public String toString() {
- return "OutputData [jsonObj=" + jsonObj + ", inputMarker="
- + inputMarker + "]";
+ return "OutputData [jsonObj=" + jsonObj + ", inputMarker=" + inputMarker + "]";
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
index 2d41a0b..fa4e17b 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputDevNull.java
@@ -28,16 +28,15 @@ import org.apache.log4j.Logger;
*/
public class OutputDevNull extends Output {
- private static Logger logger = Logger.getLogger(OutputDevNull.class);
+ private static final Logger LOG = Logger.getLogger(OutputDevNull.class);
@Override
public void write(String block, InputMarker inputMarker){
- logger.trace("Ignore log block: " + block);
+ LOG.trace("Ignore log block: " + block);
}
@Override
public void copyFile(File inputFile, InputMarker inputMarker) {
- throw new UnsupportedOperationException(
- "copyFile method is not yet supported for output=dev_null");
+ throw new UnsupportedOperationException("copyFile method is not yet supported for output=dev_null");
}
}
[03/10] ambari git commit: AMBARI-18246. Clean up Log Feeder (Miklos
Gergely via oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/FileUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/FileUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/FileUtil.java
index ec26a88..ffd6cec 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/FileUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/FileUtil.java
@@ -20,25 +20,73 @@
package org.apache.ambari.logfeeder.util;
import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.type.TypeReference;
public class FileUtil {
- private static final Logger logger = Logger.getLogger(FileUtil.class);
-
- public static List<File> getAllFileFromDir(File directory,
- String[] searchFileWithExtensions, boolean checkInSubDir) {
+ private static final Logger LOG = Logger.getLogger(FileUtil.class);
+
+ private FileUtil() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static List<File> getAllFileFromDir(File directory, String extension, boolean checkInSubDir) {
if (!directory.exists()) {
- logger.error(directory.getAbsolutePath() + " is not exists ");
- } else if (directory.isDirectory()) {
- return (List<File>) FileUtils.listFiles(directory,
- searchFileWithExtensions, checkInSubDir);
+ LOG.error(directory.getAbsolutePath() + " is not exists ");
+ } else if (!directory.isDirectory()) {
+ LOG.error(directory.getAbsolutePath() + " is not Directory ");
} else {
- logger.error(directory.getAbsolutePath() + " is not Directory ");
+ return (List<File>) FileUtils.listFiles(directory, new String[]{extension}, checkInSubDir);
}
return new ArrayList<File>();
}
+
+
+ public static Object getFileKey(File file) {
+ try {
+ Path fileFullPath = Paths.get(file.getAbsolutePath());
+ if (fileFullPath != null) {
+ BasicFileAttributes basicAttr = Files.readAttributes(fileFullPath, BasicFileAttributes.class);
+ return basicAttr.fileKey();
+ }
+ } catch (Throwable ex) {
+ LOG.error("Error getting file attributes for file=" + file, ex);
+ }
+ return file.toString();
+ }
+
+ public static File getFileFromClasspath(String filename) {
+ URL fileCompleteUrl = Thread.currentThread().getContextClassLoader().getResource(filename);
+ LOG.debug("File Complete URI :" + fileCompleteUrl);
+ File file = null;
+ try {
+ file = new File(fileCompleteUrl.toURI());
+ } catch (Exception exception) {
+ LOG.debug(exception.getMessage(), exception.getCause());
+ }
+ return file;
+ }
+
+ public static HashMap<String, Object> readJsonFromFile(File jsonFile) {
+ ObjectMapper mapper = new ObjectMapper();
+ try {
+ HashMap<String, Object> jsonmap = mapper.readValue(jsonFile, new TypeReference<HashMap<String, Object>>() {});
+ return jsonmap;
+ } catch (IOException e) {
+ LOG.error(e, e.getCause());
+ }
+ return new HashMap<String, Object>();
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
index 32029ff..5bf600e 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java
@@ -22,36 +22,23 @@ package org.apache.ambari.logfeeder.util;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
-import java.io.IOException;
import java.lang.reflect.Type;
import java.net.InetAddress;
-import java.net.URL;
import java.net.UnknownHostException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.TimeZone;
import org.apache.ambari.logfeeder.LogFeeder;
-import org.apache.ambari.logfeeder.filter.Filter;
-import org.apache.ambari.logfeeder.input.Input;
-import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
-import org.apache.ambari.logfeeder.mapper.Mapper;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
-import org.apache.ambari.logfeeder.output.Output;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
-import org.codehaus.jackson.JsonParseException;
-import org.codehaus.jackson.map.JsonMappingException;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.type.TypeReference;
-import com.google.common.collect.ObjectArrays;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
@@ -60,109 +47,80 @@ import com.google.gson.reflect.TypeToken;
* This class contains utility methods used by LogFeeder
*/
public class LogFeederUtil {
- private static final Logger logger = Logger.getLogger(LogFeederUtil.class);
+ private static final Logger LOG = Logger.getLogger(LogFeederUtil.class);
- private static final int HASH_SEED = 31174077;
- public final static String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";
- public final static String SOLR_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
- private static Gson gson = new GsonBuilder().setDateFormat(DATE_FORMAT).create();
-
- private static Properties props;
-
- private static Map<String, LogHistory> logHistoryList = new Hashtable<String, LogHistory>();
- private static int logInterval = 30000; // 30 seconds
+ private final static String GSON_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";
+ private static Gson gson = new GsonBuilder().setDateFormat(GSON_DATE_FORMAT).create();
+
+ public static Gson getGson() {
+ return gson;
+ }
public static String hostName = null;
public static String ipAddress = null;
- private static String logfeederTempDir = null;
-
- private static final Object _LOCK = new Object();
-
static{
- setHostNameAndIP();
+ try {
+ InetAddress ip = InetAddress.getLocalHost();
+ ipAddress = ip.getHostAddress();
+ String getHostName = ip.getHostName();
+ String getCanonicalHostName = ip.getCanonicalHostName();
+ if (!getCanonicalHostName.equalsIgnoreCase(ipAddress)) {
+ LOG.info("Using getCanonicalHostName()=" + getCanonicalHostName);
+ hostName = getCanonicalHostName;
+ } else {
+ LOG.info("Using getHostName()=" + getHostName);
+ hostName = getHostName;
+ }
+ LOG.info("ipAddress=" + ipAddress + ", getHostName=" + getHostName + ", getCanonicalHostName=" + getCanonicalHostName +
+ ", hostName=" + hostName);
+ } catch (UnknownHostException e) {
+ LOG.error("Error getting hostname.", e);
+ }
}
- public static Gson getGson() {
- return gson;
- }
-
- private static ThreadLocal<SimpleDateFormat> dateFormatter = new ThreadLocal<SimpleDateFormat>() {
- @Override
- protected SimpleDateFormat initialValue() {
- SimpleDateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT);
- sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
- return sdf;
- }
- };
+ private static Properties props;
/**
- * This method will read the properties from System, followed by propFile
- * and finally from the map
+ * This method will read the properties from System, followed by propFile and finally from the map
*/
- public static void loadProperties(String propFile, String[] propNVList)
- throws Exception {
- logger.info("Loading properties. propFile=" + propFile);
+ public static void loadProperties(String propFile, String[] propNVList) throws Exception {
+ LOG.info("Loading properties. propFile=" + propFile);
props = new Properties(System.getProperties());
boolean propLoaded = false;
// First get properties file path from environment value
String propertiesFilePath = System.getProperty("properties");
- if (propertiesFilePath != null && !propertiesFilePath.isEmpty()) {
+ if (StringUtils.isNotEmpty(propertiesFilePath)) {
File propertiesFile = new File(propertiesFilePath);
if (propertiesFile.exists() && propertiesFile.isFile()) {
- logger.info("Properties file path set in environment. Loading properties file="
- + propertiesFilePath);
- FileInputStream fileInputStream = null;
- try {
- fileInputStream = new FileInputStream(propertiesFile);
- props.load(fileInputStream);
+ LOG.info("Properties file path set in environment. Loading properties file=" + propertiesFilePath);
+ try (FileInputStream fis = new FileInputStream(propertiesFile)) {
+ props.load(fis);
propLoaded = true;
} catch (Throwable t) {
- logger.error("Error loading properties file. properties file="
- + propertiesFile.getAbsolutePath());
- } finally {
- if (fileInputStream != null) {
- try {
- fileInputStream.close();
- } catch (Throwable t) {
- // Ignore error
- }
- }
+ LOG.error("Error loading properties file. properties file=" + propertiesFile.getAbsolutePath());
}
} else {
- logger.error("Properties file path set in environment, but file not found. properties file="
- + propertiesFilePath);
+ LOG.error("Properties file path set in environment, but file not found. properties file=" + propertiesFilePath);
}
}
if (!propLoaded) {
- BufferedInputStream fileInputStream = null;
- try {
+ try (BufferedInputStream bis = (BufferedInputStream) LogFeeder.class.getClassLoader().getResourceAsStream(propFile)) {
// Properties not yet loaded, let's try from class loader
- fileInputStream = (BufferedInputStream) LogFeeder.class
- .getClassLoader().getResourceAsStream(propFile);
- if (fileInputStream != null) {
- logger.info("Loading properties file " + propFile
- + " from classpath");
- props.load(fileInputStream);
+ if (bis != null) {
+ LOG.info("Loading properties file " + propFile + " from classpath");
+ props.load(bis);
propLoaded = true;
} else {
- logger.fatal("Properties file not found in classpath. properties file name= "
- + propFile);
- }
- } finally {
- if (fileInputStream != null) {
- try {
- fileInputStream.close();
- } catch (IOException e) {
- }
+ LOG.fatal("Properties file not found in classpath. properties file name= " + propFile);
}
}
}
if (!propLoaded) {
- logger.fatal("Properties file is not loaded.");
+ LOG.fatal("Properties file is not loaded.");
throw new Exception("Properties not loaded");
} else {
updatePropertiesFromMap(propNVList);
@@ -173,162 +131,124 @@ public class LogFeederUtil {
if (nvList == null) {
return;
}
- logger.info("Trying to load additional proeprties from argument paramters. nvList.length="
- + nvList.length);
- if (nvList != null && nvList.length > 0) {
- for (String nv : nvList) {
- logger.info("Passed nv=" + nv);
- if (nv.startsWith("-") && nv.length() > 1) {
- nv = nv.substring(1);
- logger.info("Stripped nv=" + nv);
- int i = nv.indexOf("=");
- if (nv.length() > i) {
- logger.info("Candidate nv=" + nv);
- String name = nv.substring(0, i);
- String value = nv.substring(i + 1);
- logger.info("Adding property from argument to properties. name="
- + name + ", value=" + value);
- props.put(name, value);
- }
+ LOG.info("Trying to load additional proeprties from argument paramters. nvList.length=" + nvList.length);
+ for (String nv : nvList) {
+ LOG.info("Passed nv=" + nv);
+ if (nv.startsWith("-") && nv.length() > 1) {
+ nv = nv.substring(1);
+ LOG.info("Stripped nv=" + nv);
+ int i = nv.indexOf("=");
+ if (nv.length() > i) {
+ LOG.info("Candidate nv=" + nv);
+ String name = nv.substring(0, i);
+ String value = nv.substring(i + 1);
+ LOG.info("Adding property from argument to properties. name=" + name + ", value=" + value);
+ props.put(name, value);
}
}
}
}
- static public String getStringProperty(String key) {
- if (props != null) {
- return props.getProperty(key);
- }
- return null;
+ public static String getStringProperty(String key) {
+ return props == null ? null : props.getProperty(key);
}
- static public String getStringProperty(String key, String defaultValue) {
- if (props != null) {
- return props.getProperty(key, defaultValue);
- }
- return defaultValue;
+ public static String getStringProperty(String key, String defaultValue) {
+ return props == null ? defaultValue : props.getProperty(key, defaultValue);
}
- static public boolean getBooleanProperty(String key, boolean defaultValue) {
- String strValue = getStringProperty(key);
- return toBoolean(strValue, defaultValue);
+ public static boolean getBooleanProperty(String key, boolean defaultValue) {
+ String value = getStringProperty(key);
+ return toBoolean(value, defaultValue);
}
- private static boolean toBoolean(String strValue, boolean defaultValue) {
- boolean retValue = defaultValue;
- if (!StringUtils.isEmpty(strValue)) {
- if (strValue.equalsIgnoreCase("true")
- || strValue.equalsIgnoreCase("yes")) {
- retValue = true;
- } else {
- retValue = false;
- }
+ private static boolean toBoolean(String value, boolean defaultValue) {
+ if (StringUtils.isEmpty(value)) {
+ return defaultValue;
}
- return retValue;
+
+ return "true".equalsIgnoreCase(value) || "yes".equalsIgnoreCase(value);
}
- static public int getIntProperty(String key, int defaultValue) {
- String strValue = getStringProperty(key);
- int retValue = defaultValue;
- retValue = objectToInt(strValue, retValue, ", key=" + key);
+ public static int getIntProperty(String key, int defaultValue) {
+ String value = getStringProperty(key);
+ int retValue = objectToInt(value, defaultValue, ", key=" + key);
return retValue;
}
- public static int objectToInt(Object objValue, int retValue,
- String errMessage) {
+ public static int objectToInt(Object objValue, int retValue, String errMessage) {
if (objValue == null) {
return retValue;
}
String strValue = objValue.toString();
- if (!StringUtils.isEmpty(strValue)) {
+ if (StringUtils.isNotEmpty(strValue)) {
try {
retValue = Integer.parseInt(strValue);
} catch (Throwable t) {
- logger.error("Error parsing integer value. str=" + strValue
- + ", " + errMessage);
+ LOG.error("Error parsing integer value. str=" + strValue + ", " + errMessage);
}
}
return retValue;
}
- public static boolean isEnabled(Map<String, Object> conditionConfigs,
- Map<String, Object> valueConfigs) {
- boolean allow = toBoolean((String) valueConfigs.get("is_enabled"), true);
- @SuppressWarnings("unchecked")
- Map<String, Object> conditions = (Map<String, Object>) conditionConfigs
- .get("conditions");
- if (conditions != null && conditions.size() > 0) {
- allow = false;
- for (String conditionType : conditions.keySet()) {
- if (conditionType.equalsIgnoreCase("fields")) {
- @SuppressWarnings("unchecked")
- Map<String, Object> fields = (Map<String, Object>) conditions
- .get("fields");
- for (String fieldName : fields.keySet()) {
- Object values = fields.get(fieldName);
- if (values instanceof String) {
- allow = isFieldConditionMatch(valueConfigs,
- fieldName, (String) values);
- } else {
- @SuppressWarnings("unchecked")
- List<String> listValues = (List<String>) values;
- for (String stringValue : listValues) {
- allow = isFieldConditionMatch(valueConfigs,
- fieldName, stringValue);
- if (allow) {
- break;
- }
- }
- }
- if (allow) {
- break;
+ @SuppressWarnings("unchecked")
+ public static boolean isEnabled(Map<String, Object> conditionConfigs, Map<String, Object> valueConfigs) {
+ Map<String, Object> conditions = (Map<String, Object>) conditionConfigs.get("conditions");
+ if (MapUtils.isEmpty(conditions)) {
+ return toBoolean((String) valueConfigs.get("is_enabled"), true);
+ }
+
+ for (String conditionType : conditions.keySet()) {
+ if (!conditionType.equalsIgnoreCase("fields")) {
+ continue;
+ }
+
+ Map<String, Object> fields = (Map<String, Object>) conditions.get("fields");
+ for (Map.Entry<String, Object> field : fields.entrySet()) {
+ if (field.getValue() instanceof String) {
+ if (isFieldConditionMatch(valueConfigs, field.getKey(), (String) field.getValue())) {
+ return true;
+ }
+ } else {
+ for (String stringValue : (List<String>) field.getValue()) {
+ if (isFieldConditionMatch(valueConfigs, field.getKey(), stringValue)) {
+ return true;
}
}
}
- if (allow) {
- break;
- }
}
}
- return allow;
+
+ return false;
}
- public static boolean isFieldConditionMatch(Map<String, Object> configs,
- String fieldName, String stringValue) {
+ private static boolean isFieldConditionMatch(Map<String, Object> configs, String fieldName, String stringValue) {
boolean allow = false;
String fieldValue = (String) configs.get(fieldName);
if (fieldValue != null && fieldValue.equalsIgnoreCase(stringValue)) {
allow = true;
} else {
@SuppressWarnings("unchecked")
- Map<String, Object> addFields = (Map<String, Object>) configs
- .get("add_fields");
+ Map<String, Object> addFields = (Map<String, Object>) configs.get("add_fields");
if (addFields != null && addFields.get(fieldName) != null) {
String addFieldValue = (String) addFields.get(fieldName);
if (stringValue.equalsIgnoreCase(addFieldValue)) {
allow = true;
}
}
-
}
return allow;
}
- public static void logStatForMetric(MetricCount metric, String prefixStr,
- String postFix) {
- long currStat = metric.count;
+ public static void logStatForMetric(MetricData metric, String prefixStr, String postFix) {
+ long currStat = metric.value;
long currMS = System.currentTimeMillis();
- if (currStat > metric.prevLogCount) {
- if (postFix == null) {
- postFix = "";
- }
- logger.info(prefixStr + ": total_count=" + metric.count
- + ", duration=" + (currMS - metric.prevLogMS) / 1000
- + " secs, count=" + (currStat - metric.prevLogCount)
- + postFix);
+ if (currStat > metric.prevLogValue) {
+ LOG.info(prefixStr + ": total_count=" + metric.value + ", duration=" + (currMS - metric.prevLogTime) / 1000 +
+ " secs, count=" + (currStat - metric.prevLogValue) + postFix);
}
- metric.prevLogCount = currStat;
- metric.prevLogMS = currMS;
+ metric.prevLogValue = currStat;
+ metric.prevLogTime = currMS;
}
public static Map<String, Object> cloneObject(Map<String, Object> map) {
@@ -336,221 +256,74 @@ public class LogFeederUtil {
return null;
}
String jsonStr = gson.toJson(map);
- Type type = new TypeToken<Map<String, Object>>() {
- }.getType();
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
return gson.fromJson(jsonStr, type);
}
public static Map<String, Object> toJSONObject(String jsonStr) {
- if(jsonStr==null || jsonStr.trim().isEmpty()){
+ if (StringUtils.isBlank(jsonStr)) {
return new HashMap<String, Object>();
}
- Type type = new TypeToken<Map<String, Object>>() {
- }.getType();
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
return gson.fromJson(jsonStr, type);
}
- static public boolean logErrorMessageByInterval(String key, String message,
- Throwable e, Logger callerLogger, Level level) {
+ private static class LogHistory {
+ private long lastLogTime = 0;
+ private int counter = 0;
+ }
+
+ private static Map<String, LogHistory> logHistoryList = new Hashtable<String, LogHistory>();
+ public static boolean logErrorMessageByInterval(String key, String message, Throwable e, Logger callerLogger, Level level) {
LogHistory log = logHistoryList.get(key);
if (log == null) {
log = new LogHistory();
logHistoryList.put(key, log);
}
- if ((System.currentTimeMillis() - log.lastLogTime) > logInterval) {
+
+ if ((System.currentTimeMillis() - log.lastLogTime) > 30 * 1000) {
log.lastLogTime = System.currentTimeMillis();
- int counter = log.counter;
- log.counter = 0;
- if (counter > 0) {
- message += ". Messages suppressed before: " + counter;
- }
- if (e == null) {
- callerLogger.log(level, message);
- } else {
- callerLogger.log(level, message, e);
+ if (log.counter > 0) {
+ message += ". Messages suppressed before: " + log.counter;
}
+ log.counter = 0;
+ callerLogger.log(level, message, e);
return true;
} else {
log.counter++;
- }
- return false;
-
- }
-
- static public String subString(String str, int maxLength) {
- if (str == null || str.length() == 0) {
- return "";
- }
- maxLength = str.length() < maxLength ? str.length() : maxLength;
- return str.substring(0, maxLength);
- }
-
- public static long genHash(String value) {
- if (value == null) {
- value = "null";
- }
- return MurmurHash.hash64A(value.getBytes(), HASH_SEED);
- }
-
- private static class LogHistory {
- private long lastLogTime = 0;
- private int counter = 0;
- }
-
- public static String getDate(String timeStampStr) {
- try {
- return dateFormatter.get().format(new Date(Long.parseLong(timeStampStr)));
- } catch (Exception ex) {
- logger.error(ex);
- return null;
+ return false;
}
}
- public static String getActualDateStr() {
- try {
- return dateFormatter.get().format(new Date());
- } catch (Exception ex) {
- logger.error(ex);
- return null;
- }
- }
-
- public static File getFileFromClasspath(String filename) {
- URL fileCompleteUrl = Thread.currentThread().getContextClassLoader()
- .getResource(filename);
- logger.debug("File Complete URI :" + fileCompleteUrl);
- File file = null;
- try {
- file = new File(fileCompleteUrl.toURI());
- } catch (Exception exception) {
- logger.debug(exception.getMessage(), exception.getCause());
- }
- return file;
- }
-
- public static Object getClassInstance(String classFullName, AliasUtil.ALIAS_TYPE aliasType) {
- Object instance = null;
- try {
- instance = (Object) Class.forName(classFullName).getConstructor().newInstance();
- } catch (Exception exception) {
- logger.error("Unsupported class =" + classFullName, exception.getCause());
+ public static boolean isListContains(List<String> list, String str, boolean caseSensitive) {
+ if (list == null) {
+ return false;
}
- // check instance class as par aliasType
- if (instance != null) {
- boolean isValid = false;
- switch (aliasType) {
- case FILTER:
- isValid = Filter.class.isAssignableFrom(instance.getClass());
- break;
- case INPUT:
- isValid = Input.class.isAssignableFrom(instance.getClass());
- break;
- case OUTPUT:
- isValid = Output.class.isAssignableFrom(instance.getClass());
- break;
- case MAPPER:
- isValid = Mapper.class.isAssignableFrom(instance.getClass());
- break;
- default:
- // by default consider all are valid class
- isValid = true;
+
+ for (String value : list) {
+ if (value == null) {
+ continue;
}
- if (!isValid) {
- logger.error("Not a valid class :" + classFullName + " AliasType :" + aliasType.name());
- }
- }
- return instance;
- }
-
- public static HashMap<String, Object> readJsonFromFile(File jsonFile) {
- ObjectMapper mapper = new ObjectMapper();
- try {
- HashMap<String, Object> jsonmap = mapper.readValue(jsonFile, new TypeReference<HashMap<String, Object>>() {
- });
- return jsonmap;
- } catch (JsonParseException e) {
- logger.error(e, e.getCause());
- } catch (JsonMappingException e) {
- logger.error(e, e.getCause());
- } catch (IOException e) {
- logger.error(e, e.getCause());
- }
- return new HashMap<String, Object>();
- }
-
- public static boolean isListContains(List<String> list, String str, boolean caseSensitive) {
- if (list != null) {
- for (String value : list) {
- if (value != null) {
- if (caseSensitive) {
- if (value.equals(str)) {
- return true;
- }
- } else {
- if (value.equalsIgnoreCase(str)) {
- return true;
- }
- }
- if (value.equalsIgnoreCase(LogFeederConstants.ALL)) {
- return true;
- }
- }
+
+ if (caseSensitive ? value.equals(str) : value.equalsIgnoreCase(str) ||
+ value.equalsIgnoreCase(LogFeederConstants.ALL)) {
+ return true;
}
}
return false;
}
+ private static String logfeederTempDir = null;
- private static synchronized String setHostNameAndIP() {
- if (hostName == null || ipAddress == null) {
- try {
- InetAddress ip = InetAddress.getLocalHost();
- ipAddress = ip.getHostAddress();
- String getHostName = ip.getHostName();
- String getCanonicalHostName = ip.getCanonicalHostName();
- if (!getCanonicalHostName.equalsIgnoreCase(ipAddress)) {
- logger.info("Using getCanonicalHostName()=" + getCanonicalHostName);
- hostName = getCanonicalHostName;
- } else {
- logger.info("Using getHostName()=" + getHostName);
- hostName = getHostName;
- }
- logger.info("ipAddress=" + ipAddress + ", getHostName=" + getHostName
- + ", getCanonicalHostName=" + getCanonicalHostName + ", hostName="
- + hostName);
- } catch (UnknownHostException e) {
- logger.error("Error getting hostname.", e);
- }
- }
- return hostName;
- }
-
- public static String[] mergeArray(String[] first, String[] second) {
- if (first == null) {
- first = new String[0];
- }
- if (second == null) {
- second = new String[0];
- }
- String[] mergedArray = ObjectArrays.concat(first, second, String.class);
- return mergedArray;
- }
-
- public static String getLogfeederTempDir() {
+ public synchronized static String getLogfeederTempDir() {
if (logfeederTempDir == null) {
- synchronized (_LOCK) {
- if (logfeederTempDir == null) {
- String tempDirValue = getStringProperty("logfeeder.tmp.dir",
- "/tmp/$username/logfeeder/");
- HashMap<String, String> contextParam = new HashMap<String, String>();
- String username = System.getProperty("user.name");
- contextParam.put("username", username);
- logfeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue,
- contextParam);
- }
- }
+ String tempDirValue = getStringProperty("logfeeder.tmp.dir", "/tmp/$username/logfeeder/");
+ HashMap<String, String> contextParam = new HashMap<String, String>();
+ String username = System.getProperty("user.name");
+ contextParam.put("username", username);
+ logfeederTempDir = PlaceholderUtil.replaceVariables(tempDirValue, contextParam);
}
return logfeederTempDir;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
index fd96f8a..c975b99 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogfeederHDFSUtil.java
@@ -25,71 +25,53 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.Logger;
-public enum LogfeederHDFSUtil {
- INSTANCE;
- private static Logger logger = Logger.getLogger(LogfeederHDFSUtil.class);
+public class LogfeederHDFSUtil {
+ private static final Logger LOG = Logger.getLogger(LogfeederHDFSUtil.class);
- public void createHDFSDir(String dirPath, FileSystem dfs) {
- Path src = new Path(dirPath);
- try {
- if (dfs.isDirectory(src)) {
- logger.info("hdfs dir dirPath=" + dirPath + " is already exist.");
- return;
- }
- boolean isDirCreated = dfs.mkdirs(src);
- if (isDirCreated) {
- logger.debug("HDFS dirPath=" + dirPath + " created successfully.");
- } else {
- logger.warn("HDFS dir creation failed dirPath=" + dirPath);
- }
- } catch (IOException e) {
- logger.error("HDFS dir creation failed dirPath=" + dirPath, e.getCause());
- }
+ private LogfeederHDFSUtil() {
+ throw new UnsupportedOperationException();
}
-
- public boolean copyFromLocal(String sourceFilepath, String destFilePath,
- FileSystem fileSystem, boolean overwrite, boolean delSrc) {
+
+ public static boolean copyFromLocal(String sourceFilepath, String destFilePath, FileSystem fileSystem, boolean overwrite,
+ boolean delSrc) {
Path src = new Path(sourceFilepath);
Path dst = new Path(destFilePath);
boolean isCopied = false;
try {
- logger.info("copying localfile := " + sourceFilepath + " to hdfsPath := "
- + destFilePath);
+ LOG.info("copying localfile := " + sourceFilepath + " to hdfsPath := " + destFilePath);
fileSystem.copyFromLocalFile(delSrc, overwrite, src, dst);
isCopied = true;
} catch (Exception e) {
- logger.error("Error copying local file :" + sourceFilepath
- + " to hdfs location : " + destFilePath, e);
+ LOG.error("Error copying local file :" + sourceFilepath + " to hdfs location : " + destFilePath, e);
}
return isCopied;
}
- public FileSystem buildFileSystem(String hdfsHost, String hdfsPort) {
+ public static FileSystem buildFileSystem(String hdfsHost, String hdfsPort) {
try {
Configuration configuration = buildHdfsConfiguration(hdfsHost, hdfsPort);
FileSystem fs = FileSystem.get(configuration);
return fs;
} catch (Exception e) {
- logger.error("Exception is buildFileSystem :", e);
+ LOG.error("Exception is buildFileSystem :", e);
}
return null;
}
- public void closeFileSystem(FileSystem fileSystem) {
+ private static Configuration buildHdfsConfiguration(String hdfsHost, String hdfsPort) {
+ String url = "hdfs://" + hdfsHost + ":" + hdfsPort + "/";
+ Configuration configuration = new Configuration();
+ configuration.set("fs.default.name", url);
+ return configuration;
+ }
+
+ public static void closeFileSystem(FileSystem fileSystem) {
if (fileSystem != null) {
try {
fileSystem.close();
} catch (IOException e) {
- logger.error(e.getLocalizedMessage(), e.getCause());
+ LOG.error(e.getLocalizedMessage(), e.getCause());
}
}
}
-
- public Configuration buildHdfsConfiguration(String hdfsHost, String hdfsPort) {
- String url = "hdfs://" + hdfsHost + ":" + hdfsPort + "/";
- Configuration configuration = new Configuration();
- configuration.set("fs.default.name", url);
- return configuration;
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/PlaceholderUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/PlaceholderUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/PlaceholderUtil.java
index d6c3117..13f2865 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/PlaceholderUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/PlaceholderUtil.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -21,38 +21,34 @@ import java.util.HashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-public class PlaceholderUtil {
+import org.apache.commons.lang3.StringUtils;
- private static Pattern placeHolderPattern;
- static {
- placeHolderPattern = Pattern.compile("\\$\\s*(\\w+)");
+public class PlaceholderUtil {
+ private PlaceholderUtil() {
+ throw new UnsupportedOperationException();
}
+
+ private static final Pattern placeHolderPattern = Pattern.compile("\\$\\s*(\\w+)");
- public static String replaceVariables(String inputStr,
- HashMap<String, String> contextParam) {
+ public static String replaceVariables(String inputStr, HashMap<String, String> contextParam) {
Matcher m = placeHolderPattern.matcher(inputStr);
- String placeholder;
- String replacement;
String output = new String(inputStr);
while (m.find()) {
- placeholder = m.group();
+ String placeholder = m.group();
if (placeholder != null && !placeholder.isEmpty()) {
- String key = placeholder.replace("$","").toLowerCase();// remove
- // brace
- replacement = getFromContext(contextParam, placeholder, key);
+ String key = placeholder.replace("$","").toLowerCase();// remove brace
+ String replacement = getFromContext(contextParam, placeholder, key);
output = output.replace(placeholder, replacement);
}
}
return output;
}
- private static String getFromContext(HashMap<String, String> contextParam,
- String defaultValue, String key) {
- String returnValue = defaultValue;// by default set default value as a
- // return
+ private static String getFromContext(HashMap<String, String> contextParam, String defaultValue, String key) {
+ String returnValue = defaultValue; // by default set default value as a return
if (contextParam != null) {
String value = contextParam.get(key);
- if (value != null && !value.trim().isEmpty()) {
+ if (StringUtils.isNotBlank(value)) {
returnValue = value;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/S3Util.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/S3Util.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/S3Util.java
index 10ea2c2..31a38d0 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/S3Util.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/S3Util.java
@@ -19,7 +19,6 @@
package org.apache.ambari.logfeeder.util;
import java.io.BufferedReader;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@@ -27,6 +26,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.zip.GZIPInputStream;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
@@ -39,22 +39,19 @@ import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.transfer.TransferManager;
-import com.amazonaws.services.s3.transfer.Upload;
/**
* Utility to connect to s3
*/
public class S3Util {
- public static final S3Util INSTANCE = new S3Util();
-
private static final Logger LOG = Logger.getLogger(S3Util.class);
- public static final String S3_PATH_START_WITH = "s3://";
- public static final String S3_PATH_SEPARATOR = "/";
-
- public AmazonS3 getS3Client(String accessKey, String secretKey) {
- AWSCredentials awsCredentials = AWSUtil.INSTANCE.createAWSCredentials(
- accessKey, secretKey);
+ private S3Util() {
+ throw new UnsupportedOperationException();
+ }
+
+ public static AmazonS3 getS3Client(String accessKey, String secretKey) {
+ AWSCredentials awsCredentials = AWSUtil.createAWSCredentials(accessKey, secretKey);
AmazonS3 s3client;
if (awsCredentials != null) {
s3client = new AmazonS3Client(awsCredentials);
@@ -64,9 +61,8 @@ public class S3Util {
return s3client;
}
- public TransferManager getTransferManager(String accessKey, String secretKey) {
- AWSCredentials awsCredentials = AWSUtil.INSTANCE.createAWSCredentials(
- accessKey, secretKey);
+ public static TransferManager getTransferManager(String accessKey, String secretKey) {
+ AWSCredentials awsCredentials = AWSUtil.createAWSCredentials(accessKey, secretKey);
TransferManager transferManager;
if (awsCredentials != null) {
transferManager = new TransferManager(awsCredentials);
@@ -76,35 +72,31 @@ public class S3Util {
return transferManager;
}
- public void shutdownTransferManager(TransferManager transferManager) {
+ public static void shutdownTransferManager(TransferManager transferManager) {
if (transferManager != null) {
transferManager.shutdownNow();
}
}
- public String getBucketName(String s3Path) {
+ public static String getBucketName(String s3Path) {
String bucketName = null;
// s3path
if (s3Path != null) {
- String[] s3PathParts = s3Path.replace(S3_PATH_START_WITH, "").split(
- S3_PATH_SEPARATOR);
+ String[] s3PathParts = s3Path.replace(LogFeederConstants.S3_PATH_START_WITH, "").split(LogFeederConstants.S3_PATH_SEPARATOR);
bucketName = s3PathParts[0];
}
return bucketName;
}
- public String getS3Key(String s3Path) {
+ public static String getS3Key(String s3Path) {
StringBuilder s3Key = new StringBuilder();
- // s3path
if (s3Path != null) {
- String[] s3PathParts = s3Path.replace(S3_PATH_START_WITH, "").split(
- S3_PATH_SEPARATOR);
- ArrayList<String> s3PathList = new ArrayList<String>(
- Arrays.asList(s3PathParts));
+ String[] s3PathParts = s3Path.replace(LogFeederConstants.S3_PATH_START_WITH, "").split(LogFeederConstants.S3_PATH_SEPARATOR);
+ ArrayList<String> s3PathList = new ArrayList<String>(Arrays.asList(s3PathParts));
s3PathList.remove(0);// remove bucketName
for (int index = 0; index < s3PathList.size(); index++) {
if (index > 0) {
- s3Key.append(S3_PATH_SEPARATOR);
+ s3Key.append(LogFeederConstants.S3_PATH_SEPARATOR);
}
s3Key.append(s3PathList.get(index));
}
@@ -112,63 +104,41 @@ public class S3Util {
return s3Key.toString();
}
- public void uploadFileTos3(String bucketName, String s3Key, File localFile,
- String accessKey, String secretKey) {
- TransferManager transferManager = getTransferManager(accessKey, secretKey);
- try {
- Upload upload = transferManager.upload(bucketName, s3Key, localFile);
- upload.waitForUploadResult();
- } catch (AmazonClientException | InterruptedException e) {
- LOG.error("s3 uploading failed for file :" + localFile.getAbsolutePath(),
- e);
- } finally {
- shutdownTransferManager(transferManager);
- }
- }
-
/**
* Get the buffer reader to read s3 file as a stream
*/
- public BufferedReader getReader(String s3Path, String accessKey,
- String secretKey) throws IOException {
+ public static BufferedReader getReader(String s3Path, String accessKey, String secretKey) throws IOException {
// TODO error handling
// Compression support
// read header and decide the compression(auto detection)
// For now hard-code GZIP compression
String s3Bucket = getBucketName(s3Path);
String s3Key = getS3Key(s3Path);
- S3Object fileObj = getS3Client(accessKey, secretKey).getObject(
- new GetObjectRequest(s3Bucket, s3Key));
- GZIPInputStream objectInputStream;
+ S3Object fileObj = getS3Client(accessKey, secretKey).getObject(new GetObjectRequest(s3Bucket, s3Key));
try {
- objectInputStream = new GZIPInputStream(fileObj.getObjectContent());
- BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(
- objectInputStream));
+ GZIPInputStream objectInputStream = new GZIPInputStream(fileObj.getObjectContent());
+ BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(objectInputStream));
return bufferedReader;
} catch (IOException e) {
- LOG.error("Error in creating stream reader for s3 file :" + s3Path,
- e.getCause());
+ LOG.error("Error in creating stream reader for s3 file :" + s3Path, e.getCause());
throw e;
}
}
- public void writeIntoS3File(String data, String bucketName, String s3Key,
- String accessKey, String secretKey) {
+ public static void writeIntoS3File(String data, String bucketName, String s3Key, String accessKey, String secretKey) {
InputStream in = null;
try {
in = IOUtils.toInputStream(data, "UTF-8");
} catch (IOException e) {
LOG.error(e);
}
+
if (in != null) {
TransferManager transferManager = getTransferManager(accessKey, secretKey);
try {
if (transferManager != null) {
- transferManager.upload(
- new PutObjectRequest(bucketName, s3Key, in,
- new ObjectMetadata())).waitForUploadResult();
- LOG.debug("Data Uploaded to s3 file :" + s3Key + " in bucket :"
- + bucketName);
+ transferManager.upload(new PutObjectRequest(bucketName, s3Key, in, new ObjectMetadata())).waitForUploadResult();
+ LOG.debug("Data Uploaded to s3 file :" + s3Key + " in bucket :" + bucketName);
}
} catch (AmazonClientException | InterruptedException e) {
LOG.error(e);
@@ -182,5 +152,4 @@ public class S3Util {
}
}
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
deleted file mode 100644
index 44113e1..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.util;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.SolrRequest.METHOD;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.SolrException;
-
-public class SolrUtil {
-
- private static final Logger logger = Logger.getLogger(SolrUtil.class);
-
- private static SolrUtil instance = null;
-
- private SolrClient solrClient = null;
- private CloudSolrClient solrClouldClient = null;
-
- private String solrDetail = "";
-
- private SolrUtil() throws Exception {
- String url = LogFeederUtil.getStringProperty("logfeeder.solr.url");
- String zkConnectString = LogFeederUtil.getStringProperty("logfeeder.solr.zk_connect_string");
- String collection = LogFeederUtil.getStringProperty("logfeeder.solr.core.config.name", "history");
- connectToSolr(url, zkConnectString, collection);
- }
-
- public static SolrUtil getInstance() {
- if (instance == null) {
- synchronized (SolrUtil.class) {
- if (instance == null) {
- try {
- instance = new SolrUtil();
- } catch (Exception e) {
- final String LOG_MESSAGE_KEY = SolrUtil.class
- .getSimpleName() + "_SOLR_UTIL";
- LogFeederUtil.logErrorMessageByInterval(
- LOG_MESSAGE_KEY,
- "Error constructing solrUtil", e, logger,
- Level.WARN);
- }
- }
- }
- }
- return instance;
- }
-
- private SolrClient connectToSolr(String url, String zkConnectString,
- String collection) throws Exception {
- solrDetail = "zkConnectString=" + zkConnectString + ", collection=" + collection
- + ", url=" + url;
-
- logger.info("connectToSolr() " + solrDetail);
- if (collection == null || collection.isEmpty()) {
- throw new Exception("For solr, collection name is mandatory. "
- + solrDetail);
- }
- if (zkConnectString != null && !zkConnectString.isEmpty()) {
- solrDetail = "zkConnectString=" + zkConnectString + ", collection=" + collection;
- logger.info("Using zookeepr. " + solrDetail);
- solrClouldClient = new CloudSolrClient(zkConnectString);
- solrClouldClient.setDefaultCollection(collection);
- solrClient = solrClouldClient;
- int waitDurationMS = 3 * 60 * 1000;
- checkSolrStatus(waitDurationMS);
- } else {
- if (url == null || url.trim().isEmpty()) {
- throw new Exception("Both zkConnectString and URL are empty. zkConnectString="
- + zkConnectString + ", collection=" + collection + ", url="
- + url);
- }
- solrDetail = "collection=" + collection + ", url=" + url;
- String collectionURL = url + "/" + collection;
- logger.info("Connecting to solr : " + collectionURL);
- solrClient = new HttpSolrClient(collectionURL);
-
- }
- return solrClient;
- }
-
- private boolean checkSolrStatus(int waitDurationMS) {
- boolean status = false;
- try {
- long beginTimeMS = System.currentTimeMillis();
- long waitIntervalMS = 2000;
- int pingCount = 0;
- while (true) {
- pingCount++;
- CollectionAdminResponse response = null;
- try {
- CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List();
- response = colListReq.process(solrClient);
- } catch (Exception ex) {
- logger.error("Con't connect to Solr. solrDetail=" + solrDetail, ex);
- }
- if (response != null && response.getStatus() == 0) {
- logger.info("Solr getCollections() is success. solr=" + solrDetail);
- status = true;
- break;
- }
- if (System.currentTimeMillis() - beginTimeMS > waitDurationMS) {
- logger.error("Solr is not reachable even after "
- + (System.currentTimeMillis() - beginTimeMS)
- + " ms. If you are using alias, then you might have to restart LogSearch after Solr is up and running. solr="
- + solrDetail + ", response=" + response);
- break;
- } else {
- logger.warn("Solr is not reachable yet. getCollections() attempt count=" + pingCount
- + ". Will sleep for " + waitIntervalMS + " ms and try again." + " solr=" + solrDetail
- + ", response=" + response);
-
- }
- Thread.sleep(waitIntervalMS);
- }
- } catch (Throwable t) {
- logger.error("Seems Solr is not up. solrDetail=" + solrDetail);
- }
- return status;
- }
-
- private QueryResponse process(SolrQuery solrQuery) throws SolrServerException, IOException, SolrException {
- if (solrClient != null) {
- QueryResponse queryResponse = solrClient.query(solrQuery, METHOD.POST);
- return queryResponse;
- } else {
- logger.error("solrClient can't be null");
- return null;
- }
- }
-
- public HashMap<String, Object> getConfigDoc() {
- HashMap<String, Object> configMap = new HashMap<String, Object>();
- SolrQuery solrQuery = new SolrQuery();
- solrQuery.setQuery("*:*");
- String fq = LogFeederConstants.ROW_TYPE + ":" + LogFeederConstants.LOGFEEDER_FILTER_NAME;
- solrQuery.setFilterQueries(fq);
- try {
- QueryResponse response = process(solrQuery);
- if (response != null) {
- SolrDocumentList documentList = response.getResults();
- if (documentList != null && documentList.size() > 0) {
- SolrDocument configDoc = documentList.get(0);
- String configJson = LogFeederUtil.getGson().toJson(configDoc);
- configMap = (HashMap<String, Object>) LogFeederUtil
- .toJSONObject(configJson);
- }
- }
- } catch (Exception e) {
- final String logMessageKey = this.getClass().getSimpleName()
- + "_FETCH_FILTER_CONFIG_ERROR";
- LogFeederUtil.logErrorMessageByInterval(logMessageKey,
- "Error getting filter config from solr", e, logger, Level.ERROR);
- }
- return configMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilter.java
deleted file mode 100644
index f030040..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilter.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.view;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class VLogfeederFilter {
-
- private String label;
- private List<String> hosts;
- private List<String> defaultLevels;
- private List<String> overrideLevels;
- private String expiryTime;
-
- public VLogfeederFilter() {
- hosts = new ArrayList<String>();
- defaultLevels = new ArrayList<String>();
- overrideLevels = new ArrayList<String>();
- }
-
- public String getLabel() {
- return label;
- }
-
- public void setLabel(String label) {
- this.label = label;
- }
-
- public List<String> getHosts() {
- return hosts;
- }
-
- public void setHosts(List<String> hosts) {
- this.hosts = hosts;
- }
-
- public List<String> getDefaultLevels() {
- return defaultLevels;
- }
-
- public void setDefaultLevels(List<String> defaultLevels) {
- this.defaultLevels = defaultLevels;
- }
-
- public List<String> getOverrideLevels() {
- return overrideLevels;
- }
-
- public void setOverrideLevels(List<String> overrideLevels) {
- this.overrideLevels = overrideLevels;
- }
-
- public String getExpiryTime() {
- return expiryTime;
- }
-
- public void setExpiryTime(String expiryTime) {
- this.expiryTime = expiryTime;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilterWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilterWrapper.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilterWrapper.java
deleted file mode 100644
index 4ddef3f..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/view/VLogfeederFilterWrapper.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder.view;
-
-import java.util.HashMap;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-
-@JsonAutoDetect(getterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE, fieldVisibility = Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
-@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
-public class VLogfeederFilterWrapper {
-
- private HashMap<String, VLogfeederFilter> filter;
- private String id;
-
- public HashMap<String, VLogfeederFilter> getFilter() {
- return filter;
- }
-
- public void setFilter(HashMap<String, VLogfeederFilter> filter) {
- this.filter = filter;
- }
-
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/AppTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/AppTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/AppTest.java
deleted file mode 100644
index 193cb48..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/AppTest.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logfeeder;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.ambari.logfeeder.filter.FilterGrok;
-import org.apache.log4j.Logger;
-
-import junit.framework.Test;
-import junit.framework.TestCase;
-import junit.framework.TestSuite;
-
-/**
- * Unit test for simple App.
- */
-public class AppTest extends TestCase {
- static Logger logger = Logger.getLogger(AppTest.class);
-
- /**
- * Create the test case
- *
- * @param testName name of the test case
- */
- public AppTest(String testName) {
- super(testName);
- }
-
- /**
- * @return the suite of tests being tested
- */
- public static Test suite() {
- return new TestSuite(AppTest.class);
- }
-
- /**
- * Rigourous Test :-)
- */
- public void testApp() {
- assertTrue(true);
- }
-
- public void testGrok() {
- logger.info("testGrok()");
- FilterGrok grokFilter = new FilterGrok();
- try {
- Map<String, Object> map = new HashMap<String, Object>();
- map.put("message_pattern",
- "^%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
- grokFilter.loadConfig(map);
- grokFilter.init();
- String out = grokFilter.grokParse("INFO This is a test");
- logger.info("out=" + out);
-
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- assertFalse(true);
- }
-
- assertTrue(true);
- }
-
- public void testGrokUGI() {
- logger.info("testGrok()");
- String[] ugis = new String[]{"user1@xyz.com (auth:TOKEN)",
- "ambari-qa@example.com (auth:kerberos)",
- "my_user@example.com (auth:kerberos)",
- "hive/bdurai-dojran-2.novalocal@example.com (auth:kerberos)",
- "just_me",
- "ambari-qa (auth:PROXY) via hive/myhost.novalocal@EXAMPLE.COM (auth:KERBEROS)"};
-
- FilterGrok grokFilter = new FilterGrok();
- try {
- Map<String, Object> map = new HashMap<String, Object>();
- // map.put("message_pattern",
- // "(?<user>([\\w\\d\\-]+))\\/|(?<user>([\\w\\d\\-]+))@|(?<user>([\\w\\d\\-]+))/[\\w\\d\\-.]+@|(?<user>([\\w\\d.\\-_]+))[\\s(]+");
- // map.put("message_pattern",
- // "(?<user>([\\w\\d\\-]+))/[\\w\\d\\-.]+@");
- // *(auth:(?<auth>[\\w\\d\\-]+))
- // GOOD: map.put("message_pattern", "(?<user>([\\w\\d\\-]+)).+auth:(?<auth>([\\w\\d\\-]+))");
- // OK: map.put("message_pattern", "(?<user>([\\w\\d\\-]+)).+auth:(?<auth>([\\w\\d\\-]+))|%{USERNAME:xuser}");
- //map.put("message_pattern", "%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}");
- map.put("message_pattern", "%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}");
- grokFilter.loadConfig(map);
- grokFilter.init();
- for (String ugi : ugis) {
- String out = grokFilter.grokParse(ugi);
- logger.info(ugi + "=" + out);
- }
-
- } catch (Exception e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- assertFalse(true);
- }
- assertTrue(true);
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
index 3aa8d7b..99565c5 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.output.OutputMgr;
+import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
@@ -40,16 +40,16 @@ public class FilterGrokTest {
private static final Logger LOG = Logger.getLogger(FilterGrokTest.class);
private FilterGrok filterGrok;
- private OutputMgr mockOutputMgr;
+ private OutputManager mockOutputManager;
private Capture<Map<String, Object>> capture;
public void init(Map<String, Object> config) throws Exception {
- mockOutputMgr = EasyMock.strictMock(OutputMgr.class);
+ mockOutputManager = EasyMock.strictMock(OutputManager.class);
capture = EasyMock.newCapture(CaptureType.LAST);
filterGrok = new FilterGrok();
filterGrok.loadConfig(config);
- filterGrok.setOutputMgr(mockOutputMgr);
+ filterGrok.setOutputManager(mockOutputManager);
filterGrok.setInput(EasyMock.mock(Input.class));
filterGrok.init();
}
@@ -59,19 +59,18 @@ public class FilterGrokTest {
LOG.info("testFilterGrok_parseMessage()");
Map<String, Object> config = new HashMap<String, Object>();
- config.put("message_pattern",
- "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
+ config.put("message_pattern", "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})");
init(config);
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterGrok.apply("2016-04-08 15:55:23,548 INFO This is a test", new InputMarker());
- filterGrok.apply("2016-04-08 15:55:24,548 WARN Next message", new InputMarker());
+ filterGrok.apply("2016-04-08 15:55:23,548 INFO This is a test", new InputMarker(null, null, 0));
+ filterGrok.apply("2016-04-08 15:55:24,548 WARN Next message", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertNotNull(jsonParams);
@@ -86,23 +85,22 @@ public class FilterGrokTest {
LOG.info("testFilterGrok_parseMultiLineMessage()");
Map<String, Object> config = new HashMap<String, Object>();
- config.put("message_pattern",
- "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
+ config.put("message_pattern", "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})");
init(config);
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
String multiLineMessage = "This is a multiline test message\r\n" + "having multiple lines\r\n"
+ "as one may expect";
String[] messageLines = multiLineMessage.split("\r\n");
for (int i = 0; i < messageLines.length; i++)
- filterGrok.apply((i == 0 ? "2016-04-08 15:55:23,548 INFO " : "") + messageLines[i], new InputMarker());
+ filterGrok.apply((i == 0 ? "2016-04-08 15:55:23,548 INFO " : "") + messageLines[i], new InputMarker(null, null, 0));
filterGrok.flush();
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertNotNull(jsonParams);
@@ -117,19 +115,18 @@ public class FilterGrokTest {
LOG.info("testFilterGrok_notMatchingMesagePattern()");
Map<String, Object> config = new HashMap<String, Object>();
- config.put("message_pattern",
- "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
+ config.put("message_pattern", "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}");
config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})");
init(config);
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall().anyTimes();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterGrok.apply("04/08/2016 15:55:23,548 INFO This is a test", new InputMarker());
- filterGrok.apply("04/08/2016 15:55:24,548 WARN Next message", new InputMarker());
+ filterGrok.apply("04/08/2016 15:55:23,548 INFO This is a test", new InputMarker(null, null, 0));
+ filterGrok.apply("04/08/2016 15:55:24,548 WARN Next message", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
assertFalse("Something was captured!", capture.hasCaptured());
}
@@ -141,12 +138,12 @@ public class FilterGrokTest {
config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})");
init(config);
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterGrok.apply("2016-04-08 15:55:23,548 INFO This is a test", new InputMarker());
- filterGrok.apply("2016-04-08 15:55:24,548 WARN Next message", new InputMarker());
+ filterGrok.apply("2016-04-08 15:55:23,548 INFO This is a test", new InputMarker(null, null, 0));
+ filterGrok.apply("2016-04-08 15:55:24,548 WARN Next message", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
assertFalse("Something was captured", capture.hasCaptured());
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 64e9b69..06d8db2 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -27,8 +27,7 @@ import java.util.TimeZone;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.output.OutputMgr;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
@@ -44,16 +43,16 @@ public class FilterJSONTest {
private static final Logger LOG = Logger.getLogger(FilterJSONTest.class);
private FilterJSON filterJson;
- private OutputMgr mockOutputMgr;
+ private OutputManager mockOutputManager;
private Capture<Map<String, Object>> capture;
public void init(Map<String, Object> params) throws Exception {
- mockOutputMgr = EasyMock.strictMock(OutputMgr.class);
+ mockOutputManager = EasyMock.strictMock(OutputManager.class);
capture = EasyMock.newCapture(CaptureType.LAST);
filterJson = new FilterJSON();
filterJson.loadConfig(params);
- filterJson.setOutputMgr(mockOutputMgr);
+ filterJson.setOutputManager(mockOutputManager);
filterJson.init();
}
@@ -63,17 +62,17 @@ public class FilterJSONTest {
init(new HashMap<String, Object>());
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
Date d = new Date();
- DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT);
+ DateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
String dateString = sdf.format(d);
- filterJson.apply("{ logtime: '" + d.getTime() + "', line_number: 100 }", new InputMarker());
+ filterJson.apply("{ logtime: '" + d.getTime() + "', line_number: 100 }", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Incorrect decoding: log time", dateString, jsonParams.remove("logtime"));
@@ -87,17 +86,17 @@ public class FilterJSONTest {
init(new HashMap<String, Object>());
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
Date d = new Date();
- DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT);
+ DateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
String dateString = sdf.format(d);
- filterJson.apply("{ logtime: '" + d.getTime() + "', some_field: 'abc' }", new InputMarker());
+ filterJson.apply("{ logtime: '" + d.getTime() + "', some_field: 'abc' }", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Incorrect decoding: log time", dateString, jsonParams.remove("logtime"));
@@ -111,13 +110,13 @@ public class FilterJSONTest {
init(new HashMap<String, Object>());
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterJson.apply("{ line_number: 100, some_field: 'abc' }", new InputMarker());
+ filterJson.apply("{ line_number: 100, some_field: 'abc' }", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Incorrect decoding: line number", 100l, jsonParams.remove("line_number"));
@@ -132,7 +131,7 @@ public class FilterJSONTest {
init(new HashMap<String, Object>());
String inputStr="invalid json";
try{
- filterJson.apply(inputStr,new InputMarker());
+ filterJson.apply(inputStr,new InputMarker(null, null, 0));
fail("Expected LogfeederException was not occured");
}catch(LogfeederException logfeederException){
assertEquals("Json parsing failed for inputstr = "+inputStr, logfeederException.getLocalizedMessage());
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
index 849e4c3..30cee42 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -22,7 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.output.OutputMgr;
+import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.log4j.Logger;
import org.easymock.Capture;
import org.easymock.CaptureType;
@@ -38,16 +38,16 @@ public class FilterKeyValueTest {
private static final Logger LOG = Logger.getLogger(FilterKeyValueTest.class);
private FilterKeyValue filterKeyValue;
- private OutputMgr mockOutputMgr;
+ private OutputManager mockOutputManager;
private Capture<Map<String, Object>> capture;
public void init(Map<String, Object> config) throws Exception {
- mockOutputMgr = EasyMock.strictMock(OutputMgr.class);
+ mockOutputManager = EasyMock.strictMock(OutputManager.class);
capture = EasyMock.newCapture(CaptureType.LAST);
filterKeyValue = new FilterKeyValue();
filterKeyValue.loadConfig(config);
- filterKeyValue.setOutputMgr(mockOutputMgr);
+ filterKeyValue.setOutputManager(mockOutputManager);
filterKeyValue.init();
}
@@ -61,13 +61,13 @@ public class FilterKeyValueTest {
// using default value split:
init(config);
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterKeyValue.apply("{ keyValueField: 'name1=value1&name2=value2' }", new InputMarker());
+ filterKeyValue.apply("{ keyValueField: 'name1=value1&name2=value2' }", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Original missing!", "name1=value1&name2=value2", jsonParams.remove("keyValueField"));
@@ -85,13 +85,13 @@ public class FilterKeyValueTest {
// using default value split: =
init(config);
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall().anyTimes();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterKeyValue.apply("{ keyValueField: 'name1=value1&name2=value2' }", new InputMarker());
+ filterKeyValue.apply("{ keyValueField: 'name1=value1&name2=value2' }", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
assertFalse("Something was captured!", capture.hasCaptured());
}
@@ -105,13 +105,13 @@ public class FilterKeyValueTest {
init(config);
// using default value split: =
- mockOutputMgr.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
+ mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class));
EasyMock.expectLastCall().anyTimes();
- EasyMock.replay(mockOutputMgr);
+ EasyMock.replay(mockOutputManager);
- filterKeyValue.apply("{ otherField: 'name1=value1&name2=value2' }", new InputMarker());
+ filterKeyValue.apply("{ otherField: 'name1=value1&name2=value2' }", new InputMarker(null, null, 0));
- EasyMock.verify(mockOutputMgr);
+ EasyMock.verify(mockOutputManager);
Map<String, Object> jsonParams = capture.getValue();
assertEquals("Original missing!", "name1=value1&name2=value2", jsonParams.remove("otherField"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
index 42e81da..08aa564 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -104,7 +104,7 @@ public class InputFileTest {
inputFile = new InputFile();
inputFile.loadConfig(config);
- inputFile.setFirstFilter(capture);
+ inputFile.addFilter(capture);
inputFile.init();
}
@@ -117,10 +117,10 @@ public class InputFileTest {
init(testFile.getAbsolutePath());
- InputMgr inputMgr = EasyMock.createStrictMock(InputMgr.class);
- EasyMock.expect(inputMgr.getCheckPointFolderFile()).andReturn(checkPointDir);
- EasyMock.replay(inputMgr);
- inputFile.setInputMgr(inputMgr);
+ InputManager inputManager = EasyMock.createStrictMock(InputManager.class);
+ EasyMock.expect(inputManager.getCheckPointFolderFile()).andReturn(checkPointDir);
+ EasyMock.replay(inputManager);
+ inputFile.setInputManager(inputManager);
inputFile.isReady();
inputFile.start();
@@ -129,7 +129,7 @@ public class InputFileTest {
for (int row = 0; row < 3; row++)
assertEquals("Row #" + (row + 1) + " not correct", TEST_LOG_FILE_ROWS[row], rows.get(row));
- EasyMock.verify(inputMgr);
+ EasyMock.verify(inputManager);
}
@Test
@@ -140,10 +140,10 @@ public class InputFileTest {
File testFile = createFile("process6.log");
init(testFile.getAbsolutePath());
- InputMgr inputMgr = EasyMock.createStrictMock(InputMgr.class);
- EasyMock.expect(inputMgr.getCheckPointFolderFile()).andReturn(checkPointDir).times(2);
- EasyMock.replay(inputMgr);
- inputFile.setInputMgr(inputMgr);
+ InputManager inputMabager = EasyMock.createStrictMock(InputManager.class);
+ EasyMock.expect(inputMabager.getCheckPointFolderFile()).andReturn(checkPointDir).times(2);
+ EasyMock.replay(inputMabager);
+ inputFile.setInputManager(inputMabager);
inputFile.isReady();
inputFile.start();
@@ -155,7 +155,7 @@ public class InputFileTest {
for (int row = 0; row < 6; row++)
assertEquals("Row #" + (row + 1) + " not correct", TEST_LOG_FILE_ROWS[row], rows.get(row));
- EasyMock.verify(inputMgr);
+ EasyMock.verify(inputMabager);
}
@Test
[06/10] ambari git commit: AMBARI-18246. Clean up Log Feeder (Miklos
Gergely via oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
index 5feb9c4..e13d9bd 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java
@@ -21,7 +21,6 @@ package org.apache.ambari.logfeeder.input;
import java.io.File;
import java.util.ArrayList;
-import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -29,88 +28,138 @@ import java.util.Map;
import org.apache.ambari.logfeeder.common.ConfigBlock;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.filter.Filter;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
+import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.output.Output;
-import org.apache.ambari.logfeeder.output.OutputMgr;
+import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.log4j.Logger;
public abstract class Input extends ConfigBlock implements Runnable {
- static private Logger logger = Logger.getLogger(Input.class);
-
- protected OutputMgr outputMgr;
- protected InputMgr inputMgr;
+ private static final Logger LOG = Logger.getLogger(Input.class);
+ private static final boolean DEFAULT_TAIL = true;
+ private static final boolean DEFAULT_USE_EVENT_MD5 = false;
+ private static final boolean DEFAULT_GEN_EVENT_MD5 = true;
+
+ protected InputManager inputManager;
+ protected OutputManager outputManager;
private List<Output> outputList = new ArrayList<Output>();
- private Filter firstFilter = null;
private Thread thread;
- private boolean isClosed = false;
- protected String filePath = null;
- private String type = null;
+ private String type;
+ protected String filePath;
+ private Filter firstFilter;
+ private boolean isClosed;
- protected boolean tail = true;
- private boolean useEventMD5 = false;
- private boolean genEventMD5 = true;
+ protected boolean tail;
+ private boolean useEventMD5;
+ private boolean genEventMD5;
- protected MetricCount readBytesMetric = new MetricCount();
+ protected MetricData readBytesMetric = new MetricData(getReadBytesMetricName(), false);
+ protected String getReadBytesMetricName() {
+ return null;
+ }
+
+ @Override
+ public void loadConfig(Map<String, Object> map) {
+ super.loadConfig(map);
+ String typeValue = getStringValue("type");
+ if (typeValue != null) {
+ // Explicitly add type and value to field list
+ contextFields.put("type", typeValue);
+ @SuppressWarnings("unchecked")
+ Map<String, Object> addFields = (Map<String, Object>) map.get("add_fields");
+ if (addFields == null) {
+ addFields = new HashMap<String, Object>();
+ map.put("add_fields", addFields);
+ }
+ addFields.put("type", typeValue);
+ }
+ }
- /**
- * This method will be called from the thread spawned for the output. This
- * method should only exit after all data are read from the source or the
- * process is exiting
- */
- abstract void start() throws Exception;
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public void setInputManager(InputManager inputManager) {
+ this.inputManager = inputManager;
+ }
+
+ public void setOutputManager(OutputManager outputManager) {
+ this.outputManager = outputManager;
+ }
+
+ public void addFilter(Filter filter) {
+ if (firstFilter == null) {
+ firstFilter = filter;
+ } else {
+ Filter f = firstFilter;
+ while (f.getNextFilter() != null) {
+ f = f.getNextFilter();
+ }
+ f.setNextFilter(filter);
+ }
+ }
+
+ public void addOutput(Output output) {
+ outputList.add(output);
+ }
@Override
public void init() throws Exception {
super.init();
- tail = getBooleanValue("tail", tail);
- useEventMD5 = getBooleanValue("use_event_md5_as_id", useEventMD5);
- genEventMD5 = getBooleanValue("gen_event_md5", genEventMD5);
+ tail = getBooleanValue("tail", DEFAULT_TAIL);
+ useEventMD5 = getBooleanValue("use_event_md5_as_id", DEFAULT_USE_EVENT_MD5);
+ genEventMD5 = getBooleanValue("gen_event_md5", DEFAULT_GEN_EVENT_MD5);
if (firstFilter != null) {
firstFilter.init();
}
}
- @Override
- public String getNameForThread() {
- if (filePath != null) {
- try {
- return (type + "=" + (new File(filePath)).getName());
- } catch (Throwable ex) {
- logger.warn("Couldn't get basename for filePath=" + filePath,
- ex);
- }
+ boolean monitor() {
+ if (isReady()) {
+ LOG.info("Starting thread. " + getShortDescription());
+ thread = new Thread(this, getNameForThread());
+ thread.start();
+ return true;
+ } else {
+ return false;
}
- return super.getNameForThread() + ":" + type;
}
+ public abstract boolean isReady();
+
@Override
public void run() {
try {
- logger.info("Started to monitor. " + getShortDescription());
+ LOG.info("Started to monitor. " + getShortDescription());
start();
} catch (Exception e) {
- logger.error("Error writing to output.", e);
+ LOG.error("Error writing to output.", e);
}
- logger.info("Exiting thread. " + getShortDescription());
+ LOG.info("Exiting thread. " + getShortDescription());
}
+ /**
+ * This method will be called from the thread spawned for the output. This
+ * method should only exit after all data are read from the source or the
+ * process is exiting
+ */
+ abstract void start() throws Exception;
+
protected void outputLine(String line, InputMarker marker) {
- statMetric.count++;
- readBytesMetric.count += (line.length());
+ statMetric.value++;
+ readBytesMetric.value += (line.length());
if (firstFilter != null) {
try {
firstFilter.apply(line, marker);
} catch (LogfeederException e) {
- logger.error(e.getLocalizedMessage(),e);
+ LOG.error(e.getLocalizedMessage(), e);
}
} else {
- // TODO: For now, let's make filter mandatory, so that no one
- // accidently forgets to write filter
- // outputMgr.write(line, this);
+ // TODO: For now, let's make filter mandatory, so that no one accidently forgets to write filter
+ // outputManager.write(line, this);
}
}
@@ -120,60 +169,10 @@ public abstract class Input extends ConfigBlock implements Runnable {
}
}
- public boolean monitor() {
- if (isReady()) {
- logger.info("Starting thread. " + getShortDescription());
- thread = new Thread(this, getNameForThread());
- thread.start();
- return true;
- } else {
- return false;
- }
- }
-
- public void checkIn(InputMarker inputMarker) {
- // Default implementation is to ignore.
- }
-
- /**
- * This is generally used by final checkin
- */
- public void checkIn() {
- }
-
- public boolean isReady() {
- return true;
- }
-
- public boolean isTail() {
- return tail;
- }
-
- public void setTail(boolean tail) {
- this.tail = tail;
- }
-
- public boolean isUseEventMD5() {
- return useEventMD5;
- }
-
- public void setUseEventMD5(boolean useEventMD5) {
- this.useEventMD5 = useEventMD5;
- }
-
- public boolean isGenEventMD5() {
- return genEventMD5;
- }
-
- public void setGenEventMD5(boolean genEventMD5) {
- this.genEventMD5 = genEventMD5;
- }
-
@Override
public void setDrain(boolean drain) {
- logger.info("Request to drain. " + getShortDescription());
+ LOG.info("Request to drain. " + getShortDescription());
super.setDrain(drain);
- ;
try {
thread.interrupt();
} catch (Throwable t) {
@@ -181,38 +180,36 @@ public abstract class Input extends ConfigBlock implements Runnable {
}
}
- public Filter getFirstFilter() {
- return firstFilter;
- }
-
- public void setFirstFilter(Filter filter) {
- firstFilter = filter;
+ public void addMetricsContainers(List<MetricData> metricsList) {
+ super.addMetricsContainers(metricsList);
+ if (firstFilter != null) {
+ firstFilter.addMetricsContainers(metricsList);
+ }
+ metricsList.add(readBytesMetric);
}
- public void setInputMgr(InputMgr inputMgr) {
- this.inputMgr = inputMgr;
- }
+ @Override
+ public void logStat() {
+ super.logStat();
+ logStatForMetric(readBytesMetric, "Stat: Bytes Read");
- public void setOutputMgr(OutputMgr outputMgr) {
- this.outputMgr = outputMgr;
+ if (firstFilter != null) {
+ firstFilter.logStat();
+ }
}
- public String getFilePath() {
- return filePath;
- }
+ public abstract void checkIn(InputMarker inputMarker);
- public void setFilePath(String filePath) {
- this.filePath = filePath;
- }
+ public abstract void lastCheckIn();
public void close() {
- logger.info("Close called. " + getShortDescription());
+ LOG.info("Close called. " + getShortDescription());
try {
if (firstFilter != null) {
firstFilter.close();
} else {
- outputMgr.close();
+ outputManager.close();
}
} catch (Throwable t) {
// Ignore
@@ -220,86 +217,60 @@ public abstract class Input extends ConfigBlock implements Runnable {
isClosed = true;
}
- public void setClosed(boolean isClosed) {
- this.isClosed = isClosed;
- }
-
- public boolean isClosed() {
- return isClosed;
- }
-
- @Override
- public void loadConfig(Map<String, Object> map) {
- super.loadConfig(map);
- String typeValue = getStringValue("type");
- if (typeValue != null) {
- // Explicitly add type and value to field list
- contextFields.put("type", typeValue);
- @SuppressWarnings("unchecked")
- Map<String, Object> addFields = (Map<String, Object>) map
- .get("add_fields");
- if (addFields == null) {
- addFields = new HashMap<String, Object>();
- map.put("add_fields", addFields);
- }
- addFields.put("type", typeValue);
- }
+ public boolean isTail() {
+ return tail;
}
- @Override
- public String getShortDescription() {
- return null;
+ public boolean isUseEventMD5() {
+ return useEventMD5;
}
- @Override
- public void logStat() {
- super.logStat();
- logStatForMetric(readBytesMetric, "Stat: Bytes Read");
-
- if (firstFilter != null) {
- firstFilter.logStat();
- }
+ public boolean isGenEventMD5() {
+ return genEventMD5;
}
- @Override
- public String toString() {
- return getShortDescription();
+ public Filter getFirstFilter() {
+ return firstFilter;
}
- public void rollOver() {
- // Only some inputs support it. E.g. InputFile
+ public String getFilePath() {
+ return filePath;
}
- public String getType() {
- return type;
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
}
- public void setType(String type) {
- this.type = type;
+ public void setClosed(boolean isClosed) {
+ this.isClosed = isClosed;
}
- public Date getEventTime() {
- return null;
+ public boolean isClosed() {
+ return isClosed;
}
public List<Output> getOutputList() {
return outputList;
}
-
- public void addOutput(Output output) {
- outputList.add(output);
- }
-
- public void addMetricsContainers(List<MetricCount> metricsList) {
- super.addMetricsContainers(metricsList);
- if (firstFilter != null) {
- firstFilter.addMetricsContainers(metricsList);
- }
- metricsList.add(readBytesMetric);
- }
public Thread getThread(){
return thread;
}
+ @Override
+ public String getNameForThread() {
+ if (filePath != null) {
+ try {
+ return (type + "=" + (new File(filePath)).getName());
+ } catch (Throwable ex) {
+ LOG.warn("Couldn't get basename for filePath=" + filePath, ex);
+ }
+ }
+ return super.getNameForThread() + ":" + type;
+ }
+
+ @Override
+ public String toString() {
+ return getShortDescription();
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputFile.java
index c9f5ded..3737839 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputFile.java
@@ -19,528 +19,99 @@
package org.apache.ambari.logfeeder.input;
import java.io.BufferedReader;
-import java.io.EOFException;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.attribute.BasicFileAttributes;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.ambari.logfeeder.input.reader.LogsearchReaderFactory;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.ambari.logfeeder.util.FileUtil;
import org.apache.commons.io.filefilter.WildcardFileFilter;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.solr.common.util.Base64;
-public class InputFile extends Input {
- private static final Logger logger = Logger.getLogger(InputFile.class);
-
- private String logPath = null;
- private boolean isStartFromBegining = true;
-
- private boolean isReady = false;
- private File[] logPathFiles = null;
- private Object fileKey = null;
- private String base64FileKey = null;
-
- private boolean isRolledOver = false;
- private boolean addWildCard = false;
-
- private long lastCheckPointTimeMS = 0;
- private int checkPointIntervalMS = 5 * 1000; // 5 seconds
- private RandomAccessFile checkPointWriter = null;
- private Map<String, Object> jsonCheckPoint = null;
-
- private File checkPointFile = null;
-
- private InputMarker lastCheckPointInputMarker = null;
-
- private String checkPointExtension = ".cp";
-
- @Override
- public void init() throws Exception {
- logger.info("init() called");
- statMetric.metricsName = "input.files.read_lines";
- readBytesMetric.metricsName = "input.files.read_bytes";
- checkPointExtension = LogFeederUtil.getStringProperty(
- "logfeeder.checkpoint.extension", checkPointExtension);
-
- // Let's close the file and set it to true after we start monitoring it
- setClosed(true);
- logPath = getStringValue("path");
- tail = getBooleanValue("tail", tail);
- addWildCard = getBooleanValue("add_wild_card", addWildCard);
- checkPointIntervalMS = getIntValue("checkpoint.interval.ms",
- checkPointIntervalMS);
-
- if (logPath == null || logPath.isEmpty()) {
- logger.error("path is empty for file input. "
- + getShortDescription());
- return;
- }
-
- String startPosition = getStringValue("start_position");
- if (StringUtils.isEmpty(startPosition)
- || startPosition.equalsIgnoreCase("beginning")
- || startPosition.equalsIgnoreCase("begining")) {
- isStartFromBegining = true;
- }
-
- if (!tail) {
- // start position end doesn't apply if we are not tailing
- isStartFromBegining = true;
- }
-
- setFilePath(logPath);
- boolean isFileReady = isReady();
-
- logger.info("File to monitor " + logPath + ", tail=" + tail
- + ", addWildCard=" + addWildCard + ", isReady=" + isFileReady);
-
- super.init();
- }
+public class InputFile extends AbstractInputFile {
@Override
public boolean isReady() {
if (!isReady) {
// Let's try to check whether the file is available
- logPathFiles = getActualFiles(logPath);
- if (logPathFiles != null && logPathFiles.length > 0
- && logPathFiles[0].isFile()) {
-
- if (isTail() && logPathFiles.length > 1) {
- logger.warn("Found multiple files (" + logPathFiles.length
- + ") for the file filter " + filePath
- + ". Will use only the first one. Using "
- + logPathFiles[0].getAbsolutePath());
+ logFiles = getActualFiles(logPath);
+ if (!ArrayUtils.isEmpty(logFiles) && logFiles[0].isFile()) {
+ if (tail && logFiles.length > 1) {
+ LOG.warn("Found multiple files (" + logFiles.length + ") for the file filter " + filePath +
+ ". Will use only the first one. Using " + logFiles[0].getAbsolutePath());
}
- logger.info("File filter " + filePath + " expanded to "
- + logPathFiles[0].getAbsolutePath());
+ LOG.info("File filter " + filePath + " expanded to " + logFiles[0].getAbsolutePath());
isReady = true;
} else {
- logger.debug(logPath + " file doesn't exist. Ignoring for now");
+ LOG.debug(logPath + " file doesn't exist. Ignoring for now");
}
}
return isReady;
}
private File[] getActualFiles(String searchPath) {
- if (addWildCard) {
- if (!searchPath.endsWith("*")) {
- searchPath = searchPath + "*";
- }
- }
- File checkFile = new File(searchPath);
- if (checkFile.isFile()) {
- return new File[]{checkFile};
+ File searchFile = new File(searchPath);
+ if (searchFile.isFile()) {
+ return new File[]{searchFile};
+ } else {
+ FileFilter fileFilter = new WildcardFileFilter(searchFile.getName());
+ return searchFile.getParentFile().listFiles(fileFilter);
}
- // Let's do wild card search
- // First check current folder
- File checkFiles[] = findFileForWildCard(searchPath, new File("."));
- if (checkFiles == null || checkFiles.length == 0) {
- // Let's check from the parent folder
- File parentDir = (new File(searchPath)).getParentFile();
- if (parentDir != null) {
- String wildCard = (new File(searchPath)).getName();
- checkFiles = findFileForWildCard(wildCard, parentDir);
- }
- }
- return checkFiles;
- }
-
- private File[] findFileForWildCard(String searchPath, File dir) {
- logger.debug("findFileForWildCard(). filePath=" + searchPath + ", dir="
- + dir + ", dir.fullpath=" + dir.getAbsolutePath());
- FileFilter fileFilter = new WildcardFileFilter(searchPath);
- return dir.listFiles(fileFilter);
- }
-
- @Override
- synchronized public void checkIn(InputMarker inputMarker) {
- super.checkIn(inputMarker);
- if (checkPointWriter != null) {
- try {
- int lineNumber = LogFeederUtil.objectToInt(
- jsonCheckPoint.get("line_number"), 0, "line_number");
- if (lineNumber > inputMarker.lineNumber) {
- // Already wrote higher line number for this input
- return;
- }
- // If interval is greater than last checkPoint time, then write
- long currMS = System.currentTimeMillis();
- if (!isClosed()
- && (currMS - lastCheckPointTimeMS) < checkPointIntervalMS) {
- // Let's save this one so we can update the check point file
- // on flush
- lastCheckPointInputMarker = inputMarker;
- return;
- }
- lastCheckPointTimeMS = currMS;
-
- jsonCheckPoint.put("line_number", ""
- + new Integer(inputMarker.lineNumber));
- jsonCheckPoint.put("last_write_time_ms", "" + new Long(currMS));
- jsonCheckPoint.put("last_write_time_date", new Date());
-
- String jsonStr = LogFeederUtil.getGson().toJson(jsonCheckPoint);
-
- // Let's rewind
- checkPointWriter.seek(0);
- checkPointWriter.writeInt(jsonStr.length());
- checkPointWriter.write(jsonStr.getBytes());
-
- if (isClosed()) {
- final String LOG_MESSAGE_KEY = this.getClass()
- .getSimpleName() + "_FINAL_CHECKIN";
- LogFeederUtil.logErrorMessageByInterval(
- LOG_MESSAGE_KEY,
- "Wrote final checkPoint, input="
- + getShortDescription()
- + ", checkPointFile="
- + checkPointFile.getAbsolutePath()
- + ", checkPoint=" + jsonStr, null, logger,
- Level.INFO);
- }
- } catch (Throwable t) {
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_CHECKIN_EXCEPTION";
- LogFeederUtil
- .logErrorMessageByInterval(LOG_MESSAGE_KEY,
- "Caught exception checkIn. , input="
- + getShortDescription(), t, logger,
- Level.ERROR);
- }
- }
-
- }
-
- @Override
- public void checkIn() {
- super.checkIn();
- if (lastCheckPointInputMarker != null) {
- checkIn(lastCheckPointInputMarker);
- }
- }
-
- @Override
- public void rollOver() {
- logger.info("Marking this input file for rollover. "
- + getShortDescription());
- isRolledOver = true;
}
@Override
void start() throws Exception {
-
- if (logPathFiles == null || logPathFiles.length == 0) {
- return;
- }
boolean isProcessFile = getBooleanValue("process_file", true);
if (isProcessFile) {
- if (isTail()) {
- processFile(logPathFiles[0]);
+ if (tail) {
+ processFile(logFiles[0]);
} else {
- for (File file : logPathFiles) {
+ for (File file : logFiles) {
try {
processFile(file);
if (isClosed() || isDrain()) {
- logger.info("isClosed or isDrain. Now breaking loop.");
+ LOG.info("isClosed or isDrain. Now breaking loop.");
break;
}
} catch (Throwable t) {
- logger.error("Error processing file=" + file.getAbsolutePath(), t);
+ LOG.error("Error processing file=" + file.getAbsolutePath(), t);
}
}
}
close();
- }else{
- copyFiles(logPathFiles);
+ } else {
+ copyFiles(logFiles);
}
-
}
@Override
- public void close() {
- super.close();
- logger.info("close() calling checkPoint checkIn(). "
- + getShortDescription());
- checkIn();
- }
-
- private void processFile(File logPathFile) throws FileNotFoundException,
- IOException {
- logger.info("Monitoring logPath=" + logPath + ", logPathFile="
- + logPathFile);
- BufferedReader br = null;
- checkPointFile = null;
- checkPointWriter = null;
- jsonCheckPoint = null;
- int resumeFromLineNumber = 0;
-
- int lineCount = 0;
- try {
- setFilePath(logPathFile.getAbsolutePath());
- br = new BufferedReader(LogsearchReaderFactory.INSTANCE.getReader(logPathFile));
-
- // Whether to send to output from the beginning.
- boolean resume = isStartFromBegining;
-
- // Seems FileWatch is not reliable, so let's only use file key comparison
- fileKey = getFileKey(logPathFile);
- base64FileKey = Base64.byteArrayToBase64(fileKey.toString()
- .getBytes());
- logger.info("fileKey=" + fileKey + ", base64=" + base64FileKey
- + ". " + getShortDescription());
-
- if (isTail()) {
- try {
- logger.info("Checking existing checkpoint file. "
- + getShortDescription());
-
- String fileBase64 = Base64.byteArrayToBase64(fileKey
- .toString().getBytes());
- String checkPointFileName = fileBase64
- + checkPointExtension;
- File checkPointFolder = inputMgr.getCheckPointFolderFile();
- checkPointFile = new File(checkPointFolder,
- checkPointFileName);
- checkPointWriter = new RandomAccessFile(checkPointFile,
- "rw");
-
- try {
- int contentSize = checkPointWriter.readInt();
- byte b[] = new byte[contentSize];
- int readSize = checkPointWriter.read(b, 0, contentSize);
- if (readSize != contentSize) {
- logger.error("Couldn't read expected number of bytes from checkpoint file. expected="
- + contentSize
- + ", read="
- + readSize
- + ", checkPointFile="
- + checkPointFile
- + ", input=" + getShortDescription());
- } else {
- String jsonCheckPointStr = new String(b, 0, readSize);
- jsonCheckPoint = LogFeederUtil
- .toJSONObject(jsonCheckPointStr);
-
- resumeFromLineNumber = LogFeederUtil.objectToInt(
- jsonCheckPoint.get("line_number"), 0,
- "line_number");
-
- if (resumeFromLineNumber > 0) {
- // Let's read from last line read
- resume = false;
- }
- logger.info("CheckPoint. checkPointFile="
- + checkPointFile + ", json="
- + jsonCheckPointStr
- + ", resumeFromLineNumber="
- + resumeFromLineNumber + ", resume="
- + resume);
- }
- } catch (EOFException eofEx) {
- logger.info("EOFException. Will reset checkpoint file "
- + checkPointFile.getAbsolutePath() + " for "
- + getShortDescription());
- }
- if (jsonCheckPoint == null) {
- // This seems to be first time, so creating the initial
- // checkPoint object
- jsonCheckPoint = new HashMap<String, Object>();
- jsonCheckPoint.put("file_path", filePath);
- jsonCheckPoint.put("file_key", fileBase64);
- }
-
- } catch (Throwable t) {
- logger.error(
- "Error while configuring checkpoint file. Will reset file. checkPointFile="
- + checkPointFile, t);
- }
- }
-
- setClosed(false);
- int sleepStep = 2;
- int sleepIteration = 0;
- while (true) {
- try {
- if (isDrain()) {
- break;
- }
-
- String line = br.readLine();
- if (line == null) {
- if (!resume) {
- resume = true;
- }
- sleepIteration++;
- try {
- // Since FileWatch service is not reliable, we will check
- // file inode every n seconds after no write
- if (sleepIteration > 4) {
- Object newFileKey = getFileKey(logPathFile);
- if (newFileKey != null) {
- if (fileKey == null
- || !newFileKey.equals(fileKey)) {
- logger.info("File key is different. Calling rollover. oldKey="
- + fileKey
- + ", newKey="
- + newFileKey
- + ". "
- + getShortDescription());
- // File has rotated.
- rollOver();
- }
- }
- }
- // Flush on the second iteration
- if (!tail && sleepIteration >= 2) {
- logger.info("End of file. Done with filePath="
- + logPathFile.getAbsolutePath()
- + ", lineCount=" + lineCount);
- flush();
- break;
- } else if (sleepIteration == 2) {
- flush();
- } else if (sleepIteration >= 2) {
- if (isRolledOver) {
- isRolledOver = false;
- // Close existing file
- try {
- logger.info("File is rolled over. Closing current open file."
- + getShortDescription()
- + ", lineCount=" + lineCount);
- br.close();
- } catch (Exception ex) {
- logger.error("Error closing file"
- + getShortDescription());
- break;
- }
- try {
- logger.info("Opening new rolled over file."
- + getShortDescription());
- br = new BufferedReader(LogsearchReaderFactory.
- INSTANCE.getReader(logPathFile));
- lineCount = 0;
- fileKey = getFileKey(logPathFile);
- base64FileKey = Base64
- .byteArrayToBase64(fileKey
- .toString().getBytes());
- logger.info("fileKey=" + fileKey
- + ", base64=" + base64FileKey
- + ", " + getShortDescription());
- } catch (Exception ex) {
- logger.error("Error opening rolled over file. "
- + getShortDescription());
- // Let's add this to monitoring and exit this thread
- logger.info("Added input to not ready list."
- + getShortDescription());
- isReady = false;
- inputMgr.addToNotReady(this);
- break;
- }
- logger.info("File is successfully rolled over. "
- + getShortDescription());
- continue;
- }
- }
- Thread.sleep(sleepStep * 1000);
- sleepStep = (sleepStep * 2);
- sleepStep = sleepStep > 10 ? 10 : sleepStep;
- } catch (InterruptedException e) {
- logger.info("Thread interrupted."
- + getShortDescription());
- }
- } else {
- lineCount++;
- sleepStep = 1;
- sleepIteration = 0;
-
- if (!resume && lineCount > resumeFromLineNumber) {
- logger.info("Resuming to read from last line. lineCount="
- + lineCount
- + ", input="
- + getShortDescription());
- resume = true;
- }
- if (resume) {
- InputMarker marker = new InputMarker();
- marker.base64FileKey = base64FileKey;
- marker.input = this;
- marker.lineNumber = lineCount;
- outputLine(line, marker);
- }
- }
- } catch (Throwable t) {
- final String LOG_MESSAGE_KEY = this.getClass()
- .getSimpleName() + "_READ_LOOP_EXCEPTION";
- LogFeederUtil.logErrorMessageByInterval(LOG_MESSAGE_KEY,
- "Caught exception in read loop. lineNumber="
- + lineCount + ", input="
- + getShortDescription(), t, logger,
- Level.ERROR);
-
- }
- }
- } finally {
- if (br != null) {
- logger.info("Closing reader." + getShortDescription()
- + ", lineCount=" + lineCount);
- try {
- br.close();
- } catch (Throwable t) {
- // ignore
- }
- }
- }
- }
-
- static public Object getFileKey(File file) {
- try {
- Path fileFullPath = Paths.get(file.getAbsolutePath());
- if (fileFullPath != null) {
- BasicFileAttributes basicAttr = Files.readAttributes(
- fileFullPath, BasicFileAttributes.class);
- return basicAttr.fileKey();
- }
- } catch (Throwable ex) {
- logger.error("Error getting file attributes for file=" + file, ex);
- }
- return file.toString();
+ protected BufferedReader openLogFile(File logFile) throws FileNotFoundException {
+ BufferedReader br = new BufferedReader(LogsearchReaderFactory.INSTANCE.getReader(logFile));
+ fileKey = getFileKey(logFile);
+ base64FileKey = Base64.byteArrayToBase64(fileKey.toString().getBytes());
+ LOG.info("fileKey=" + fileKey + ", base64=" + base64FileKey + ". " + getShortDescription());
+ return br;
}
@Override
- public String getShortDescription() {
- return "input:source="
- + getStringValue("source")
- + ", path="
- + (logPathFiles != null && logPathFiles.length > 0 ? logPathFiles[0]
- .getAbsolutePath() : getStringValue("path"));
+ protected Object getFileKey(File logFile) {
+ return FileUtil.getFileKey(logFile);
}
-
- public void copyFiles(File[] files) {
+
+ private void copyFiles(File[] files) {
boolean isCopyFile = getBooleanValue("copy_file", false);
if (isCopyFile && files != null) {
for (File file : files) {
try {
- InputMarker marker = new InputMarker();
- marker.input = this;
- outputMgr.copyFile(file, marker);
+ InputMarker marker = new InputMarker(this, null, 0);
+ outputManager.copyFile(file, marker);
if (isClosed() || isDrain()) {
- logger.info("isClosed or isDrain. Now breaking loop.");
+ LOG.info("isClosed or isDrain. Now breaking loop.");
break;
}
} catch (Throwable t) {
- logger.error("Error processing file=" + file.getAbsolutePath(), t);
+ LOG.error("Error processing file=" + file.getAbsolutePath(), t);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
new file mode 100644
index 0000000..8e70850
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputManager.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.input;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logfeeder.util.FileUtil;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.solr.common.util.Base64;
+
+public class InputManager {
+ private static final Logger LOG = Logger.getLogger(InputManager.class);
+
+ private static final String CHECKPOINT_SUBFOLDER_NAME = "logfeeder_checkpoints";
+ public static final String DEFAULT_CHECKPOINT_EXTENSION = ".cp";
+
+ private List<Input> inputList = new ArrayList<Input>();
+ private Set<Input> notReadyList = new HashSet<Input>();
+
+ private boolean isDrain = false;
+ private boolean isAnyInputTail = false;
+
+ private File checkPointFolderFile = null;
+
+ private MetricData filesCountMetric = new MetricData("input.files.count", true);
+
+ private String checkPointExtension;
+
+ private Thread inputIsReadyMonitor = null;
+
+ public List<Input> getInputList() {
+ return inputList;
+ }
+
+ public void add(Input input) {
+ inputList.add(input);
+ }
+
+ public void removeInput(Input input) {
+ LOG.info("Trying to remove from inputList. " + input.getShortDescription());
+ Iterator<Input> iter = inputList.iterator();
+ while (iter.hasNext()) {
+ Input iterInput = iter.next();
+ if (iterInput.equals(input)) {
+ LOG.info("Removing Input from inputList. " + input.getShortDescription());
+ iter.remove();
+ }
+ }
+ }
+
+ private int getActiveFilesCount() {
+ int count = 0;
+ for (Input input : inputList) {
+ if (input.isReady()) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ public void init() {
+ checkPointExtension = LogFeederUtil.getStringProperty("logfeeder.checkpoint.extension", DEFAULT_CHECKPOINT_EXTENSION);
+ for (Input input : inputList) {
+ try {
+ input.init();
+ if (input.isTail()) {
+ isAnyInputTail = true;
+ }
+ } catch (Exception e) {
+ LOG.error("Error initializing input. " + input.getShortDescription(), e);
+ }
+ }
+
+ if (isAnyInputTail) {
+ LOG.info("Determining valid checkpoint folder");
+ boolean isCheckPointFolderValid = false;
+ // We need to keep track of the files we are reading.
+ String checkPointFolder = LogFeederUtil.getStringProperty("logfeeder.checkpoint.folder");
+ if (!StringUtils.isEmpty(checkPointFolder)) {
+ checkPointFolderFile = new File(checkPointFolder);
+ isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
+ }
+ if (!isCheckPointFolderValid) {
+ // Let's try home folder
+ String userHome = LogFeederUtil.getStringProperty("user.home");
+ if (userHome != null) {
+ checkPointFolderFile = new File(userHome, CHECKPOINT_SUBFOLDER_NAME);
+ LOG.info("Checking if home folder can be used for checkpoints. Folder=" + checkPointFolderFile);
+ isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
+ }
+ }
+ if (!isCheckPointFolderValid) {
+ // Let's use tmp folder
+ String tmpFolder = LogFeederUtil.getStringProperty("java.io.tmpdir");
+ if (tmpFolder == null) {
+ tmpFolder = "/tmp";
+ }
+ checkPointFolderFile = new File(tmpFolder, CHECKPOINT_SUBFOLDER_NAME);
+ LOG.info("Checking if tmps folder can be used for checkpoints. Folder=" + checkPointFolderFile);
+ isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
+ if (isCheckPointFolderValid) {
+ LOG.warn("Using tmp folder " + checkPointFolderFile + " to store check points. This is not recommended." +
+ "Please set logfeeder.checkpoint.folder property");
+ }
+ }
+
+ if (isCheckPointFolderValid) {
+ LOG.info("Using folder " + checkPointFolderFile + " for storing checkpoints");
+ }
+ }
+
+ }
+
+ private boolean verifyCheckPointFolder(File folderPathFile) {
+ if (!folderPathFile.exists()) {
+ try {
+ if (!folderPathFile.mkdir()) {
+ LOG.warn("Error creating folder for check point. folder=" + folderPathFile);
+ }
+ } catch (Throwable t) {
+ LOG.warn("Error creating folder for check point. folder=" + folderPathFile, t);
+ }
+ }
+
+ if (folderPathFile.exists() && folderPathFile.isDirectory()) {
+ // Let's check whether we can create a file
+ File testFile = new File(folderPathFile, UUID.randomUUID().toString());
+ try {
+ testFile.createNewFile();
+ return testFile.delete();
+ } catch (IOException e) {
+ LOG.warn("Couldn't create test file in " + folderPathFile.getAbsolutePath() + " for checkPoint", e);
+ }
+ }
+ return false;
+ }
+
+ public File getCheckPointFolderFile() {
+ return checkPointFolderFile;
+ }
+
+ public void monitor() {
+ for (Input input : inputList) {
+ if (input.isReady()) {
+ input.monitor();
+ } else {
+ if (input.isTail()) {
+ LOG.info("Adding input to not ready list. Note, it is possible this component is not run on this host. " +
+ "So it might not be an issue. " + input.getShortDescription());
+ notReadyList.add(input);
+ } else {
+ LOG.info("Input is not ready, so going to ignore it " + input.getShortDescription());
+ }
+ }
+ }
+ // Start the monitoring thread if any file is in tail mode
+ if (isAnyInputTail) {
+ inputIsReadyMonitor = new Thread("InputIsReadyMonitor") {
+ @Override
+ public void run() {
+ LOG.info("Going to monitor for these missing files: " + notReadyList.toString());
+ while (true) {
+ if (isDrain) {
+ LOG.info("Exiting missing file monitor.");
+ break;
+ }
+ try {
+ Iterator<Input> iter = notReadyList.iterator();
+ while (iter.hasNext()) {
+ Input input = iter.next();
+ try {
+ if (input.isReady()) {
+ input.monitor();
+ iter.remove();
+ }
+ } catch (Throwable t) {
+ LOG.error("Error while enabling monitoring for input. " + input.getShortDescription());
+ }
+ }
+ Thread.sleep(30 * 1000);
+ } catch (Throwable t) {
+ // Ignore
+ }
+ }
+ }
+ };
+ inputIsReadyMonitor.start();
+ }
+ }
+
+ void addToNotReady(Input notReadyInput) {
+ notReadyList.add(notReadyInput);
+ }
+
+ public void addMetricsContainers(List<MetricData> metricsList) {
+ for (Input input : inputList) {
+ input.addMetricsContainers(metricsList);
+ }
+ filesCountMetric.value = getActiveFilesCount();
+ metricsList.add(filesCountMetric);
+ }
+
+ public void logStats() {
+ for (Input input : inputList) {
+ input.logStat();
+ }
+
+ filesCountMetric.value = getActiveFilesCount();
+ LogFeederUtil.logStatForMetric(filesCountMetric, "Stat: Files Monitored Count", "");
+ }
+
+
+ public void cleanCheckPointFiles() {
+
+ if (checkPointFolderFile == null) {
+ LOG.info("Will not clean checkPoint files. checkPointFolderFile=" + checkPointFolderFile);
+ return;
+ }
+ LOG.info("Cleaning checkPoint files. checkPointFolderFile=" + checkPointFolderFile.getAbsolutePath());
+ try {
+ // Loop over the check point files and if filePath is not present, then move to closed
+ String searchPath = "*" + checkPointExtension;
+ FileFilter fileFilter = new WildcardFileFilter(searchPath);
+ File[] checkPointFiles = checkPointFolderFile.listFiles(fileFilter);
+ int totalCheckFilesDeleted = 0;
+ for (File checkPointFile : checkPointFiles) {
+ try (RandomAccessFile checkPointReader = new RandomAccessFile(checkPointFile, "r")) {
+ int contentSize = checkPointReader.readInt();
+ byte b[] = new byte[contentSize];
+ int readSize = checkPointReader.read(b, 0, contentSize);
+ if (readSize != contentSize) {
+ LOG.error("Couldn't read expected number of bytes from checkpoint file. expected=" + contentSize + ", read="
+ + readSize + ", checkPointFile=" + checkPointFile);
+ } else {
+ String jsonCheckPointStr = new String(b, 0, readSize);
+ Map<String, Object> jsonCheckPoint = LogFeederUtil.toJSONObject(jsonCheckPointStr);
+
+ String logFilePath = (String) jsonCheckPoint.get("file_path");
+ String logFileKey = (String) jsonCheckPoint.get("file_key");
+ if (logFilePath != null && logFileKey != null) {
+ boolean deleteCheckPointFile = false;
+ File logFile = new File(logFilePath);
+ if (logFile.exists()) {
+ Object fileKeyObj = FileUtil.getFileKey(logFile);
+ String fileBase64 = Base64.byteArrayToBase64(fileKeyObj.toString().getBytes());
+ if (!logFileKey.equals(fileBase64)) {
+ deleteCheckPointFile = true;
+ LOG.info("CheckPoint clean: File key has changed. old=" + logFileKey + ", new=" + fileBase64 + ", filePath=" +
+ logFilePath + ", checkPointFile=" + checkPointFile.getAbsolutePath());
+ }
+ } else {
+ LOG.info("CheckPoint clean: Log file doesn't exist. filePath=" + logFilePath + ", checkPointFile=" +
+ checkPointFile.getAbsolutePath());
+ deleteCheckPointFile = true;
+ }
+ if (deleteCheckPointFile) {
+ LOG.info("Deleting CheckPoint file=" + checkPointFile.getAbsolutePath() + ", logFile=" + logFilePath);
+ checkPointFile.delete();
+ totalCheckFilesDeleted++;
+ }
+ }
+ }
+ } catch (EOFException eof) {
+ LOG.warn("Caught EOFException. Ignoring reading existing checkPoint file. " + checkPointFile);
+ } catch (Throwable t) {
+ LOG.error("Error while checking checkPoint file. " + checkPointFile, t);
+ }
+ }
+ LOG.info("Deleted " + totalCheckFilesDeleted + " checkPoint file(s). checkPointFolderFile=" +
+ checkPointFolderFile.getAbsolutePath());
+
+ } catch (Throwable t) {
+ LOG.error("Error while cleaning checkPointFiles", t);
+ }
+ }
+
+ public void waitOnAllInputs() {
+ //wait on inputs
+ for (Input input : inputList) {
+ if (input != null) {
+ Thread inputThread = input.getThread();
+ if (inputThread != null) {
+ try {
+ inputThread.join();
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ }
+ }
+ // wait on monitor
+ if (inputIsReadyMonitor != null) {
+ try {
+ this.close();
+ inputIsReadyMonitor.join();
+ } catch (InterruptedException e) {
+ // ignore
+ }
+ }
+ }
+
+ public void checkInAll() {
+ for (Input input : inputList) {
+ input.lastCheckIn();
+ }
+ }
+
+ public void close() {
+ for (Input input : inputList) {
+ try {
+ input.setDrain(true);
+ } catch (Throwable t) {
+ LOG.error("Error while draining. input=" + input.getShortDescription(), t);
+ }
+ }
+ isDrain = true;
+
+ // Need to get this value from property
+ int iterations = 30;
+ int waitTimeMS = 1000;
+ for (int i = 0; i < iterations; i++) {
+ boolean allClosed = true;
+ for (Input input : inputList) {
+ if (!input.isClosed()) {
+ try {
+ allClosed = false;
+ LOG.warn("Waiting for input to close. " + input.getShortDescription() + ", " + (iterations - i) + " more seconds");
+ Thread.sleep(waitTimeMS);
+ } catch (Throwable t) {
+ // Ignore
+ }
+ }
+ }
+ if (allClosed) {
+ LOG.info("All inputs are closed. Iterations=" + i);
+ return;
+ }
+ }
+
+ LOG.warn("Some inputs were not closed after " + iterations + " iterations");
+ for (Input input : inputList) {
+ if (!input.isClosed()) {
+ LOG.warn("Input not closed. Will ignore it." + input.getShortDescription());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMarker.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMarker.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMarker.java
index 48a7f1d..6767687 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMarker.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMarker.java
@@ -23,13 +23,18 @@ package org.apache.ambari.logfeeder.input;
* This file contains the file inode, line number of the log currently been read
*/
public class InputMarker {
- public int lineNumber = 0;
- public Input input;
- public String base64FileKey = null;
-
+ public final Input input;
+ public final String base64FileKey;
+ public final int lineNumber;
+
+ public InputMarker(Input input, String base64FileKey, int lineNumber) {
+ this.input = input;
+ this.base64FileKey = base64FileKey;
+ this.lineNumber = lineNumber;
+ }
+
@Override
public String toString() {
- return "InputMarker [lineNumber=" + lineNumber + ", input="
- + input.getShortDescription() + "]";
+ return "InputMarker [lineNumber=" + lineNumber + ", input=" + input.getShortDescription() + "]";
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMgr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMgr.java
deleted file mode 100644
index b18c9b0..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputMgr.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.input;
-
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.apache.ambari.logfeeder.metrics.MetricCount;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.commons.io.filefilter.WildcardFileFilter;
-import org.apache.log4j.Logger;
-import org.apache.solr.common.util.Base64;
-
-public class InputMgr {
- private static final Logger logger = Logger.getLogger(InputMgr.class);
-
- private List<Input> inputList = new ArrayList<Input>();
- private Set<Input> notReadyList = new HashSet<Input>();
-
- private boolean isDrain = false;
- private boolean isAnyInputTail = false;
-
- private String checkPointSubFolderName = "logfeeder_checkpoints";
- private File checkPointFolderFile = null;
-
- private MetricCount filesCountMetric = new MetricCount();
-
- private String checkPointExtension = ".cp";
-
- private Thread inputIsReadyMonitor = null;
-
- public List<Input> getInputList() {
- return inputList;
- }
-
- public void add(Input input) {
- inputList.add(input);
- }
-
- public void removeInput(Input input) {
- logger.info("Trying to remove from inputList. "
- + input.getShortDescription());
- Iterator<Input> iter = inputList.iterator();
- while (iter.hasNext()) {
- Input iterInput = iter.next();
- if (iterInput.equals(input)) {
- logger.info("Removing Input from inputList. "
- + input.getShortDescription());
- iter.remove();
- }
- }
- }
-
- public int getActiveFilesCount() {
- int count = 0;
- for (Input input : inputList) {
- if (input.isReady()) {
- count++;
- }
- }
- return count;
- }
-
- public void init() {
- filesCountMetric.metricsName = "input.files.count";
- filesCountMetric.isPointInTime = true;
-
- checkPointExtension = LogFeederUtil.getStringProperty(
- "logfeeder.checkpoint.extension", checkPointExtension);
- for (Input input : inputList) {
- try {
- input.init();
- if (input.isTail()) {
- isAnyInputTail = true;
- }
- } catch (Exception e) {
- logger.error(
- "Error initializing input. "
- + input.getShortDescription(), e);
- }
- }
-
- if (isAnyInputTail) {
- logger.info("Determining valid checkpoint folder");
- boolean isCheckPointFolderValid = false;
- // We need to keep track of the files we are reading.
- String checkPointFolder = LogFeederUtil
- .getStringProperty("logfeeder.checkpoint.folder");
- if (checkPointFolder != null && !checkPointFolder.isEmpty()) {
- checkPointFolderFile = new File(checkPointFolder);
- isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
- }
- if (!isCheckPointFolderValid) {
- // Let's try home folder
- String userHome = LogFeederUtil.getStringProperty("user.home");
- if (userHome != null) {
- checkPointFolderFile = new File(userHome,
- checkPointSubFolderName);
- logger.info("Checking if home folder can be used for checkpoints. Folder="
- + checkPointFolderFile);
- isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
- }
- }
- if (!isCheckPointFolderValid) {
- // Let's use tmp folder
- String tmpFolder = LogFeederUtil
- .getStringProperty("java.io.tmpdir");
- if (tmpFolder == null) {
- tmpFolder = "/tmp";
- }
- checkPointFolderFile = new File(tmpFolder,
- checkPointSubFolderName);
- logger.info("Checking if tmps folder can be used for checkpoints. Folder="
- + checkPointFolderFile);
- isCheckPointFolderValid = verifyCheckPointFolder(checkPointFolderFile);
- if (isCheckPointFolderValid) {
- logger.warn("Using tmp folder "
- + checkPointFolderFile
- + " to store check points. This is not recommended."
- + "Please set logfeeder.checkpoint.folder property");
- }
- }
-
- if (isCheckPointFolderValid) {
- logger.info("Using folder " + checkPointFolderFile
- + " for storing checkpoints");
- }
- }
-
- }
-
- public File getCheckPointFolderFile() {
- return checkPointFolderFile;
- }
-
- private boolean verifyCheckPointFolder(File folderPathFile) {
- if (!folderPathFile.exists()) {
- // Create the folder
- try {
- if (!folderPathFile.mkdir()) {
- logger.warn("Error creating folder for check point. folder="
- + folderPathFile);
- }
- } catch (Throwable t) {
- logger.warn("Error creating folder for check point. folder="
- + folderPathFile, t);
- }
- }
-
- if (folderPathFile.exists() && folderPathFile.isDirectory()) {
- // Let's check whether we can create a file
- File testFile = new File(folderPathFile, UUID.randomUUID()
- .toString());
- try {
- testFile.createNewFile();
- return testFile.delete();
- } catch (IOException e) {
- logger.warn(
- "Couldn't create test file in "
- + folderPathFile.getAbsolutePath()
- + " for checkPoint", e);
- }
- }
- return false;
- }
-
- public void monitor() {
- for (Input input : inputList) {
- if (input.isReady()) {
- input.monitor();
- } else {
- if (input.isTail()) {
- logger.info("Adding input to not ready list. Note, it is possible this component is not run on this host. So it might not be an issue. "
- + input.getShortDescription());
- notReadyList.add(input);
- } else {
- logger.info("Input is not ready, so going to ignore it "
- + input.getShortDescription());
- }
- }
- }
- // Start the monitoring thread if any file is in tail mode
- if (isAnyInputTail) {
- inputIsReadyMonitor = new Thread("InputIsReadyMonitor") {
- @Override
- public void run() {
- logger.info("Going to monitor for these missing files: "
- + notReadyList.toString());
- while (true) {
- if (isDrain) {
- logger.info("Exiting missing file monitor.");
- break;
- }
- try {
- Iterator<Input> iter = notReadyList.iterator();
- while (iter.hasNext()) {
- Input input = iter.next();
- try {
- if (input.isReady()) {
- input.monitor();
- iter.remove();
- }
- } catch (Throwable t) {
- logger.error("Error while enabling monitoring for input. "
- + input.getShortDescription());
- }
- }
- Thread.sleep(30 * 1000);
- } catch (Throwable t) {
- // Ignore
- }
- }
- }
- };
- inputIsReadyMonitor.start();
- }
- }
-
- public void addToNotReady(Input notReadyInput) {
- notReadyList.add(notReadyInput);
- }
-
- public void addMetricsContainers(List<MetricCount> metricsList) {
- for (Input input : inputList) {
- input.addMetricsContainers(metricsList);
- }
- filesCountMetric.count = getActiveFilesCount();
- metricsList.add(filesCountMetric);
- }
-
- public void logStats() {
- for (Input input : inputList) {
- input.logStat();
- }
-
- filesCountMetric.count = getActiveFilesCount();
- LogFeederUtil.logStatForMetric(filesCountMetric,
- "Stat: Files Monitored Count", null);
- }
-
- public void close() {
- for (Input input : inputList) {
- try {
- input.setDrain(true);
- } catch (Throwable t) {
- logger.error(
- "Error while draining. input="
- + input.getShortDescription(), t);
- }
- }
- isDrain = true;
-
- // Need to get this value from property
- int iterations = 30;
- int waitTimeMS = 1000;
- int i = 0;
- boolean allClosed = true;
- for (i = 0; i < iterations; i++) {
- allClosed = true;
- for (Input input : inputList) {
- if (!input.isClosed()) {
- try {
- allClosed = false;
- logger.warn("Waiting for input to close. "
- + input.getShortDescription() + ", "
- + (iterations - i) + " more seconds");
- Thread.sleep(waitTimeMS);
- } catch (Throwable t) {
- // Ignore
- }
- }
- }
- if (allClosed) {
- break;
- }
- }
- if (!allClosed) {
- logger.warn("Some inputs were not closed. Iterations=" + i);
- for (Input input : inputList) {
- if (!input.isClosed()) {
- logger.warn("Input not closed. Will ignore it."
- + input.getShortDescription());
- }
- }
- } else {
- logger.info("All inputs are closed. Iterations=" + i);
- }
-
- }
-
- public void checkInAll() {
- for (Input input : inputList) {
- input.checkIn();
- }
- }
-
- public void cleanCheckPointFiles() {
-
- if (checkPointFolderFile == null) {
- logger.info("Will not clean checkPoint files. checkPointFolderFile="
- + checkPointFolderFile);
- return;
- }
- logger.info("Cleaning checkPoint files. checkPointFolderFile="
- + checkPointFolderFile.getAbsolutePath());
- try {
- // Loop over the check point files and if filePath is not present, then move to closed
- String searchPath = "*" + checkPointExtension;
- FileFilter fileFilter = new WildcardFileFilter(searchPath);
- File[] checkPointFiles = checkPointFolderFile.listFiles(fileFilter);
- int totalCheckFilesDeleted = 0;
- for (File checkPointFile : checkPointFiles) {
- RandomAccessFile checkPointReader = null;
- try {
- checkPointReader = new RandomAccessFile(checkPointFile, "r");
-
- int contentSize = checkPointReader.readInt();
- byte b[] = new byte[contentSize];
- int readSize = checkPointReader.read(b, 0, contentSize);
- if (readSize != contentSize) {
- logger.error("Couldn't read expected number of bytes from checkpoint file. expected="
- + contentSize
- + ", read="
- + readSize
- + ", checkPointFile=" + checkPointFile);
- } else {
- // Create JSON string
- String jsonCheckPointStr = new String(b, 0, readSize);
- Map<String, Object> jsonCheckPoint = LogFeederUtil
- .toJSONObject(jsonCheckPointStr);
-
- String logFilePath = (String) jsonCheckPoint
- .get("file_path");
- String logFileKey = (String) jsonCheckPoint
- .get("file_key");
- if (logFilePath != null && logFileKey != null) {
- boolean deleteCheckPointFile = false;
- File logFile = new File(logFilePath);
- if (logFile.exists()) {
- Object fileKeyObj = InputFile
- .getFileKey(logFile);
- String fileBase64 = Base64
- .byteArrayToBase64(fileKeyObj
- .toString().getBytes());
- if (!logFileKey.equals(fileBase64)) {
- deleteCheckPointFile = true;
- logger.info("CheckPoint clean: File key has changed. old="
- + logFileKey
- + ", new="
- + fileBase64
- + ", filePath="
- + logFilePath
- + ", checkPointFile="
- + checkPointFile.getAbsolutePath());
- }
- } else {
- logger.info("CheckPoint clean: Log file doesn't exist. filePath="
- + logFilePath
- + ", checkPointFile="
- + checkPointFile.getAbsolutePath());
- deleteCheckPointFile = true;
- }
- if (deleteCheckPointFile) {
- logger.info("Deleting CheckPoint file="
- + checkPointFile.getAbsolutePath()
- + ", logFile=" + logFilePath);
- checkPointFile.delete();
- totalCheckFilesDeleted++;
- }
- }
- }
- } catch (EOFException eof) {
- logger.warn("Caught EOFException. Ignoring reading existing checkPoint file. "
- + checkPointFile);
- } catch (Throwable t) {
- logger.error("Error while checking checkPoint file. "
- + checkPointFile, t);
- } finally {
- if (checkPointReader != null) {
- try {
- checkPointReader.close();
- } catch (Throwable t) {
- logger.error("Error closing checkPoint file. "
- + checkPointFile, t);
- }
- }
- }
- }
- logger.info("Deleted " + totalCheckFilesDeleted
- + " checkPoint file(s). checkPointFolderFile="
- + checkPointFolderFile.getAbsolutePath());
-
- } catch (Throwable t) {
- logger.error("Error while cleaning checkPointFiles", t);
- }
- }
-
- public void waitOnAllInputs() {
- //wait on inputs
- if (inputList != null) {
- for (Input input : inputList) {
- if (input != null) {
- Thread inputThread = input.getThread();
- if (inputThread != null) {
- try {
- inputThread.join();
- } catch (InterruptedException e) {
- // ignore
- }
- }
- }
- }
- }
- // wait on monitor
- if (inputIsReadyMonitor != null) {
- try {
- this.close();
- inputIsReadyMonitor.join();
- } catch (InterruptedException e) {
- // ignore
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputS3File.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputS3File.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputS3File.java
index c9d28bd..f560379 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputS3File.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputS3File.java
@@ -19,201 +19,57 @@
package org.apache.ambari.logfeeder.input;
import java.io.BufferedReader;
-import java.io.EOFException;
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.S3Util;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.solr.common.util.Base64;
-public class InputS3File extends Input {
- private static final Logger logger = Logger.getLogger(InputS3File.class);
-
- private String logPath = null;
- private boolean isStartFromBegining = true;
-
- private boolean isReady = false;
- private String[] s3LogPathFiles = null;
- private Object fileKey = null;
- private String base64FileKey = null;
-
- private boolean isRolledOver = false;
- private boolean addWildCard = false;
-
- private long lastCheckPointTimeMS = 0;
- private int checkPointIntervalMS = 5 * 1000; // 5 seconds
- private RandomAccessFile checkPointWriter = null;
- private Map<String, Object> jsonCheckPoint = null;
-
- private File checkPointFile = null;
-
- private InputMarker lastCheckPointInputMarker = null;
-
- private String checkPointExtension = ".cp";
-
-
- @Override
- public void init() throws Exception {
- logger.info("init() called");
- statMetric.metricsName = "input.files.read_lines";
- readBytesMetric.metricsName = "input.files.read_bytes";
- checkPointExtension = LogFeederUtil.getStringProperty(
- "logfeeder.checkpoint.extension", checkPointExtension);
-
- // Let's close the file and set it to true after we start monitoring it
- setClosed(true);
- logPath = getStringValue("path");
- tail = getBooleanValue("tail", tail);
- addWildCard = getBooleanValue("add_wild_card", addWildCard);
- checkPointIntervalMS = getIntValue("checkpoint.interval.ms",
- checkPointIntervalMS);
- if (logPath == null || logPath.isEmpty()) {
- logger.error("path is empty for file input. " + getShortDescription());
- return;
- }
-
- String startPosition = getStringValue("start_position");
- if (StringUtils.isEmpty(startPosition)
- || startPosition.equalsIgnoreCase("beginning")
- || startPosition.equalsIgnoreCase("begining")) {
- isStartFromBegining = true;
- }
-
- if (!tail) {
- // start position end doesn't apply if we are not tailing
- isStartFromBegining = true;
- }
-
- setFilePath(logPath);
- boolean isFileReady = isReady();
-
- logger.info("File to monitor " + logPath + ", tail=" + tail
- + ", addWildCard=" + addWildCard + ", isReady=" + isFileReady);
-
- super.init();
- }
+public class InputS3File extends AbstractInputFile {
@Override
public boolean isReady() {
if (!isReady) {
// Let's try to check whether the file is available
- s3LogPathFiles = getActualFiles(logPath);
- if (s3LogPathFiles != null && s3LogPathFiles.length > 0) {
- if (isTail() && s3LogPathFiles.length > 1) {
- logger.warn("Found multiple files (" + s3LogPathFiles.length
- + ") for the file filter " + filePath
- + ". Will use only the first one. Using " + s3LogPathFiles[0]);
+ logFiles = getActualFiles(logPath);
+ if (!ArrayUtils.isEmpty(logFiles)) {
+ if (tail && logFiles.length > 1) {
+ LOG.warn("Found multiple files (" + logFiles.length + ") for the file filter " + filePath +
+ ". Will use only the first one. Using " + logFiles[0].getAbsolutePath());
}
- logger.info("File filter " + filePath + " expanded to "
- + s3LogPathFiles[0]);
+ LOG.info("File filter " + filePath + " expanded to " + logFiles[0].getAbsolutePath());
isReady = true;
} else {
- logger.debug(logPath + " file doesn't exist. Ignoring for now");
+ LOG.debug(logPath + " file doesn't exist. Ignoring for now");
}
}
return isReady;
}
- private String[] getActualFiles(String searchPath) {
+ private File[] getActualFiles(String searchPath) {
// TODO search file on s3
- return new String[] { searchPath };
- }
-
- @Override
- synchronized public void checkIn(InputMarker inputMarker) {
- super.checkIn(inputMarker);
- if (checkPointWriter != null) {
- try {
- int lineNumber = LogFeederUtil.objectToInt(
- jsonCheckPoint.get("line_number"), 0, "line_number");
- if (lineNumber > inputMarker.lineNumber) {
- // Already wrote higher line number for this input
- return;
- }
- // If interval is greater than last checkPoint time, then write
- long currMS = System.currentTimeMillis();
- if (!isClosed()
- && (currMS - lastCheckPointTimeMS) < checkPointIntervalMS) {
- // Let's save this one so we can update the check point file
- // on flush
- lastCheckPointInputMarker = inputMarker;
- return;
- }
- lastCheckPointTimeMS = currMS;
-
- jsonCheckPoint.put("line_number", ""
- + new Integer(inputMarker.lineNumber));
- jsonCheckPoint.put("last_write_time_ms", "" + new Long(currMS));
- jsonCheckPoint.put("last_write_time_date", new Date());
-
- String jsonStr = LogFeederUtil.getGson().toJson(jsonCheckPoint);
-
- // Let's rewind
- checkPointWriter.seek(0);
- checkPointWriter.writeInt(jsonStr.length());
- checkPointWriter.write(jsonStr.getBytes());
-
- if (isClosed()) {
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_FINAL_CHECKIN";
- LogFeederUtil.logErrorMessageByInterval(LOG_MESSAGE_KEY,
- "Wrote final checkPoint, input=" + getShortDescription()
- + ", checkPointFile=" + checkPointFile.getAbsolutePath()
- + ", checkPoint=" + jsonStr, null, logger, Level.INFO);
- }
- } catch (Throwable t) {
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_CHECKIN_EXCEPTION";
- LogFeederUtil.logErrorMessageByInterval(LOG_MESSAGE_KEY,
- "Caught exception checkIn. , input=" + getShortDescription(), t,
- logger, Level.ERROR);
- }
- }
-
- }
-
- @Override
- public void checkIn() {
- super.checkIn();
- if (lastCheckPointInputMarker != null) {
- checkIn(lastCheckPointInputMarker);
- }
- }
-
- @Override
- public void rollOver() {
- logger.info("Marking this input file for rollover. "
- + getShortDescription());
- isRolledOver = true;
+ return new File[] { new File(searchPath) };
}
@Override
void start() throws Exception {
- if (s3LogPathFiles == null || s3LogPathFiles.length == 0) {
+ if (ArrayUtils.isEmpty(logFiles)) {
return;
}
- if (isTail()) {
- processFile(s3LogPathFiles[0]);
+ if (tail) {
+ processFile(logFiles[0]);
} else {
- for (String s3FilePath : s3LogPathFiles) {
+ for (File s3FilePath : logFiles) {
try {
processFile(s3FilePath);
if (isClosed() || isDrain()) {
- logger.info("isClosed or isDrain. Now breaking loop.");
+ LOG.info("isClosed or isDrain. Now breaking loop.");
break;
}
} catch (Throwable t) {
- logger.error("Error processing file=" + s3FilePath, t);
+ LOG.error("Error processing file=" + s3FilePath, t);
}
}
}
@@ -221,244 +77,18 @@ public class InputS3File extends Input {
}
@Override
- public void close() {
- super.close();
- logger.info("close() calling checkPoint checkIn(). "
- + getShortDescription());
- checkIn();
- }
-
- private void processFile(String logPathFile) throws FileNotFoundException,
- IOException {
- logger.info("Monitoring logPath=" + logPath + ", logPathFile="
- + logPathFile);
- BufferedReader br = null;
- checkPointFile = null;
- checkPointWriter = null;
- jsonCheckPoint = null;
- int resumeFromLineNumber = 0;
-
- int lineCount = 0;
- try {
- setFilePath(logPathFile);
- String s3AccessKey = getStringValue("s3_access_key");
- String s3SecretKey = getStringValue("s3_secret_key");
- br = S3Util.INSTANCE.getReader(logPathFile,s3AccessKey,s3SecretKey);
- if(br==null){
- //log err
- return;
- }
-
- // Whether to send to output from the beginning.
- boolean resume = isStartFromBegining;
-
- // Seems FileWatch is not reliable, so let's only use file key comparison
- fileKey = getFileKey(logPathFile);
- base64FileKey = Base64.byteArrayToBase64(fileKey.toString().getBytes());
- logger.info("fileKey=" + fileKey + ", base64=" + base64FileKey + ". "
- + getShortDescription());
-
- if (isTail()) {
- try {
- // Let's see if there is a checkpoint for this file
- logger.info("Checking existing checkpoint file. "
- + getShortDescription());
-
- String fileBase64 = Base64.byteArrayToBase64(fileKey.toString()
- .getBytes());
- String checkPointFileName = fileBase64 + checkPointExtension;
- File checkPointFolder = inputMgr.getCheckPointFolderFile();
- checkPointFile = new File(checkPointFolder, checkPointFileName);
- checkPointWriter = new RandomAccessFile(checkPointFile, "rw");
-
- try {
- int contentSize = checkPointWriter.readInt();
- byte b[] = new byte[contentSize];
- int readSize = checkPointWriter.read(b, 0, contentSize);
- if (readSize != contentSize) {
- logger
- .error("Couldn't read expected number of bytes from checkpoint file. expected="
- + contentSize
- + ", read="
- + readSize
- + ", checkPointFile="
- + checkPointFile
- + ", input="
- + getShortDescription());
- } else {
- String jsonCheckPointStr = new String(b, 0, readSize);
- jsonCheckPoint = LogFeederUtil.toJSONObject(jsonCheckPointStr);
-
- resumeFromLineNumber = LogFeederUtil.objectToInt(
- jsonCheckPoint.get("line_number"), 0, "line_number");
-
- if (resumeFromLineNumber > 0) {
- // Let's read from last line read
- resume = false;
- }
- logger.info("CheckPoint. checkPointFile=" + checkPointFile
- + ", json=" + jsonCheckPointStr + ", resumeFromLineNumber="
- + resumeFromLineNumber + ", resume=" + resume);
- }
- } catch (EOFException eofEx) {
- logger.info("EOFException. Will reset checkpoint file "
- + checkPointFile.getAbsolutePath() + " for "
- + getShortDescription());
- }
- if (jsonCheckPoint == null) {
- // This seems to be first time, so creating the initial
- // checkPoint object
- jsonCheckPoint = new HashMap<String, Object>();
- jsonCheckPoint.put("file_path", filePath);
- jsonCheckPoint.put("file_key", fileBase64);
- }
-
- } catch (Throwable t) {
- logger.error(
- "Error while configuring checkpoint file. Will reset file. checkPointFile="
- + checkPointFile, t);
- }
- }
-
- setClosed(false);
- int sleepStep = 2;
- int sleepIteration = 0;
- while (true) {
- try {
- if (isDrain()) {
- break;
- }
-
- String line = br.readLine();
- if (line == null) {
- if (!resume) {
- resume = true;
- }
- sleepIteration++;
- try {
- // Since FileWatch service is not reliable, we will check
- // file inode every n seconds after no write
- if (sleepIteration > 4) {
- Object newFileKey = getFileKey(logPathFile);
- if (newFileKey != null) {
- if (fileKey == null || !newFileKey.equals(fileKey)) {
- logger
- .info("File key is different. Calling rollover. oldKey="
- + fileKey
- + ", newKey="
- + newFileKey
- + ". "
- + getShortDescription());
- // File has rotated.
- rollOver();
- }
- }
- }
- // Flush on the second iteration
- if (!tail && sleepIteration >= 2) {
- logger.info("End of file. Done with filePath=" + logPathFile
- + ", lineCount=" + lineCount);
- flush();
- break;
- } else if (sleepIteration == 2) {
- flush();
- } else if (sleepIteration >= 2) {
- if (isRolledOver) {
- isRolledOver = false;
- // Close existing file
- try {
- logger
- .info("File is rolled over. Closing current open file."
- + getShortDescription() + ", lineCount="
- + lineCount);
- br.close();
- } catch (Exception ex) {
- logger.error("Error closing file" + getShortDescription());
- break;
- }
- try {
- // Open new file
- logger.info("Opening new rolled over file."
- + getShortDescription());
- br = S3Util.INSTANCE.getReader(logPathFile,s3AccessKey,s3SecretKey);
- lineCount = 0;
- fileKey = getFileKey(logPathFile);
- base64FileKey = Base64.byteArrayToBase64(fileKey.toString()
- .getBytes());
- logger.info("fileKey=" + fileKey + ", base64="
- + base64FileKey + ", " + getShortDescription());
- } catch (Exception ex) {
- logger.error("Error opening rolled over file. "
- + getShortDescription());
- // Let's add this to monitoring and exit this thread
- logger.info("Added input to not ready list."
- + getShortDescription());
- isReady = false;
- inputMgr.addToNotReady(this);
- break;
- }
- logger.info("File is successfully rolled over. "
- + getShortDescription());
- continue;
- }
- }
- Thread.sleep(sleepStep * 1000);
- sleepStep = (sleepStep * 2);
- sleepStep = sleepStep > 10 ? 10 : sleepStep;
- } catch (InterruptedException e) {
- logger.info("Thread interrupted." + getShortDescription());
- }
- } else {
- lineCount++;
- sleepStep = 1;
- sleepIteration = 0;
-
- if (!resume && lineCount > resumeFromLineNumber) {
- logger.info("Resuming to read from last line. lineCount="
- + lineCount + ", input=" + getShortDescription());
- resume = true;
- }
- if (resume) {
- InputMarker marker = new InputMarker();
- marker.base64FileKey = base64FileKey;
- marker.input = this;
- marker.lineNumber = lineCount;
- outputLine(line, marker);
- }
- }
- } catch (Throwable t) {
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_READ_LOOP_EXCEPTION";
- LogFeederUtil.logErrorMessageByInterval(LOG_MESSAGE_KEY,
- "Caught exception in read loop. lineNumber=" + lineCount
- + ", input=" + getShortDescription(), t, logger, Level.ERROR);
-
- }
- }
- } finally {
- if (br != null) {
- logger.info("Closing reader." + getShortDescription() + ", lineCount="
- + lineCount);
- try {
- br.close();
- } catch (Throwable t) {
- // ignore
- }
- }
- }
- }
-
- static public Object getFileKey(String s3FilePath) {
- return s3FilePath.toString();
+ protected BufferedReader openLogFile(File logPathFile) throws IOException {
+ String s3AccessKey = getStringValue("s3_access_key");
+ String s3SecretKey = getStringValue("s3_secret_key");
+ BufferedReader br = S3Util.getReader(logPathFile.getPath(), s3AccessKey, s3SecretKey);
+ fileKey = getFileKey(logPathFile);
+ base64FileKey = Base64.byteArrayToBase64(fileKey.toString().getBytes());
+ LOG.info("fileKey=" + fileKey + ", base64=" + base64FileKey + ". " + getShortDescription());
+ return br;
}
@Override
- public String getShortDescription() {
- return "input:source="
- + getStringValue("source")
- + ", path="
- + (s3LogPathFiles != null && s3LogPathFiles.length > 0 ? s3LogPathFiles[0]
- : getStringValue("path"));
+ protected Object getFileKey(File logFile) {
+ return logFile.getPath();
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
index 5ba56a5..743be69 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/InputSimulate.java
@@ -18,7 +18,7 @@
*/
package org.apache.ambari.logfeeder.input;
-import java.net.Inet4Address;
+import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
@@ -66,7 +66,7 @@ public class InputSimulate extends Input {
Filter filter = new FilterJSON();
filter.setInput(this);
- setFirstFilter(filter);
+ addFilter(filter);
}
private List<String> getSimulatedLogTypes() {
@@ -88,23 +88,18 @@ public class InputSimulate extends Input {
return LOG_TEXT_PATTERN.replaceAll("<LOG_MESSAGE_PATTERN>", logMessagePattern);
}
-
- @Override
- public String getNameForThread() {
- return "Simulated input";
- }
@Override
- public String getShortDescription() {
- return "Simulated input";
+ public boolean isReady() {
+ return true;
}
-
+
@Override
void start() throws Exception {
if (types.isEmpty())
return;
- getFirstFilter().setOutputMgr(outputMgr);
+ getFirstFilter().setOutputManager(outputManager);
while (true) {
String type = imitateRandomLogFile();
@@ -129,10 +124,7 @@ public class InputSimulate extends Input {
}
private InputMarker getInputMarker(String type) throws Exception {
- InputMarker marker = new InputMarker();
- marker.input = this;
- marker.lineNumber = getLineNumber(type);
- marker.base64FileKey = getBase64FileKey();
+ InputMarker marker = new InputMarker(this, getBase64FileKey(), getLineNumber(type));
return marker;
}
@@ -147,7 +139,7 @@ public class InputSimulate extends Input {
}
private String getBase64FileKey() throws Exception {
- String fileKey = Inet4Address.getLocalHost().getHostAddress() + "|" + filePath;
+ String fileKey = InetAddress.getLocalHost().getHostAddress() + "|" + filePath;
return Base64.byteArrayToBase64(fileKey.getBytes());
}
@@ -155,4 +147,20 @@ public class InputSimulate extends Input {
Date d = new Date();
return String.format(logText, d.getTime(), level, marker.lineNumber);
}
+
+ @Override
+ public void checkIn(InputMarker inputMarker) {}
+
+ @Override
+ public void lastCheckIn() {}
+
+ @Override
+ public String getNameForThread() {
+ return "Simulated input";
+ }
+
+ @Override
+ public String getShortDescription() {
+ return "Simulated input";
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/GZIPReader.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/GZIPReader.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/GZIPReader.java
index a2a9db2..9ccc4f2 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/GZIPReader.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/reader/GZIPReader.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.logfeeder.input.reader;
-import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -30,15 +29,11 @@ import org.apache.log4j.Logger;
class GZIPReader extends InputStreamReader {
- private static Logger logger = Logger.getLogger(GZIPReader.class);
+ private static final Logger LOG = Logger.getLogger(GZIPReader.class);
GZIPReader(String fileName) throws FileNotFoundException {
super(getStream(fileName));
- logger.info("Created GZIPReader for file : " + fileName);
- }
-
- GZIPReader(File file) throws FileNotFoundException {
- super(getStream(file.getName()));
+ LOG.info("Created GZIPReader for file : " + fileName);
}
private static InputStream getStream(String fileName) {
@@ -48,7 +43,7 @@ class GZIPReader extends InputStreamReader {
fileStream = new FileInputStream(fileName);
gzipStream = new GZIPInputStream(fileStream);
} catch (Exception e) {
- logger.error(e, e.getCause());
+ LOG.error(e, e.getCause());
}
return gzipStream;
}
@@ -58,21 +53,13 @@ class GZIPReader extends InputStreamReader {
*/
static boolean isValidFile(String fileName) {
// TODO make it generic and put in factory itself
- InputStream is = null;
- try {
- is = new FileInputStream(fileName);
+
+ try (InputStream is = new FileInputStream(fileName)) {
byte[] signature = new byte[2];
int nread = is.read(signature); // read the gzip signature
return nread == 2 && signature[0] == (byte) 0x1f && signature[1] == (byte) 0x8b;
} catch (IOException e) {
return false;
- } finally {
- if (is != null) {
- try {
- is.close();
- } catch (IOException e) {
- }
- }
}
}
}
[09/10] ambari git commit: AMBARI-18310. Refactor logsearch portal
side code - Part 2 (oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonServiceLogRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonServiceLogRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonServiceLogRequestConverter.java
index a02d585..2282c11 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonServiceLogRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractCommonServiceLogRequestConverter.java
@@ -18,37 +18,38 @@
*/
package org.apache.ambari.logsearch.query.converter;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.impl.BaseServiceLogRequest;
-import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
+import org.apache.ambari.logsearch.query.model.CommonServiceLogSearchCriteria;
import org.apache.commons.lang.StringEscapeUtils;
-public abstract class AbstractCommonServiceLogRequestConverter<SOURCE extends BaseServiceLogRequest, RESULT extends CommonSearchCriteria>
+public abstract class AbstractCommonServiceLogRequestConverter<SOURCE extends BaseServiceLogRequest, RESULT extends CommonServiceLogSearchCriteria>
extends AbstractCommonSearchRequestConverter<SOURCE, RESULT> {
@Override
public RESULT convertToSearchCriteria(SOURCE request) {
RESULT criteria = createCriteria(request);
+ // TODO: check are these used from the UI or not?
criteria.addParam("advanceSearch", StringEscapeUtils.unescapeXml(request.getAdvancedSearch()));
criteria.addParam("q", request.getQuery());
- criteria.addParam("treeParams", StringEscapeUtils.unescapeHtml(request.getTreeParams()));
- criteria.addParam("level", request.getLevel());
criteria.addParam("gMustNot", request.getgMustNot());
- criteria.addParam("from", request.getFrom());
- criteria.addParam("to", request.getTo());
- criteria.addParam("selectComp", request.getMustBe());
- criteria.addParam("unselectComp", request.getMustNot());
- criteria.addParam("iMessage", StringEscapeUtils.unescapeXml(request.getiMessage()));
criteria.addParam("gEMessage", StringEscapeUtils.unescapeXml(request.getgEMessage()));
- criteria.addParam("eMessage", StringEscapeUtils.unescapeXml(request.getgEMessage()));
- criteria.addParam(LogSearchConstants.BUNDLE_ID, request.getBundleId());
- criteria.addParam("host_name", request.getHostName());
- criteria.addParam("component_name", request.getComponentName());
- criteria.addParam("file_name", request.getFileName());
- criteria.addParam("startDate", request.getStartTime());
- criteria.addParam("endDate", request.getEndTime());
- criteria.addParam("excludeQuery", StringEscapeUtils.unescapeXml(request.getExcludeQuery()));
- criteria.addParam("includeQuery", StringEscapeUtils.unescapeXml(request.getIncludeQuery()));
+ criteria.addParam("unselectComp", request.getMustNot());
+
+ criteria.setTreeParams(StringEscapeUtils.unescapeHtml(request.getTreeParams()));
+ criteria.setLevel(request.getLevel());
+ criteria.setFrom(request.getFrom());
+ criteria.setTo(request.getTo());
+ criteria.setSelectComp(request.getMustBe());
+ criteria.setBundleId(request.getBundleId());
+ criteria.setHostName(request.getHostName());
+ criteria.setComponentName(request.getComponentName());
+ criteria.setFileName(request.getFileName());
+ criteria.setIncludeMessage(StringEscapeUtils.unescapeXml(request.getiMessage()));
+ criteria.setExcludeMessage(StringEscapeUtils.unescapeXml(request.getgEMessage()));
+ criteria.setStartTime(request.getStartTime());
+ criteria.setEndTime(request.getEndTime());
+ criteria.setExcludeQuery(StringEscapeUtils.unescapeXml(request.getExcludeQuery()));
+ criteria.setIncludeQuery(StringEscapeUtils.unescapeXml(request.getIncludeQuery()));
return criteria;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractConverterAware.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractConverterAware.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractConverterAware.java
new file mode 100644
index 0000000..18a71c1
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AbstractConverterAware.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.query.converter;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.core.convert.ConversionService;
+import org.springframework.core.convert.converter.Converter;
+import org.springframework.core.convert.converter.ConverterRegistry;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
+public abstract class AbstractConverterAware<SOURCE, RESULT> implements Converter<SOURCE, RESULT> {
+
+ @Inject
+ @Qualifier("conversionService")
+ private ConversionService conversionService;
+
+ public ConversionService getConversionService() {
+ return conversionService;
+ }
+
+ @PostConstruct
+ private void register() {
+ if (conversionService instanceof ConverterRegistry) {
+ ((ConverterRegistry) conversionService).addConverter(this);
+ } else {
+ throw new IllegalStateException("Can't register Converter to ConverterRegistry");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AnyGraphRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AnyGraphRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AnyGraphRequestConverter.java
index 0372168..1639563 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AnyGraphRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AnyGraphRequestConverter.java
@@ -28,12 +28,12 @@ public class AnyGraphRequestConverter extends AbstractCommonSearchRequestConvert
@Override
public AnyGraphSearchCriteria convertToSearchCriteria(AnyGraphRequest anyGraphRequest) {
AnyGraphSearchCriteria criteria = new AnyGraphSearchCriteria();
- criteria.addParam("xAxis", anyGraphRequest.getxAxis());
- criteria.addParam("yAxis", anyGraphRequest.getyAxis());
- criteria.addParam("stackBy", anyGraphRequest.getStackBy());
- criteria.addParam("unit", anyGraphRequest.getUnit());
- criteria.addParam("from", anyGraphRequest.getFrom());
- criteria.addParam("to", anyGraphRequest.getTo());
+ criteria.setxAxis(anyGraphRequest.getxAxis());
+ criteria.setyAxis(anyGraphRequest.getyAxis());
+ criteria.setStackBy(anyGraphRequest.getStackBy());
+ criteria.setUnit(anyGraphRequest.getUnit());
+ criteria.setFrom(anyGraphRequest.getFrom());
+ criteria.setTo(anyGraphRequest.getTo());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditBarGraphRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditBarGraphRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditBarGraphRequestConverter.java
index f72a673..ac74287 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditBarGraphRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditBarGraphRequestConverter.java
@@ -28,7 +28,7 @@ public class AuditBarGraphRequestConverter extends AbstractCommonAuditLogRequest
@Override
public AuditBarGraphSearchCriteria createCriteria(AuditBarGraphRequest request) {
AuditBarGraphSearchCriteria criteria = new AuditBarGraphSearchCriteria();
- criteria.addParam("unit", request.getUnit());
+ criteria.setUnit(request.getUnit());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditLogRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditLogRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditLogRequestConverter.java
index 27d314d..5ec7632 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditLogRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/AuditLogRequestConverter.java
@@ -28,7 +28,7 @@ public class AuditLogRequestConverter extends AbstractCommonAuditLogRequestConve
@Override
public AuditLogSearchCriteria createCriteria(AuditLogRequest request) {
AuditLogSearchCriteria criteria = new AuditLogSearchCriteria();
- criteria.addParam("isLastPage", request.isLastPage());
+ criteria.setLastPage(request.isLastPage());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/BaseServiceLogRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/BaseServiceLogRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/BaseServiceLogRequestConverter.java
index cfd544c..efc9bc9 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/BaseServiceLogRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/BaseServiceLogRequestConverter.java
@@ -20,13 +20,14 @@ package org.apache.ambari.logsearch.query.converter;
import org.apache.ambari.logsearch.model.request.impl.BaseServiceLogRequest;
import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
+import org.apache.ambari.logsearch.query.model.CommonServiceLogSearchCriteria;
import org.springframework.stereotype.Component;
@Component
-public class BaseServiceLogRequestConverter extends AbstractCommonServiceLogRequestConverter<BaseServiceLogRequest, CommonSearchCriteria> {
+public class BaseServiceLogRequestConverter extends AbstractCommonServiceLogRequestConverter<BaseServiceLogRequest, CommonServiceLogSearchCriteria> {
@Override
- public CommonSearchCriteria createCriteria(BaseServiceLogRequest request) {
- return new CommonSearchCriteria();
+ public CommonServiceLogSearchCriteria createCriteria(BaseServiceLogRequest request) {
+ return new CommonServiceLogSearchCriteria();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldAuditLogRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldAuditLogRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldAuditLogRequestConverter.java
index 089b593..6197d48 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldAuditLogRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldAuditLogRequestConverter.java
@@ -28,7 +28,7 @@ public class FieldAuditLogRequestConverter extends AbstractCommonAuditLogRequest
@Override
public FieldAuditLogSearchCriteria createCriteria(FieldAuditLogRequest request) {
FieldAuditLogSearchCriteria criteria = new FieldAuditLogSearchCriteria();
- criteria.addParam("field", request.getField());
+ criteria.setField(request.getField());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldBarGraphRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldBarGraphRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldBarGraphRequestConverter.java
index dd518f8..74b0dac 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldBarGraphRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/FieldBarGraphRequestConverter.java
@@ -28,8 +28,8 @@ public class FieldBarGraphRequestConverter extends AbstractCommonAuditLogRequest
@Override
public FieldAuditBarGraphSearchCriteria createCriteria(FieldAuditBarGraphRequest request) {
FieldAuditBarGraphSearchCriteria criteria = new FieldAuditBarGraphSearchCriteria();
- criteria.addParam("unit", request.getUnit());
- criteria.addParam("field", request.getField());
+ criteria.setUnit(request.getUnit());
+ criteria.setField(request.getField());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileRequestConverter.java
index 7c3038e..62c2fbe 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileRequestConverter.java
@@ -29,9 +29,9 @@ public class LogFileRequestConverter implements Converter<LogFileRequest, LogFil
@Override
public LogFileSearchCriteria convert(LogFileRequest request) {
LogFileSearchCriteria criteria = new LogFileSearchCriteria();
- criteria.addParam("component", request.getComponent());
- criteria.addParam("host", request.getHost());
- criteria.addParam("logType", request.getLogType());
+ criteria.setLogFileComponent(request.getComponent());
+ criteria.setLogFileHost(request.getHost());
+ criteria.setLogType(request.getLogType());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileTailRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileTailRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileTailRequestConverter.java
index 88b1a34..08c6ecc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileTailRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/LogFileTailRequestConverter.java
@@ -29,10 +29,10 @@ public class LogFileTailRequestConverter implements Converter<LogFileTailRequest
@Override
public LogFileTailSearchCriteria convert(LogFileTailRequest request) {
LogFileTailSearchCriteria criteria = new LogFileTailSearchCriteria();
- criteria.addParam("component", request.getComponent());
- criteria.addParam("host", request.getHost());
- criteria.addParam("logType", request.getLogType());
- criteria.addParam("tailSize", request.getTailSize());
+ criteria.setLogFileComponent(request.getComponent());
+ criteria.setLogFileHost(request.getHost());
+ criteria.setLogType(request.getLogType());
+ criteria.setLogTailSize(request.getTailSize());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceAnyGraphRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceAnyGraphRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceAnyGraphRequestConverter.java
index 7a559de..8f1aaf0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceAnyGraphRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceAnyGraphRequestConverter.java
@@ -28,12 +28,12 @@ public class ServiceAnyGraphRequestConverter extends AbstractCommonServiceLogReq
@Override
public ServiceAnyGraphSearchCriteria createCriteria(ServiceAnyGraphRequest anyGraphRequest) {
ServiceAnyGraphSearchCriteria criteria = new ServiceAnyGraphSearchCriteria();
- criteria.addParam("xAxis", anyGraphRequest.getxAxis());
- criteria.addParam("yAxis", anyGraphRequest.getyAxis());
- criteria.addParam("stackBy", anyGraphRequest.getStackBy());
- criteria.addParam("unit", anyGraphRequest.getUnit());
- criteria.addParam("from", anyGraphRequest.getFrom());
- criteria.addParam("to", anyGraphRequest.getTo());
+ criteria.setxAxis(anyGraphRequest.getxAxis());
+ criteria.setyAxis(anyGraphRequest.getyAxis());
+ criteria.setStackBy(anyGraphRequest.getStackBy());
+ criteria.setUnit(anyGraphRequest.getUnit());
+ criteria.setFrom(anyGraphRequest.getFrom());
+ criteria.setTo(anyGraphRequest.getTo());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceExtremeDatesRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceExtremeDatesRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceExtremeDatesRequestConverter.java
index fe81468..489e879 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceExtremeDatesRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceExtremeDatesRequestConverter.java
@@ -18,7 +18,6 @@
*/
package org.apache.ambari.logsearch.query.converter;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.impl.ServiceExtremeDatesRequest;
import org.apache.ambari.logsearch.query.model.ServiceExtremeDatesCriteria;
import org.springframework.stereotype.Component;
@@ -29,7 +28,7 @@ public class ServiceExtremeDatesRequestConverter extends AbstractCommonSearchReq
@Override
public ServiceExtremeDatesCriteria convertToSearchCriteria(ServiceExtremeDatesRequest request) {
ServiceExtremeDatesCriteria criteria = new ServiceExtremeDatesCriteria();
- criteria.addParam(LogSearchConstants.BUNDLE_ID, request.getBundleId());
+ criteria.setBundleId(request.getBundleId());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceGraphRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceGraphRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceGraphRequestConverter.java
index 19165c0..6c501ae 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceGraphRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceGraphRequestConverter.java
@@ -28,9 +28,9 @@ public class ServiceGraphRequestConverter extends AbstractCommonServiceLogReques
@Override
public ServiceGraphSearchCriteria createCriteria(ServiceGraphRequest request) {
ServiceGraphSearchCriteria criteria = new ServiceGraphSearchCriteria();
- criteria.addParam("hostLogFile", request.getHostLogFile());
- criteria.addParam("compLogFile", request.getComponentLogFile());
- criteria.addParam("unit", request.getUnit());
+ criteria.setLogFileHostName(request.getHostLogFile());
+ criteria.setLogFileComponentName(request.getComponentLogFile());
+ criteria.setUnit(request.getUnit());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogExportRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogExportRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogExportRequestConverter.java
index 7d83e49..bd06546 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogExportRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogExportRequestConverter.java
@@ -28,11 +28,10 @@ public class ServiceLogExportRequestConverter extends AbstractCommonServiceLogRe
@Override
public ServiceLogExportSearchCriteria createCriteria(ServiceLogExportRequest request) {
ServiceLogExportSearchCriteria criteria = new ServiceLogExportSearchCriteria();
- criteria.addParam("hostLogFile", request.getHostLogFile());
- criteria.addParam("compLogFile",
- request.getComponentLogFile());
- criteria.addParam("format", request.getFormat());
- criteria.addParam("utcOffset", request.getUtcOffset());
+ criteria.setLogFileHostName(request.getHostLogFile());
+ criteria.setLogFileComponentName(request.getComponentLogFile());
+ criteria.setFormat(request.getFormat());
+ criteria.setUtcOffset(request.getUtcOffset());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogFileRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogFileRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogFileRequestConverter.java
index f5148f3..392c9d3 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogFileRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogFileRequestConverter.java
@@ -29,8 +29,8 @@ public class ServiceLogFileRequestConverter
@Override
public ServiceLogFileSearchCriteria createCriteria(ServiceLogFileRequest request) {
ServiceLogFileSearchCriteria criteria = new ServiceLogFileSearchCriteria();
- criteria.addParam("hostLogFile", request.getHostLogFile());
- criteria.addParam("compLogFile", request.getComponentLogFile());
+ criteria.setLogFileHostName(request.getHostLogFile());
+ criteria.setLogFileComponentName(request.getComponentLogFile());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogRequestConverter.java
index 6a70d55..aa93c6a 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogRequestConverter.java
@@ -29,13 +29,13 @@ public class ServiceLogRequestConverter extends AbstractCommonServiceLogRequestC
@Override
public ServiceLogSearchCriteria createCriteria(ServiceLogRequest request) {
ServiceLogSearchCriteria criteria = new ServiceLogSearchCriteria();
- criteria.addParam("hostLogFile", request.getHostLogFile());
- criteria.addParam("compLogFile", request.getComponentLogFile());
- criteria.addParam("keyword", StringEscapeUtils.unescapeXml(request.getKeyWord()));
- criteria.addParam("sourceLogId", request.getSourceLogId());
- criteria.addParam("keywordType", request.getKeywordType());
- criteria.addParam("token", request.getToken());
- criteria.addParam("isLastPage", request.isLastPage());
+ criteria.setLogFileComponentName(request.getHostLogFile());
+ criteria.setLogFileComponentName(request.getComponentLogFile());
+ criteria.setKeyword(StringEscapeUtils.unescapeXml(request.getKeyWord()));
+ criteria.setKeywordType(request.getKeywordType());
+ criteria.setSourceLogId(request.getSourceLogId());
+ criteria.setToken(request.getToken());
+ criteria.setLastPage(request.isLastPage());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogTruncatedRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogTruncatedRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogTruncatedRequestConverter.java
index 676f049..1e58b79 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogTruncatedRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/ServiceLogTruncatedRequestConverter.java
@@ -28,11 +28,11 @@ public class ServiceLogTruncatedRequestConverter extends AbstractCommonServiceLo
@Override
public ServiceLogTruncatedSearchCriteria createCriteria(ServiceLogTruncatedRequest request) {
ServiceLogTruncatedSearchCriteria criteria = new ServiceLogTruncatedSearchCriteria();
- criteria.addParam("hostLogFile", request.getHostLogFile());
- criteria.addParam("compLogFile", request.getComponentLogFile());
- criteria.addParam("id", request.getId());
- criteria.addParam("scrollType", request.getScrollType());
- criteria.addParam("numberRows", request.getNumberRows());
+ criteria.setLogFileHostName(request.getHostLogFile());
+ criteria.setLogFileComponentName(request.getComponentLogFile());
+ criteria.setId(request.getId());
+ criteria.setScrollType(request.getScrollType());
+ criteria.setNumberRows(request.getNumberRows());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/SimpleQueryRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/SimpleQueryRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/SimpleQueryRequestConverter.java
index 8c50f66..1f084fd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/SimpleQueryRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/SimpleQueryRequestConverter.java
@@ -19,16 +19,17 @@
package org.apache.ambari.logsearch.query.converter;
import org.apache.ambari.logsearch.model.request.impl.SimpleQueryRequest;
-import org.apache.ambari.logsearch.query.model.SearchCriteria;
-import org.springframework.core.convert.converter.Converter;
+import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
import org.springframework.stereotype.Component;
@Component
-public class SimpleQueryRequestConverter implements Converter<SimpleQueryRequest, SearchCriteria> {
+public class SimpleQueryRequestConverter extends AbstractConverterAware<SimpleQueryRequest, CommonSearchCriteria> {
+
@Override
- public SearchCriteria convert(SimpleQueryRequest simpleQueryRequest) {
- SearchCriteria searchCriteria = new SearchCriteria();
+ public CommonSearchCriteria convert(SimpleQueryRequest simpleQueryRequest) {
+ CommonSearchCriteria searchCriteria = new CommonSearchCriteria();
searchCriteria.addParam("q", simpleQueryRequest.getQuery());
return searchCriteria;
}
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserConfigRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserConfigRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserConfigRequestConverter.java
index 50847c7..8f2aaa0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserConfigRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserConfigRequestConverter.java
@@ -18,21 +18,19 @@
*/
package org.apache.ambari.logsearch.query.converter;
-import org.apache.ambari.logsearch.common.LogSearchConstants;
import org.apache.ambari.logsearch.model.request.impl.UserConfigRequest;
import org.apache.ambari.logsearch.query.model.UserConfigSearchCriteria;
-import org.springframework.core.convert.converter.Converter;
import org.springframework.stereotype.Component;
@Component
-public class UserConfigRequestConverter implements Converter<UserConfigRequest, UserConfigSearchCriteria> {
+public class UserConfigRequestConverter extends AbstractConverterAware<UserConfigRequest, UserConfigSearchCriteria> {
@Override
public UserConfigSearchCriteria convert(UserConfigRequest request) {
UserConfigSearchCriteria criteria = new UserConfigSearchCriteria();
- criteria.addParam(LogSearchConstants.USER_NAME, request.getUserId());
- criteria.addParam(LogSearchConstants.FILTER_NAME, request.getFilterName());
- criteria.addParam(LogSearchConstants.ROW_TYPE, request.getRowType());
+ criteria.setUserName(request.getUserId());
+ criteria.setFilterName(request.getFilterName());
+ criteria.setRowType(request.getRowType());
return criteria;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserExportRequestConverter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserExportRequestConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserExportRequestConverter.java
index a0b5f0f..c7f738e 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserExportRequestConverter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/converter/UserExportRequestConverter.java
@@ -28,7 +28,8 @@ public class UserExportRequestConverter extends AbstractCommonAuditLogRequestCon
@Override
public UserExportSearchCriteria createCriteria(UserExportRequest request) {
UserExportSearchCriteria criteria = new UserExportSearchCriteria();
- criteria.addParam("field", request.getField());
+ criteria.setField(request.getField());
+ criteria.setFormat(request.getFormat());
return criteria;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AnyGraphSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AnyGraphSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AnyGraphSearchCriteria.java
index a11c056..aa61851 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AnyGraphSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AnyGraphSearchCriteria.java
@@ -18,8 +18,60 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_FROM;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_STACK_BY;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_TO;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_UNIT;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_X_AXIS;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_Y_AXIS;
-@Marker
public class AnyGraphSearchCriteria extends CommonSearchCriteria {
+
+ public String getxAxis() {
+ return getParam(PARAM_X_AXIS, String.class);
+ }
+
+ public void setxAxis(String xAxis) {
+ addParam(PARAM_X_AXIS, xAxis);
+ }
+
+ public String getyAxis() {
+ return getParam(PARAM_Y_AXIS, String.class);
+ }
+
+ public void setyAxis(String yAxis) {
+ addParam(PARAM_Y_AXIS, yAxis);
+ }
+
+ public String getStackBy() {
+ return getParam(PARAM_STACK_BY, String.class);
+ }
+
+ public void setStackBy(String stackBy) {
+ addParam(PARAM_STACK_BY, stackBy);
+ }
+
+ public String getUnit() {
+ return getParam(PARAM_UNIT, String.class);
+ }
+
+ public void setUnit(String unit) {
+ addParam(PARAM_UNIT, unit);
+ }
+
+ public String getFrom() {
+ return getParam(PARAM_FROM, String.class);
+ }
+
+ public void setFrom(String from) {
+ addParam(PARAM_FROM, from);
+ }
+
+ public String getTo() {
+ return getParam(PARAM_TO, String.class);
+ }
+
+ public void setTo(String to) {
+ addParam(PARAM_TO, to);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditBarGraphSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditBarGraphSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditBarGraphSearchCriteria.java
index c41ec15..49304c4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditBarGraphSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditBarGraphSearchCriteria.java
@@ -18,8 +18,16 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_UNIT;
-@Marker
public class AuditBarGraphSearchCriteria extends CommonSearchCriteria {
+
+ public void setUnit(String unit) {
+ addParam(PARAM_UNIT, unit);
+ }
+
+ public String getUnit() {
+ return getParam(PARAM_UNIT, String.class);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditLogSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditLogSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditLogSearchCriteria.java
index f4fe207..03df3ad 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditLogSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/AuditLogSearchCriteria.java
@@ -18,8 +18,16 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_IS_LAST_PAGE;
-@Marker
public class AuditLogSearchCriteria extends CommonSearchCriteria {
+
+ public void setLastPage(boolean lastPage) {
+ addParam(PARAM_IS_LAST_PAGE, lastPage);
+ }
+
+ public boolean isLastPage() {
+ return getParam(PARAM_IS_LAST_PAGE, Boolean.class);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonSearchCriteria.java
index 47d12e5..ffdb2c7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonSearchCriteria.java
@@ -18,83 +18,78 @@
*/
package org.apache.ambari.logsearch.query.model;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_END_TIME;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_EXCLUDE_MESSAGE;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_EXCLUDE_QUERY;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_INCLUDE_MESSAGE;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_INCLUDE_QUERY;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_MUST_BE_STRING;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_MUST_NOT_STRING;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_START_TIME;
+
public class CommonSearchCriteria extends SearchCriteria {
- private int startIndex = 0;
- private int maxRows = Integer.MAX_VALUE;
- private String sortBy = null;
- private String sortType = null;
- private int page = 0;
- private String globalStartTime = null;
- private String globalEndTime = null;
+ public String getIncludeMessage() {
+ return getParam(PARAM_INCLUDE_MESSAGE, String.class);
+ }
+
+ public void setIncludeMessage(String includeMessage) {
+ addParam(PARAM_INCLUDE_MESSAGE, includeMessage);
+ }
- @Override
- public int getStartIndex() {
- return startIndex;
+ public String getExcludeMessage() {
+ return getParam(PARAM_EXCLUDE_MESSAGE, String.class);
}
- @Override
- public void setStartIndex(int startIndex) {
- this.startIndex = startIndex;
+ public void setExcludeMessage(String excludeMessage) {
+ addParam(PARAM_EXCLUDE_MESSAGE, excludeMessage);
}
- @Override
- public int getMaxRows() {
- return maxRows;
+ public String getMustBe() {
+ return getParam(PARAM_MUST_BE_STRING, String.class);
}
- @Override
- public void setMaxRows(int maxRows) {
- this.maxRows = maxRows;
+ public void setMustBe(String mustHave) {
+ addParam(PARAM_MUST_BE_STRING, mustHave);
}
- @Override
- public String getSortType() {
- return sortType;
+ public String getMustNot() {
+ return getParam(PARAM_MUST_NOT_STRING, String.class);
}
- @Override
- public void setSortType(String sortType) {
- this.sortType = sortType;
+ public void setMustNot(String mustNot) {
+ addParam(PARAM_MUST_NOT_STRING, mustNot);
}
- @Override
- public String getSortBy() {
- return sortBy;
+ public String getIncludeQuery() {
+ return getParam(PARAM_INCLUDE_QUERY, String.class);
}
- @Override
- public void setSortBy(String sortBy) {
- this.sortBy = sortBy;
+ public void setIncludeQuery(String includeQuery) {
+ addParam(PARAM_INCLUDE_QUERY, includeQuery);
}
- @Override
- public int getPage() {
- return page;
+ public String getExcludeQuery() {
+ return getParam(PARAM_EXCLUDE_QUERY, String.class);
}
- @Override
- public void setPage(int page) {
- this.page = page;
+ public void setExcludeQuery(String excludeQuery) {
+ addParam(PARAM_EXCLUDE_QUERY, excludeQuery);
}
- @Override
- public String getGlobalStartTime() {
- return globalStartTime;
+ public String getStartTime() {
+ return getParam(PARAM_START_TIME, String.class);
}
- @Override
- public void setGlobalStartTime(String globalStartTime) {
- this.globalStartTime = globalStartTime;
+ public void setStartTime(String startTime) {
+ addParam(PARAM_START_TIME, startTime);
}
- @Override
- public String getGlobalEndTime() {
- return globalEndTime;
+ public String getEndTime() {
+ return getParam(PARAM_END_TIME, String.class);
}
- @Override
- public void setGlobalEndTime(String globalEndTime) {
- this.globalEndTime = globalEndTime;
+ public void setEndTime(String endTime) {
+ addParam(PARAM_END_TIME, endTime);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonServiceLogSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonServiceLogSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonServiceLogSearchCriteria.java
new file mode 100644
index 0000000..4c79c69
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/CommonServiceLogSearchCriteria.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.query.model;
+
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
+
+public class CommonServiceLogSearchCriteria extends CommonSearchCriteria {
+
+ public String getTreeParams() {
+ return getParam(SearchCriteriaConstants.PARAM_TREE_PARAMS, String.class);
+ }
+
+ public void setTreeParams(String treeParams) {
+ addParam(SearchCriteriaConstants.PARAM_TREE_PARAMS, treeParams);
+ }
+
+ public String getLevel() {
+ return getParam(SearchCriteriaConstants.PARAM_LEVEL, String.class);
+ }
+
+ public void setLevel(String level) {
+ addParam(SearchCriteriaConstants.PARAM_LEVEL, level);
+ }
+
+ public String getSelectComp() {
+ return getParam(SearchCriteriaConstants.PARAM_SELECT_COMP, String.class);
+ }
+
+ public void setSelectComp(String selectComp) {
+ addParam(SearchCriteriaConstants.PARAM_SELECT_COMP, selectComp);
+ }
+
+ public String getBundleId() {
+ return getParam(SearchCriteriaConstants.PARAM_BUNDLE_ID, String.class);
+ }
+
+ public void setBundleId(String bunldeId) {
+ addParam(SearchCriteriaConstants.PARAM_BUNDLE_ID, bunldeId);
+ }
+
+ public String getFrom() {
+ return getParam(SearchCriteriaConstants.PARAM_FROM ,String.class);
+ }
+
+ public void setFrom(String from) {
+ addParam(SearchCriteriaConstants.PARAM_FROM, from);
+ }
+
+ public String getTo() {
+ return getParam(SearchCriteriaConstants.PARAM_TO ,String.class);
+ }
+
+ public void setTo(String to) {
+ addParam(SearchCriteriaConstants.PARAM_TO, to);
+ }
+
+ public String getHostName() {
+ return getParam(SearchCriteriaConstants.PARAM_HOST_NAME ,String.class);
+ }
+
+ public void setHostName(String hostName) {
+ addParam(SearchCriteriaConstants.PARAM_HOST_NAME, hostName);
+ }
+
+ public String getComponentName() {
+ return getParam(SearchCriteriaConstants.PARAM_COMPONENT_NAME ,String.class);
+ }
+
+ public void setComponentName(String componentName) {
+ addParam(SearchCriteriaConstants.PARAM_COMPONENT_NAME, componentName);
+ }
+
+ public String getFileName() {
+ return getParam(SearchCriteriaConstants.PARAM_FILE_NAME ,String.class);
+ }
+
+ public void setFileName(String fileName) {
+ addParam(SearchCriteriaConstants.PARAM_FILE_NAME, fileName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditBarGraphSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditBarGraphSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditBarGraphSearchCriteria.java
index f931f5d..381b3c2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditBarGraphSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditBarGraphSearchCriteria.java
@@ -18,8 +18,15 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_FIELD;
-@Marker
public class FieldAuditBarGraphSearchCriteria extends AuditBarGraphSearchCriteria {
+
+ public String getField() {
+ return getParam(PARAM_FIELD, String.class);
+ }
+
+ public void setField(String field) {
+ addParam(PARAM_FIELD, field);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditLogSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditLogSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditLogSearchCriteria.java
index 8dd5854..9d5d225 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditLogSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/FieldAuditLogSearchCriteria.java
@@ -18,8 +18,15 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_FIELD;
-@Marker
public class FieldAuditLogSearchCriteria extends CommonSearchCriteria {
+
+ public String getField() {
+ return getParam(PARAM_FIELD, String.class);
+ }
+
+ public void setField(String field) {
+ addParam(PARAM_FIELD, field);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileSearchCriteria.java
index e4e2a14..e33e5eb 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileSearchCriteria.java
@@ -20,6 +20,33 @@ package org.apache.ambari.logsearch.query.model;
import org.apache.ambari.logsearch.common.Marker;
-@Marker
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_LOG_FILE_COMPONENT;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_LOG_FILE_HOST;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_LOG_FILE_TYPE;
+
public class LogFileSearchCriteria extends SearchCriteria {
+
+ public String getLogFileComponent() {
+ return getParam(PARAM_LOG_FILE_COMPONENT, String.class);
+ }
+
+ public void setLogFileComponent(String logFileComponent) {
+ addParam(PARAM_LOG_FILE_COMPONENT, logFileComponent);
+ }
+
+ public String getLogFileHost() {
+ return getParam(PARAM_LOG_FILE_HOST, String.class);
+ }
+
+ public void setLogFileHost(String logFileHost) {
+ addParam(PARAM_LOG_FILE_HOST, logFileHost);
+ }
+
+ public String getLogType() {
+ return getParam(PARAM_LOG_FILE_TYPE, String.class);
+ }
+
+ public void setLogType(String logType) {
+ addParam(PARAM_LOG_FILE_TYPE, logType);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileTailSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileTailSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileTailSearchCriteria.java
index fecb396..ccea4ab 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileTailSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/LogFileTailSearchCriteria.java
@@ -18,8 +18,15 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_LOG_TAIL_SIZE;
-@Marker
-public class LogFileTailSearchCriteria extends SearchCriteria {
+public class LogFileTailSearchCriteria extends LogFileSearchCriteria {
+
+ public String getLogTailSize() {
+ return getParam(PARAM_LOG_TAIL_SIZE, String.class);
+ }
+
+ public void setLogTailSize(String logTailSize) {
+ addParam(PARAM_LOG_TAIL_SIZE, logTailSize);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/SearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/SearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/SearchCriteria.java
index 091194e..661337f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/SearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/SearchCriteria.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.ambari.logsearch.common.PropertiesHelper;
public class SearchCriteria {
+
private int startIndex = 0;
private int maxRows = Integer.MAX_VALUE;
private String sortBy = null;
@@ -42,6 +43,30 @@ public class SearchCriteria {
// Auto-generated constructor stub
}
+
+ public void addParam(String name, Object value) {
+ paramList.put(name, value);
+ }
+
+ public Object getParamValue(String name) {
+ return paramList.get(name);
+ }
+
+ public <T> T getParam(String key, Class<T> type) {
+ if (getParamValue(key) != null) {
+ return (T) getParamValue(key);
+ }
+ return null;
+ }
+
+ public Map<String, Object> getUrlParamMap() {
+ return urlParamMap;
+ }
+
+ public void setUrlParamMap(Map<String, Object> urlParamMap) {
+ this.urlParamMap = urlParamMap;
+ }
+
public int getStartIndex() {
return startIndex;
}
@@ -62,33 +87,6 @@ public class SearchCriteria {
return sortType;
}
-
- public void addParam(String name, Object value) {
- String solrValue = PropertiesHelper.getProperty(name);
- if (solrValue == null || solrValue.isEmpty()) {
- paramList.put(name, value);
- } else {
- try {
- String propertyFieldMappings[] = solrValue.split(",");
- HashMap<String, String> propertyFieldValue = new HashMap<String, String>();
- for (String temp : propertyFieldMappings) {
- String arrayValue[] = temp.split(":");
- propertyFieldValue.put(arrayValue[0].toLowerCase(Locale.ENGLISH), arrayValue[1].toLowerCase(Locale.ENGLISH));
- }
- String originalValue = propertyFieldValue.get(value.toString().toLowerCase(Locale.ENGLISH));
- if (originalValue != null && !originalValue.isEmpty())
- paramList.put(name, originalValue);
-
- } catch (Exception e) {
- //do nothing
- }
- }
- }
-
- public Object getParamValue(String name) {
- return paramList.get(name);
- }
-
public String getSortBy() {
return sortBy;
}
@@ -125,12 +123,4 @@ public class SearchCriteria {
this.globalEndTime = globalEndTime;
}
- public Map<String, Object> getUrlParamMap() {
- return urlParamMap;
- }
-
- public void setUrlParamMap(Map<String, Object> urlParamMap) {
- this.urlParamMap = urlParamMap;
- }
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceAnyGraphSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceAnyGraphSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceAnyGraphSearchCriteria.java
index 0ef5bdf..cd852b5 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceAnyGraphSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceAnyGraphSearchCriteria.java
@@ -20,6 +20,41 @@ package org.apache.ambari.logsearch.query.model;
import org.apache.ambari.logsearch.common.Marker;
-@Marker
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_STACK_BY;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_UNIT;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_X_AXIS;
+import static org.apache.ambari.logsearch.query.SearchCriteriaConstants.PARAM_Y_AXIS;
+
public class ServiceAnyGraphSearchCriteria extends ServiceLogFileSearchCriteria {
+ public String getxAxis() {
+ return getParam(PARAM_X_AXIS, String.class);
+ }
+
+ public void setxAxis(String xAxis) {
+ addParam(PARAM_X_AXIS, xAxis);
+ }
+
+ public String getyAxis() {
+ return getParam(PARAM_Y_AXIS, String.class);
+ }
+
+ public void setyAxis(String yAxis) {
+ addParam(PARAM_Y_AXIS, yAxis);
+ }
+
+ public String getStackBy() {
+ return getParam(PARAM_STACK_BY, String.class);
+ }
+
+ public void setStackBy(String stackBy) {
+ addParam(PARAM_STACK_BY, stackBy);
+ }
+
+ public String getUnit() {
+ return getParam(PARAM_UNIT, String.class);
+ }
+
+ public void setUnit(String unit) {
+ addParam(PARAM_UNIT, unit);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceExtremeDatesCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceExtremeDatesCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceExtremeDatesCriteria.java
index 3fc6ff8..d89ab3b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceExtremeDatesCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceExtremeDatesCriteria.java
@@ -18,8 +18,15 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class ServiceExtremeDatesCriteria extends CommonSearchCriteria {
+
+ public String getBundleId() {
+ return getParam(SearchCriteriaConstants.PARAM_BUNDLE_ID, String.class);
+ }
+
+ public void setBundleId(String bunldeId) {
+ addParam(SearchCriteriaConstants.PARAM_BUNDLE_ID, bunldeId);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceGraphSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceGraphSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceGraphSearchCriteria.java
index 31a57a4..8988af1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceGraphSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceGraphSearchCriteria.java
@@ -18,8 +18,15 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class ServiceGraphSearchCriteria extends ServiceLogFileSearchCriteria {
+
+ public String getUnit() {
+ return getParam(SearchCriteriaConstants.PARAM_UNIT, String.class);
+ }
+
+ public void setUnit(String unit) {
+ addParam(SearchCriteriaConstants.PARAM_UNIT, unit);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogExportSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogExportSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogExportSearchCriteria.java
index 8bab7f0..c7b7a39 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogExportSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogExportSearchCriteria.java
@@ -18,8 +18,23 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class ServiceLogExportSearchCriteria extends ServiceLogFileSearchCriteria {
+
+ public void setFormat(String format) {
+ addParam(SearchCriteriaConstants.PARAM_FORMAT, format);
+ }
+
+ public String getFormat() {
+ return getParam(SearchCriteriaConstants.PARAM_FORMAT, String.class);
+ }
+
+ public void setUtcOffset(String utcOffset) {
+ addParam(SearchCriteriaConstants.PARAM_UTC_OFFSET, utcOffset);
+ }
+
+ public String getUtcOffset() {
+ return getParam(SearchCriteriaConstants.PARAM_UTC_OFFSET, String.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogFileSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogFileSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogFileSearchCriteria.java
index a9f5926..a80a3ce 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogFileSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogFileSearchCriteria.java
@@ -18,8 +18,23 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
-public class ServiceLogFileSearchCriteria extends CommonSearchCriteria {
+public class ServiceLogFileSearchCriteria extends CommonServiceLogSearchCriteria {
+
+ public void setLogFileHostName(String logFileHostName) {
+ addParam(SearchCriteriaConstants.PARAM_HOST_LOG_FILE, logFileHostName);
+ }
+
+ public String getLogFileHostName() {
+ return getParam(SearchCriteriaConstants.PARAM_HOST_LOG_FILE, String.class);
+ }
+
+ public void setLogFileComponentName(String logFileComponentName) {
+ addParam(SearchCriteriaConstants.PARAM_COMPONENT_LOG_FILE, logFileComponentName);
+ }
+
+ public String getLogFileComponentName() {
+ return getParam(SearchCriteriaConstants.PARAM_COMPONENT_LOG_FILE, String.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogSearchCriteria.java
index d41c589..b3a6bf7 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogSearchCriteria.java
@@ -18,8 +18,48 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class ServiceLogSearchCriteria extends ServiceLogFileSearchCriteria {
+
+ public void setKeyword(String keyword) {
+ addParam(SearchCriteriaConstants.PARAM_KEYWORD, keyword);
+ }
+
+ public String getKeyword() {
+ return getParam(SearchCriteriaConstants.PARAM_KEYWORD, String.class);
+ }
+
+ public void setKeywordType(String keywordType) {
+ addParam(SearchCriteriaConstants.PARAM_KEYWORD_TYPE, keywordType);
+ }
+
+ public String getKeywordType() {
+ return getParam(SearchCriteriaConstants.PARAM_KEYWORD_TYPE, String.class);
+ }
+
+ public void setSourceLogId(String sourceLogId) {
+ addParam(SearchCriteriaConstants.PARAM_SOURCE_LOG_ID, sourceLogId);
+ }
+
+ public String getSourceLogId() {
+ return getParam(SearchCriteriaConstants.PARAM_SOURCE_LOG_ID, String.class);
+ }
+
+ public void setToken(String token) {
+ addParam(SearchCriteriaConstants.PARAM_TOKEN, token);
+ }
+
+ public String getToken() {
+ return getParam(SearchCriteriaConstants.PARAM_TOKEN, String.class);
+ }
+
+ public void setLastPage(boolean lastPage) {
+ addParam(SearchCriteriaConstants.PARAM_IS_LAST_PAGE, lastPage);
+ }
+
+ public boolean isLastPage() {
+ return getParam(SearchCriteriaConstants.PARAM_IS_LAST_PAGE, Boolean.class);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogTruncatedSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogTruncatedSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogTruncatedSearchCriteria.java
index 24dc9a8..bcdac5b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogTruncatedSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/ServiceLogTruncatedSearchCriteria.java
@@ -18,8 +18,31 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class ServiceLogTruncatedSearchCriteria extends ServiceLogFileSearchCriteria {
+
+ public String getId() {
+ return getParam(SearchCriteriaConstants.PARAM_ID, String.class);
+ }
+
+ public void setId(String id) {
+ addParam(SearchCriteriaConstants.PARAM_ID, id);
+ }
+
+ public String getScrollType() {
+ return getParam(SearchCriteriaConstants.PARAM_SCROLL_TYPE, String.class);
+ }
+
+ public void setScrollType(String scrollType) {
+ addParam(SearchCriteriaConstants.PARAM_SCROLL_TYPE, scrollType);
+ }
+
+ public String getNumberRows() {
+ return getParam(SearchCriteriaConstants.PARAM_NUMBER_ROWS, String.class);
+ }
+
+ public void setNumberRows(String numberRows) {
+ addParam(SearchCriteriaConstants.PARAM_NUMBER_ROWS, numberRows);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserConfigSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserConfigSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserConfigSearchCriteria.java
index 8798cd6..eafff69 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserConfigSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserConfigSearchCriteria.java
@@ -18,8 +18,31 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class UserConfigSearchCriteria extends CommonSearchCriteria {
+
+ public String getUserName() {
+ return getParam(SearchCriteriaConstants.PARAM_USER_NAME, String.class);
+ }
+
+ public void setUserName(String userName) {
+ addParam(SearchCriteriaConstants.PARAM_USER_NAME, userName);
+ }
+
+ public String getFilterName() {
+ return getParam(SearchCriteriaConstants.PARAM_FILTER_NAME, String.class);
+ }
+
+ public void setFilterName(String filterName) {
+ addParam(SearchCriteriaConstants.PARAM_FILTER_NAME, filterName);
+ }
+
+ public String getRowType() {
+ return getParam(SearchCriteriaConstants.PARAM_ROW_TYPE, String.class);
+ }
+
+ public void setRowType(String rowType) {
+ addParam(SearchCriteriaConstants.PARAM_ROW_TYPE, rowType);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserExportSearchCriteria.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserExportSearchCriteria.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserExportSearchCriteria.java
index 755c673..46d13cc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserExportSearchCriteria.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/model/UserExportSearchCriteria.java
@@ -18,8 +18,14 @@
*/
package org.apache.ambari.logsearch.query.model;
-import org.apache.ambari.logsearch.common.Marker;
+import org.apache.ambari.logsearch.query.SearchCriteriaConstants;
-@Marker
public class UserExportSearchCriteria extends FieldAuditLogSearchCriteria {
+ public void setFormat(String format) {
+ addParam(SearchCriteriaConstants.PARAM_FORMAT, format);
+ }
+
+ public String getFormat() {
+ return getParam(SearchCriteriaConstants.PARAM_FORMAT, String.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
index 82e21e8..ff185f8 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java
@@ -87,7 +87,7 @@ public class AuditLogsResource {
@Produces({"application/json"})
@ApiOperation(GET_AUDIT_COMPONENTS_OD)
public GroupListResponse getAuditComponents(@BeanParam SimpleQueryRequest request) {
- return auditLogsManager.getAuditComponents(conversionService.convert(request, SearchCriteria.class));
+ return auditLogsManager.getAuditComponents(conversionService.convert(request, CommonSearchCriteria.class));
}
@GET
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
index 5400825..6cc69d4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java
@@ -45,6 +45,7 @@ import org.apache.ambari.logsearch.model.response.NameValueDataListResponse;
import org.apache.ambari.logsearch.model.response.NodeListResponse;
import org.apache.ambari.logsearch.model.response.ServiceLogResponse;
import org.apache.ambari.logsearch.query.model.CommonSearchCriteria;
+import org.apache.ambari.logsearch.query.model.CommonServiceLogSearchCriteria;
import org.apache.ambari.logsearch.query.model.SearchCriteria;
import org.apache.ambari.logsearch.manager.ServiceLogsManager;
import org.apache.ambari.logsearch.query.model.ServiceAnyGraphSearchCriteria;
@@ -100,7 +101,7 @@ public class ServiceLogsResource {
@Produces({"application/json"})
@ApiOperation(GET_AGGREGATED_INFO_OD)
public GraphDataListResponse getAggregatedInfo(@BeanParam BaseServiceLogRequest request) {
- return serviceLogsManager.getAggregatedInfo(conversionService.convert(request, CommonSearchCriteria.class));
+ return serviceLogsManager.getAggregatedInfo(conversionService.convert(request, CommonServiceLogSearchCriteria.class));
}
@GET
@@ -132,7 +133,7 @@ public class ServiceLogsResource {
@Produces({"application/json"})
@ApiOperation(GET_TREE_EXTENSION_OD)
public NodeListResponse getTreeExtension(@QueryParam("hostName") @ApiParam String hostName, @BeanParam ServiceLogFileRequest request) {
- SearchCriteria searchCriteria = conversionService.convert(request, ServiceLogFileSearchCriteria.class);
+ ServiceLogFileSearchCriteria searchCriteria = conversionService.convert(request, ServiceLogFileSearchCriteria.class);
searchCriteria.addParam("hostName", hostName); // TODO: use host_name instead - needs UI change
return serviceLogsManager.getTreeExtension(searchCriteria);
}
@@ -175,7 +176,7 @@ public class ServiceLogsResource {
@Produces({"application/json"})
@ApiOperation(GET_HOST_LIST_BY_COMPONENT_OD)
public NodeListResponse getHostListByComponent(@BeanParam ServiceLogFileRequest request, @QueryParam("componentName") @ApiParam String componentName) {
- SearchCriteria searchCriteria = conversionService.convert(request, ServiceLogFileSearchCriteria.class);
+ ServiceLogFileSearchCriteria searchCriteria = conversionService.convert(request, ServiceLogFileSearchCriteria.class);
searchCriteria.addParam("componentName", componentName); // TODO: use component_name instead - needs UI change
return serviceLogsManager.getHostListByComponent(searchCriteria);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java
deleted file mode 100644
index 85ea69d..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrAmsClient.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logsearch.solr.metrics;
-
-import org.apache.hadoop.metrics2.sink.timeline.AbstractTimelineMetricsSink;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-
-// TODO: Refactor for failover
-public class SolrAmsClient extends AbstractTimelineMetricsSink {
- private final String collectorHost;
-
- public SolrAmsClient(String collectorHost) {
- this.collectorHost = collectorHost;
- }
-
- @Override
- public String getCollectorUri(String host) {
- return collectorHost;
- }
-
- @Override
- protected int getTimeoutSeconds() {
- return 10;
- }
-
- @Override
- protected String getZookeeperQuorum() {
- return null;
- }
-
- @Override
- protected String getConfiguredCollectors() {
- return null;
- }
-
- @Override
- protected String getHostname() {
- return null;
- }
-
- @Override
- protected boolean emitMetrics(TimelineMetrics metrics) {
- return super.emitMetrics(metrics);
- }
-
- @Override
- protected String getCollectorProtocol() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java
deleted file mode 100644
index fc58661..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrJmxAdapter.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.solr.metrics;
-
-import java.io.IOException;
-import java.lang.management.MemoryMXBean;
-import java.net.MalformedURLException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-import javax.management.JMX;
-import javax.management.MBeanServerConnection;
-import javax.management.MalformedObjectNameException;
-import javax.management.ObjectName;
-import javax.management.remote.JMXConnector;
-import javax.management.remote.JMXConnectorFactory;
-import javax.management.remote.JMXServiceURL;
-
-import com.sun.management.OperatingSystemMXBean;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@SuppressWarnings("restriction")
-public class SolrJmxAdapter {
- private static final Logger LOG = LoggerFactory.getLogger(SolrJmxAdapter.class);
-
- private static final String JMX_SERVICE_URL = "service:jmx:rmi:///jndi/rmi://%s:%d/jmxrmi";
-
- private final JMXServiceURL jmxServiceUrl;
-
- private JMXConnector jmxConnector;
- private MBeanServerConnection conn;
-
- public SolrJmxAdapter(String host, int port) throws MalformedURLException {
- String url = String.format(JMX_SERVICE_URL, host, port);
- jmxServiceUrl = new JMXServiceURL(url);
- }
-
- public double getProcessCpuLoad() throws MalformedObjectNameException {
- ObjectName objectName = new ObjectName("java.lang:type=OperatingSystem");
- OperatingSystemMXBean mxBean = JMX.newMXBeanProxy(conn, objectName, OperatingSystemMXBean.class);
- return mxBean.getProcessCpuLoad();
- }
-
- public Map<String, Long> getMemoryData() throws MalformedObjectNameException {
- Map<String, Long> memoryData = new HashMap<>();
-
- ObjectName objectName = new ObjectName("java.lang:type=Memory");
- MemoryMXBean mxBean = JMX.newMXBeanProxy(conn, objectName, MemoryMXBean.class);
-
- memoryData.put("heapMemoryUsed", mxBean.getHeapMemoryUsage().getUsed());
- memoryData.put("heapMemoryCommitted", mxBean.getHeapMemoryUsage().getCommitted());
- memoryData.put("heapMemoryMax", mxBean.getHeapMemoryUsage().getMax());
- memoryData.put("nonHeapMemoryUsed", mxBean.getNonHeapMemoryUsage().getUsed());
- memoryData.put("nonHeapMemoryCommitted", mxBean.getNonHeapMemoryUsage().getCommitted());
- memoryData.put("nonHeapMemoryMax", mxBean.getNonHeapMemoryUsage().getMax());
-
- return memoryData;
- }
-
- public long getIndexSize() throws Exception {
- long indexSize = 0;
-
- ObjectName objectNamePattern = new ObjectName(
- "solr/*shard*replica*:type=/replication,id=org.apache.solr.handler.ReplicationHandler");
- Set<ObjectName> objectNames = conn.queryNames(objectNamePattern, null);
- for (ObjectName objectName : objectNames) {
- String indexSizeString = (String) conn.getAttribute(objectName, "indexSize");
- indexSize += getIndexSizeInBytes(indexSizeString);
- }
-
- return indexSize;
- }
-
- private long getIndexSizeInBytes(String indexSizeString) {
- String[] tokens = indexSizeString.split(" ");
- double number = Double.parseDouble(tokens[0]);
-
- long multiplier = 0;
- switch (tokens[1]) {
- case "bytes":
- multiplier = 1;
- break;
- case "KB":
- multiplier = 1024;
- break;
- case "MB":
- multiplier = 1024 * 1024;
- break;
- case "GB":
- multiplier = 1024 * 1024 * 1024;
- break;
- default:
- throw new IllegalArgumentException("Unknown unit: " + tokens[1]);
- }
-
- return (long)(number * multiplier);
- }
-
- public void reConnect() throws IOException {
- if (jmxConnector != null) {
- try {
- jmxConnector.close();
- } catch (IOException e) {
- LOG.info("Could not close jmxConnector", e);
- }
- }
-
- connect();
- }
-
- public void connect() throws IOException {
- jmxConnector = JMXConnectorFactory.connect(jmxServiceUrl);
- conn = jmxConnector.getMBeanServerConnection();
- }
-}
[04/10] ambari git commit: AMBARI-18246. Clean up Log Feeder (Miklos
Gergely via oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
index 18a5a54..e1a0bb9 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputFile.java
@@ -30,26 +30,27 @@ import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
public class OutputFile extends Output {
- static Logger logger = Logger.getLogger(OutputFile.class);
+ private static final Logger LOG = Logger.getLogger(OutputFile.class);
- PrintWriter outWriter = null;
- String filePath = null;
- String codec;
+ private PrintWriter outWriter;
+ private String filePath = null;
+ private String codec;
@Override
public void init() throws Exception {
super.init();
filePath = getStringValue("path");
- if (filePath == null || filePath.isEmpty()) {
- logger.error("Filepath config property <path> is not set in config file.");
+ if (StringUtils.isEmpty(filePath)) {
+ LOG.error("Filepath config property <path> is not set in config file.");
return;
}
codec = getStringValue("codec");
- if (codec == null || codec.trim().isEmpty()) {
+ if (StringUtils.isBlank(codec)) {
codec = "json";
} else {
if (codec.trim().equalsIgnoreCase("csv")) {
@@ -57,12 +58,11 @@ public class OutputFile extends Output {
} else if (codec.trim().equalsIgnoreCase("json")) {
codec = "csv";
} else {
- logger.error("Unsupported codec type. codec=" + codec
- + ", will use json");
+ LOG.error("Unsupported codec type. codec=" + codec + ", will use json");
codec = "json";
}
}
- logger.info("Out filePath=" + filePath + ", codec=" + codec);
+ LOG.info("Out filePath=" + filePath + ", codec=" + codec);
File outFile = new File(filePath);
if (outFile.getParentFile() != null) {
File parentDir = outFile.getParentFile();
@@ -71,16 +71,14 @@ public class OutputFile extends Output {
}
}
- outWriter = new PrintWriter(new BufferedWriter(new FileWriter(outFile,
- true)));
+ outWriter = new PrintWriter(new BufferedWriter(new FileWriter(outFile, true)));
- logger.info("init() is successfull. filePath="
- + outFile.getAbsolutePath());
+ LOG.info("init() is successfull. filePath=" + outFile.getAbsolutePath());
}
@Override
public void close() {
- logger.info("Closing file." + getShortDescription());
+ LOG.info("Closing file." + getShortDescription());
if (outWriter != null) {
try {
outWriter.close();
@@ -92,8 +90,7 @@ public class OutputFile extends Output {
}
@Override
- public void write(Map<String, Object> jsonObj, InputMarker inputMarker)
- throws Exception {
+ public void write(Map<String, Object> jsonObj, InputMarker inputMarker) throws Exception {
String outStr = null;
CSVPrinter csvPrinter = null;
try {
@@ -104,7 +101,7 @@ public class OutputFile extends Output {
outStr = LogFeederUtil.getGson().toJson(jsonObj);
}
if (outWriter != null && outStr != null) {
- statMetric.count++;
+ statMetric.value++;
outWriter.println(outStr);
outWriter.flush();
@@ -122,7 +119,7 @@ public class OutputFile extends Output {
@Override
synchronized public void write(String block, InputMarker inputMarker) throws Exception {
if (outWriter != null && block != null) {
- statMetric.count++;
+ statMetric.value++;
outWriter.println(block);
outWriter.flush();
@@ -135,10 +132,7 @@ public class OutputFile extends Output {
}
@Override
- public void copyFile(File inputFile, InputMarker inputMarker)
- throws UnsupportedOperationException {
- throw new UnsupportedOperationException(
- "copyFile method is not yet supported for output=file");
+ public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("copyFile method is not yet supported for output=file");
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
index a360215..8f4b0b1 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputHDFSFile.java
@@ -43,7 +43,8 @@ import java.util.concurrent.ConcurrentLinkedQueue;
* The events are spooled on the local file system and uploaded in batches asynchronously.
*/
public class OutputHDFSFile extends Output implements RolloverHandler, RolloverCondition {
- private final static Logger logger = Logger.getLogger(OutputHDFSFile.class);
+ private static final Logger LOG = Logger.getLogger(OutputHDFSFile.class);
+
private static final long DEFAULT_ROLLOVER_THRESHOLD_TIME_SECONDS = 5 * 60L;// 5 min by default
private ConcurrentLinkedQueue<File> localReadyFiles = new ConcurrentLinkedQueue<File>();
@@ -72,23 +73,20 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
rolloverThresholdTimeMillis = rolloverThresholdTimeSeconds * 1000L;
filenamePrefix = getStringValue("file_name_prefix", filenamePrefix);
if (StringUtils.isEmpty(hdfsOutDir)) {
- logger
- .error("HDFS config property <hdfs_out_dir> is not set in config file.");
+ LOG.error("HDFS config property <hdfs_out_dir> is not set in config file.");
return;
}
if (StringUtils.isEmpty(hdfsHost)) {
- logger
- .error("HDFS config property <hdfs_host> is not set in config file.");
+ LOG.error("HDFS config property <hdfs_host> is not set in config file.");
return;
}
if (StringUtils.isEmpty(hdfsPort)) {
- logger
- .error("HDFS config property <hdfs_port> is not set in config file.");
+ LOG.error("HDFS config property <hdfs_port> is not set in config file.");
return;
}
HashMap<String, String> contextParam = buildContextParam();
hdfsOutDir = PlaceholderUtil.replaceVariables(hdfsOutDir, contextParam);
- logger.info("hdfs Output dir=" + hdfsOutDir);
+ LOG.info("hdfs Output dir=" + hdfsOutDir);
String localFileDir = LogFeederUtil.getLogfeederTempDir() + "hdfs/service/";
logSpooler = new LogSpooler(localFileDir, filenamePrefix, this, this);
this.startHDFSCopyThread();
@@ -96,18 +94,17 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
@Override
public void close() {
- logger.info("Closing file." + getShortDescription());
+ LOG.info("Closing file." + getShortDescription());
logSpooler.rollover();
this.stopHDFSCopyThread();
isClosed = true;
}
@Override
- synchronized public void write(String block, InputMarker inputMarker)
- throws Exception {
+ public synchronized void write(String block, InputMarker inputMarker) throws Exception {
if (block != null) {
logSpooler.add(block);
- statMetric.count++;
+ statMetric.value++;
}
}
@@ -127,24 +124,19 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
Iterator<File> localFileIterator = localReadyFiles.iterator();
while (localFileIterator.hasNext()) {
File localFile = localFileIterator.next();
- fileSystem = LogfeederHDFSUtil.INSTANCE.buildFileSystem(hdfsHost,
- hdfsPort);
+ fileSystem = LogfeederHDFSUtil.buildFileSystem(hdfsHost, hdfsPort);
if (fileSystem != null && localFile.exists()) {
String destFilePath = hdfsOutDir + "/" + localFile.getName();
String localPath = localFile.getAbsolutePath();
boolean overWrite = true;
boolean delSrc = true;
- boolean isCopied = LogfeederHDFSUtil.INSTANCE.copyFromLocal(
- localFile.getAbsolutePath(), destFilePath, fileSystem,
+ boolean isCopied = LogfeederHDFSUtil.copyFromLocal(localFile.getAbsolutePath(), destFilePath, fileSystem,
overWrite, delSrc);
if (isCopied) {
- logger.debug("File copy to hdfs hdfspath :" + destFilePath
- + " and deleted local file :" + localPath);
+ LOG.debug("File copy to hdfs hdfspath :" + destFilePath + " and deleted local file :" + localPath);
} else {
- // TODO Need to write retry logic, in next release we can
- // handle it
- logger.error("Hdfs file copy failed for hdfspath :"
- + destFilePath + " and localpath :" + localPath);
+ // TODO Need to write retry logic, in next release we can handle it
+ LOG.error("Hdfs file copy failed for hdfspath :" + destFilePath + " and localpath :" + localPath);
}
}
localFileIterator.remove();
@@ -157,14 +149,11 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
}
}
} catch (InterruptedException e) {
- logger.error(e.getLocalizedMessage(),e);
+ LOG.error(e.getLocalizedMessage(),e);
}
}
} catch (Exception e) {
- logger
- .error(
- "Exception in hdfsCopyThread errorMsg:"
- + e.getLocalizedMessage(), e);
+ LOG.error("Exception in hdfsCopyThread errorMsg:" + e.getLocalizedMessage(), e);
}
}
};
@@ -174,24 +163,23 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
private void stopHDFSCopyThread() {
if (hdfsCopyThread != null) {
- logger.info("waiting till copy all local files to hdfs.......");
+ LOG.info("waiting till copy all local files to hdfs.......");
while (!localReadyFiles.isEmpty()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
- logger.error(e.getLocalizedMessage(), e);
+ LOG.error(e.getLocalizedMessage(), e);
}
- logger.debug("still waiting to copy all local files to hdfs.......");
+ LOG.debug("still waiting to copy all local files to hdfs.......");
}
- logger.info("calling interrupt method for hdfsCopyThread to stop it.");
+ LOG.info("calling interrupt method for hdfsCopyThread to stop it.");
try {
hdfsCopyThread.interrupt();
} catch (SecurityException exception) {
- logger.error(" Current thread : '" + Thread.currentThread().getName()
- + "' does not have permission to interrupt the Thread: '"
- + hdfsCopyThread.getName() + "'");
+ LOG.error(" Current thread : '" + Thread.currentThread().getName() +
+ "' does not have permission to interrupt the Thread: '" + hdfsCopyThread.getName() + "'");
}
- LogfeederHDFSUtil.INSTANCE.closeFileSystem(fileSystem);
+ LogfeederHDFSUtil.closeFileSystem(fileSystem);
}
}
@@ -208,15 +196,13 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
readyMonitor.notifyAll();
}
} catch (Exception e) {
- logger.error(e.getLocalizedMessage(),e);
+ LOG.error(e.getLocalizedMessage(),e);
}
}
@Override
- public void copyFile(File inputFile, InputMarker inputMarker)
- throws UnsupportedOperationException {
- throw new UnsupportedOperationException(
- "copyFile method is not yet supported for output=hdfs");
+ public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("copyFile method is not yet supported for output=hdfs");
}
/**
@@ -242,8 +228,8 @@ public class OutputHDFSFile extends Output implements RolloverHandler, RolloverC
long timeSinceCreation = new Date().getTime() - currentSpoolerContext.getActiveLogCreationTime().getTime();
boolean shouldRollover = timeSinceCreation > rolloverThresholdTimeMillis;
if (shouldRollover) {
- logger.info("Detecting that time since file creation time " + currentSpoolerContext.getActiveLogCreationTime() +
- " has crossed threshold (msecs) " + rolloverThresholdTimeMillis);
+ LOG.info("Detecting that time since file creation time " + currentSpoolerContext.getActiveLogCreationTime() +
+ " has crossed threshold (msecs) " + rolloverThresholdTimeMillis);
}
return shouldRollover;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
index 2595d87..52fc6f8 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java
@@ -56,6 +56,16 @@ public class OutputKafka extends Output {
private boolean isKafkaBrokerUp = false;
@Override
+ protected String getStatMetricName() {
+ return "output.kafka.write_logs";
+ }
+
+ @Override
+ protected String getWriteBytesMetricName() {
+ return "output.kafka.write_bytes";
+ }
+
+ @Override
public void init() throws Exception {
super.init();
Properties props = initProperties();
@@ -65,9 +75,6 @@ public class OutputKafka extends Output {
}
private Properties initProperties() throws Exception {
- statMetric.metricsName = "output.kafka.write_logs";
- writeBytesMetric.metricsName = "output.kafka.write_bytes";
-
String brokerList = getStringValue("broker_list");
if (StringUtils.isEmpty(brokerList)) {
throw new Exception("For kafka output, bootstrap broker_list is needed");
@@ -124,17 +131,15 @@ public class OutputKafka extends Output {
if (publishMessage(kafkaCallBack.message, kafkaCallBack.inputMarker)) {
kafkaCallBack = null;
} else {
- LOG.error("Kafka is down. messageNumber=" + kafkaCallBack.thisMessageNumber + ". Going to sleep for "
- + FAILED_RETRY_INTERVAL + " seconds");
+ LOG.error("Kafka is down. messageNumber=" + kafkaCallBack.thisMessageNumber + ". Going to sleep for " +
+ FAILED_RETRY_INTERVAL + " seconds");
Thread.sleep(FAILED_RETRY_INTERVAL * 1000);
}
} catch (Throwable t) {
String logMessageKey = this.getClass().getSimpleName() + "_KAFKA_RETRY_WRITE_ERROR";
- LogFeederUtil.logErrorMessageByInterval(logMessageKey,
- "Error sending message to Kafka during retry. message="
- + (kafkaCallBack == null ? null : kafkaCallBack.message),
- t, LOG, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Error sending message to Kafka during retry. message=" +
+ (kafkaCallBack == null ? null : kafkaCallBack.message), t, LOG, Level.ERROR);
}
}
@@ -160,8 +165,8 @@ public class OutputKafka extends Output {
LOG.error("Kafka is down. Going to sleep for " + FAILED_RETRY_INTERVAL + " seconds");
Thread.sleep(FAILED_RETRY_INTERVAL * 1000);
} else {
- LOG.warn("Kafka is still catching up from previous failed messages. outstanding messages="
- + failedMessages.size() + " Going to sleep for " + CATCHUP_RETRY_INTERVAL + " seconds");
+ LOG.warn("Kafka is still catching up from previous failed messages. outstanding messages=" + failedMessages.size() +
+ " Going to sleep for " + CATCHUP_RETRY_INTERVAL + " seconds");
Thread.sleep(CATCHUP_RETRY_INTERVAL * 1000);
}
} catch (Throwable t) {
@@ -198,16 +203,15 @@ public class OutputKafka extends Output {
private boolean publishMessage(String block, InputMarker inputMarker) {
if (isAsync && isKafkaBrokerUp) { // Send asynchronously
- producer.send(new ProducerRecord<String, String>(topic, block),
- new KafkaCallBack(this, block, inputMarker, ++messageCount));
+ producer.send(new ProducerRecord<String, String>(topic, block), new KafkaCallBack(this, block, inputMarker, ++messageCount));
return true;
} else { // Send synchronously
try {
// Not using key. Let it round robin
RecordMetadata metadata = producer.send(new ProducerRecord<String, String>(topic, block)).get();
if (metadata != null) {
- statMetric.count++;
- writeBytesMetric.count += block.length();
+ statMetric.value++;
+ writeBytesMetric.value += block.length();
}
if (!isKafkaBrokerUp) {
LOG.info("Started writing to kafka. " + getShortDescription());
@@ -217,18 +221,18 @@ public class OutputKafka extends Output {
} catch (InterruptedException e) {
isKafkaBrokerUp = false;
String logKeyMessage = this.getClass().getSimpleName() + "_KAFKA_INTERRUPT";
- LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "InterruptedException-Error sending message to Kafka", e,
- LOG, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "InterruptedException-Error sending message to Kafka", e, LOG,
+ Level.ERROR);
} catch (ExecutionException e) {
isKafkaBrokerUp = false;
String logKeyMessage = this.getClass().getSimpleName() + "_KAFKA_EXECUTION";
- LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "ExecutionException-Error sending message to Kafka", e,
- LOG, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "ExecutionException-Error sending message to Kafka", e, LOG,
+ Level.ERROR);
} catch (Throwable t) {
isKafkaBrokerUp = false;
String logKeyMessage = this.getClass().getSimpleName() + "_KAFKA_WRITE_ERROR";
- LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "GenericException-Error sending message to Kafka", t,
- LOG, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "GenericException-Error sending message to Kafka", t, LOG,
+ Level.ERROR);
}
}
return false;
@@ -260,12 +264,12 @@ public class OutputKafka extends Output {
output.isKafkaBrokerUp = true;
}
output.incrementStat(1);
- output.writeBytesMetric.count += message.length();
+ output.writeBytesMetric.value += message.length();
} else {
output.isKafkaBrokerUp = false;
String logKeyMessage = this.getClass().getSimpleName() + "_KAFKA_ASYNC_ERROR";
- LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "Error sending message to Kafka. Async Callback",
- exception, LOG, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(logKeyMessage, "Error sending message to Kafka. Async Callback", exception, LOG,
+ Level.ERROR);
output.failedMessages.add(this);
}
@@ -273,9 +277,7 @@ public class OutputKafka extends Output {
}
@Override
- public void copyFile(File inputFile, InputMarker inputMarker)
- throws UnsupportedOperationException {
- throw new UnsupportedOperationException(
- "copyFile method is not yet supported for output=kafka");
+ public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("copyFile method is not yet supported for output=kafka");
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
new file mode 100644
index 0000000..2c81c19
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java
@@ -0,0 +1,250 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logfeeder.output;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
+import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.logconfig.FilterLogData;
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.ambari.logfeeder.util.MurmurHash;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public class OutputManager {
+ private static final Logger LOG = Logger.getLogger(OutputManager.class);
+
+ private static final int HASH_SEED = 31174077;
+ private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1
+
+ private List<Output> outputs = new ArrayList<Output>();
+
+ private boolean addMessageMD5 = true;
+
+ private static long docCounter = 0;
+ private MetricData messageTruncateMetric = new MetricData(null, false);
+
+ public List<Output> getOutputs() {
+ return outputs;
+ }
+
+ public void add(Output output) {
+ this.outputs.add(output);
+ }
+
+ public void retainUsedOutputs(Collection<Output> usedOutputs) {
+ outputs.retainAll(usedOutputs);
+ }
+
+ public void init() throws Exception {
+ for (Output output : outputs) {
+ output.init();
+ }
+ }
+
+ public void write(Map<String, Object> jsonObj, InputMarker inputMarker) {
+ Input input = inputMarker.input;
+
+ // Update the block with the context fields
+ for (Map.Entry<String, String> entry : input.getContextFields().entrySet()) {
+ if (jsonObj.get(entry.getKey()) == null) {
+ jsonObj.put(entry.getKey(), entry.getValue());
+ }
+ }
+
+ // TODO: Ideally most of the overrides should be configurable
+
+ if (jsonObj.get("type") == null) {
+ jsonObj.put("type", input.getStringValue("type"));
+ }
+ if (jsonObj.get("path") == null && input.getFilePath() != null) {
+ jsonObj.put("path", input.getFilePath());
+ }
+ if (jsonObj.get("path") == null && input.getStringValue("path") != null) {
+ jsonObj.put("path", input.getStringValue("path"));
+ }
+ if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) {
+ jsonObj.put("host", LogFeederUtil.hostName);
+ }
+ if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) {
+ jsonObj.put("ip", LogFeederUtil.ipAddress);
+ }
+ if (jsonObj.get("level") == null) {
+ jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
+ }
+
+ if (input.isUseEventMD5() || input.isGenEventMD5()) {
+ String prefix = "";
+ Object logtimeObj = jsonObj.get("logtime");
+ if (logtimeObj != null) {
+ if (logtimeObj instanceof Date) {
+ prefix = "" + ((Date) logtimeObj).getTime();
+ } else {
+ prefix = logtimeObj.toString();
+ }
+ }
+
+ Long eventMD5 = MurmurHash.hash64A(LogFeederUtil.getGson().toJson(jsonObj).getBytes(), HASH_SEED);
+ if (input.isGenEventMD5()) {
+ jsonObj.put("event_md5", prefix + eventMD5.toString());
+ }
+ if (input.isUseEventMD5()) {
+ jsonObj.put("id", prefix + eventMD5.toString());
+ }
+ }
+
+ jsonObj.put("seq_num", new Long(docCounter++));
+ if (jsonObj.get("id") == null) {
+ jsonObj.put("id", UUID.randomUUID().toString());
+ }
+ if (jsonObj.get("event_count") == null) {
+ jsonObj.put("event_count", new Integer(1));
+ }
+ if (inputMarker.lineNumber > 0) {
+ jsonObj.put("logfile_line_number", new Integer(inputMarker.lineNumber));
+ }
+ if (jsonObj.containsKey("log_message")) {
+ // TODO: Let's check size only for log_message for now
+ String logMessage = (String) jsonObj.get("log_message");
+ logMessage = truncateLongLogMessage(jsonObj, input, logMessage);
+ if (addMessageMD5) {
+ jsonObj.put("message_md5", "" + MurmurHash.hash64A(logMessage.getBytes(), 31174077));
+ }
+ }
+
+ if (FilterLogData.INSTANCE.isAllowed(jsonObj)) {
+ for (Output output : input.getOutputList()) {
+ try {
+ output.write(jsonObj, inputMarker);
+ } catch (Exception e) {
+ LOG.error("Error writing. to " + output.getShortDescription(), e);
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private String truncateLongLogMessage(Map<String, Object> jsonObj, Input input, String logMessage) {
+ if (logMessage != null && logMessage.getBytes().length > MAX_OUTPUT_SIZE) {
+ messageTruncateMetric.value++;
+ String logMessageKey = this.getClass().getSimpleName() + "_MESSAGESIZE";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Message is too big. size=" + logMessage.getBytes().length +
+ ", input=" + input.getShortDescription() + ". Truncating to " + MAX_OUTPUT_SIZE + ", first upto 100 characters=" +
+ StringUtils.abbreviate(logMessage, 100), null, LOG, Level.WARN);
+ logMessage = new String(logMessage.getBytes(), 0, MAX_OUTPUT_SIZE);
+ jsonObj.put("log_message", logMessage);
+ List<String> tagsList = (List<String>) jsonObj.get("tags");
+ if (tagsList == null) {
+ tagsList = new ArrayList<String>();
+ jsonObj.put("tags", tagsList);
+ }
+ tagsList.add("error_message_truncated");
+ }
+ return logMessage;
+ }
+
+ public void write(String jsonBlock, InputMarker inputMarker) {
+ if (FilterLogData.INSTANCE.isAllowed(jsonBlock)) {
+ for (Output output : inputMarker.input.getOutputList()) {
+ try {
+ output.write(jsonBlock, inputMarker);
+ } catch (Exception e) {
+ LOG.error("Error writing. to " + output.getShortDescription(), e);
+ }
+ }
+ }
+ }
+
+ public void copyFile(File inputFile, InputMarker inputMarker) {
+ Input input = inputMarker.input;
+ for (Output output : input.getOutputList()) {
+ try {
+ output.copyFile(inputFile, inputMarker);
+ }catch (Exception e) {
+ LOG.error("Error coyping file . to " + output.getShortDescription(), e);
+ }
+ }
+ }
+
+ public void logStats() {
+ for (Output output : outputs) {
+ output.logStat();
+ }
+ LogFeederUtil.logStatForMetric(messageTruncateMetric, "Stat: Messages Truncated", "");
+ }
+
+ public void addMetricsContainers(List<MetricData> metricsList) {
+ metricsList.add(messageTruncateMetric);
+ for (Output output : outputs) {
+ output.addMetricsContainers(metricsList);
+ }
+ }
+
+ public void close() {
+ LOG.info("Close called for outputs ...");
+ for (Output output : outputs) {
+ try {
+ output.setDrain(true);
+ output.close();
+ } catch (Exception e) {
+ // Ignore
+ }
+ }
+
+ // Need to get this value from property
+ int iterations = 30;
+ int waitTimeMS = 1000;
+ for (int i = 0; i < iterations; i++) {
+ boolean allClosed = true;
+ for (Output output : outputs) {
+ if (!output.isClosed()) {
+ try {
+ allClosed = false;
+ LOG.warn("Waiting for output to close. " + output.getShortDescription() + ", " + (iterations - i) + " more seconds");
+ Thread.sleep(waitTimeMS);
+ } catch (Throwable t) {
+ // Ignore
+ }
+ }
+ }
+ if (allClosed) {
+ LOG.info("All outputs are closed. Iterations=" + i);
+ return;
+ }
+ }
+
+ LOG.warn("Some outpus were not closed after " + iterations + " iterations");
+ for (Output output : outputs) {
+ if (!output.isClosed()) {
+ LOG.warn("Output not closed. Will ignore it." + output.getShortDescription() + ", pendingCound=" + output.getPendingCount());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputMgr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputMgr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputMgr.java
deleted file mode 100644
index 0a6b7fa..0000000
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputMgr.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.ambari.logfeeder.output;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.ambari.logfeeder.input.Input;
-import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.logconfig.LogFeederConstants;
-import org.apache.ambari.logfeeder.logconfig.filter.FilterLogData;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-
-public class OutputMgr {
- private static final Logger logger = Logger.getLogger(OutputMgr.class);
-
- private Collection<Output> outputList = new ArrayList<Output>();
-
- private boolean addMessageMD5 = true;
-
- private int MAX_OUTPUT_SIZE = 32765; // 32766-1
- private static long doc_counter = 0;
- private MetricCount messageTruncateMetric = new MetricCount();
-
-
- public Collection<Output> getOutputList() {
- return outputList;
- }
-
- public void setOutputList(Collection<Output> outputList) {
- this.outputList = outputList;
- }
-
- public void write(Map<String, Object> jsonObj, InputMarker inputMarker) {
- Input input = inputMarker.input;
-
- // Update the block with the context fields
- for (Map.Entry<String, String> entry : input.getContextFields()
- .entrySet()) {
- if (jsonObj.get(entry.getKey()) == null) {
- jsonObj.put(entry.getKey(), entry.getValue());
- }
- }
-
- // TODO: Ideally most of the overrides should be configurable
-
- // Add the input type
- if (jsonObj.get("type") == null) {
- jsonObj.put("type", input.getStringValue("type"));
- }
- if (jsonObj.get("path") == null && input.getFilePath() != null) {
- jsonObj.put("path", input.getFilePath());
- }
- if (jsonObj.get("path") == null && input.getStringValue("path") != null) {
- jsonObj.put("path", input.getStringValue("path"));
- }
-
- // Add host if required
- if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) {
- jsonObj.put("host", LogFeederUtil.hostName);
- }
- // Add IP if required
- if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) {
- jsonObj.put("ip", LogFeederUtil.ipAddress);
- }
-
- //Add level
- if (jsonObj.get("level") == null) {
- jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN);
- }
- if (input.isUseEventMD5() || input.isGenEventMD5()) {
- String prefix = "";
- Object logtimeObj = jsonObj.get("logtime");
- if (logtimeObj != null) {
- if (logtimeObj instanceof Date) {
- prefix = "" + ((Date) logtimeObj).getTime();
- } else {
- prefix = logtimeObj.toString();
- }
- }
- Long eventMD5 = LogFeederUtil.genHash(LogFeederUtil.getGson()
- .toJson(jsonObj));
- if (input.isGenEventMD5()) {
- jsonObj.put("event_md5", prefix + eventMD5.toString());
- }
- if (input.isUseEventMD5()) {
- jsonObj.put("id", prefix + eventMD5.toString());
- }
- }
-
- // jsonObj.put("@timestamp", new Date());
- jsonObj.put("seq_num", new Long(doc_counter++));
- if (jsonObj.get("id") == null) {
- jsonObj.put("id", UUID.randomUUID().toString());
- }
- if (jsonObj.get("event_count") == null) {
- jsonObj.put("event_count", new Integer(1));
- }
- if (inputMarker.lineNumber > 0) {
- jsonObj.put("logfile_line_number", new Integer(
- inputMarker.lineNumber));
- }
- if (jsonObj.containsKey("log_message")) {
- // TODO: Let's check size only for log_message for now
- String logMessage = (String) jsonObj.get("log_message");
- if (logMessage != null
- && logMessage.getBytes().length > MAX_OUTPUT_SIZE) {
- messageTruncateMetric.count++;
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_MESSAGESIZE";
- LogFeederUtil.logErrorMessageByInterval(LOG_MESSAGE_KEY,
- "Message is too big. size="
- + logMessage.getBytes().length + ", input="
- + input.getShortDescription()
- + ". Truncating to " + MAX_OUTPUT_SIZE
- + ", first upto 100 characters="
- + LogFeederUtil.subString(logMessage, 100),
- null, logger, Level.WARN);
- logMessage = new String(logMessage.getBytes(), 0,
- MAX_OUTPUT_SIZE);
- jsonObj.put("log_message", logMessage);
- // Add error tags
- @SuppressWarnings("unchecked")
- List<String> tagsList = (List<String>) jsonObj.get("tags");
- if (tagsList == null) {
- tagsList = new ArrayList<String>();
- jsonObj.put("tags", tagsList);
- }
- tagsList.add("error_message_truncated");
-
- }
- if (addMessageMD5) {
- jsonObj.put("message_md5",
- "" + LogFeederUtil.genHash(logMessage));
- }
- }
- //check log is allowed to send output
- if (FilterLogData.INSTANCE.isAllowed(jsonObj)) {
- for (Output output : input.getOutputList()) {
- try {
- output.write(jsonObj, inputMarker);
- } catch (Exception e) {
- logger.error("Error writing. to " + output.getShortDescription(), e);
- }
- }
- }
- }
-
- public void write(String jsonBlock, InputMarker inputMarker) {
- //check log is allowed to send output
- if (FilterLogData.INSTANCE.isAllowed(jsonBlock)) {
- for (Output output : inputMarker.input.getOutputList()) {
- try {
- output.write(jsonBlock, inputMarker);
- } catch (Exception e) {
- logger.error("Error writing. to " + output.getShortDescription(), e);
- }
- }
- }
- }
-
- public void close() {
- logger.info("Close called for outputs ...");
- for (Output output : outputList) {
- try {
- output.setDrain(true);
- output.close();
- } catch (Exception e) {
- // Ignore
- }
- }
- // Need to get this value from property
- int iterations = 30;
- int waitTimeMS = 1000;
- int i;
- boolean allClosed = true;
- for (i = 0; i < iterations; i++) {
- allClosed = true;
- for (Output output : outputList) {
- if (!output.isClosed()) {
- try {
- allClosed = false;
- logger.warn("Waiting for output to close. "
- + output.getShortDescription() + ", "
- + (iterations - i) + " more seconds");
- Thread.sleep(waitTimeMS);
- } catch (Throwable t) {
- // Ignore
- }
- }
- }
- if (allClosed) {
- break;
- }
- }
-
- if (!allClosed) {
- logger.warn("Some outpus were not closed. Iterations=" + i);
- for (Output output : outputList) {
- if (!output.isClosed()) {
- logger.warn("Output not closed. Will ignore it."
- + output.getShortDescription() + ", pendingCound="
- + output.getPendingCount());
- }
- }
- } else {
- logger.info("All outputs are closed. Iterations=" + i);
- }
- }
-
- public void logStats() {
- for (Output output : outputList) {
- output.logStat();
- }
- LogFeederUtil.logStatForMetric(messageTruncateMetric,
- "Stat: Messages Truncated", null);
- }
-
- public void addMetricsContainers(List<MetricCount> metricsList) {
- metricsList.add(messageTruncateMetric);
- for (Output output : outputList) {
- output.addMetricsContainers(metricsList);
- }
- }
-
-
- public void copyFile(File inputFile, InputMarker inputMarker) {
- Input input = inputMarker.input;
- for (Output output : input.getOutputList()) {
- try {
- output.copyFile(inputFile, inputMarker);
- }catch (Exception e) {
- logger.error("Error coyping file . to " + output.getShortDescription(),
- e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
index e95f8df..26f1ddb 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java
@@ -22,6 +22,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.ambari.logfeeder.LogFeeder;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.output.spool.LogSpooler;
@@ -47,10 +48,10 @@ import java.util.Map.Entry;
* </ul>
*/
public class OutputS3File extends Output implements RolloverCondition, RolloverHandler {
+ private static final Logger LOG = Logger.getLogger(OutputS3File.class);
public static final String INPUT_ATTRIBUTE_TYPE = "type";
public static final String GLOBAL_CONFIG_S3_PATH_SUFFIX = "global.config.json";
- static private Logger logger = Logger.getLogger(OutputS3File.class);
private LogSpooler logSpooler;
private S3OutputConfiguration s3OutputConfiguration;
@@ -72,23 +73,21 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
@Override
public void copyFile(File inputFile, InputMarker inputMarker) {
String type = inputMarker.input.getStringValue(INPUT_ATTRIBUTE_TYPE);
- S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration,
- S3Util.INSTANCE, false, type);
- String resolvedPath = s3Uploader.uploadFile(inputFile,
- inputMarker.input.getStringValue(INPUT_ATTRIBUTE_TYPE));
+ S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, false, type);
+ String resolvedPath = s3Uploader.uploadFile(inputFile, inputMarker.input.getStringValue(INPUT_ATTRIBUTE_TYPE));
uploadConfig(inputMarker, type, s3OutputConfiguration, resolvedPath);
}
- private void uploadConfig(InputMarker inputMarker, String type,
- S3OutputConfiguration s3OutputConfiguration, String resolvedPath) {
+ private void uploadConfig(InputMarker inputMarker, String type, S3OutputConfiguration s3OutputConfiguration,
+ String resolvedPath) {
ArrayList<Map<String, Object>> filters = new ArrayList<>();
addFilters(filters, inputMarker.input.getFirstFilter());
Map<String, Object> inputConfig = new HashMap<>();
inputConfig.putAll(inputMarker.input.getConfigs());
- String s3CompletePath = S3Util.S3_PATH_START_WITH + s3OutputConfiguration.getS3BucketName()
- + S3Util.S3_PATH_SEPARATOR + resolvedPath;
+ String s3CompletePath = LogFeederConstants.S3_PATH_START_WITH + s3OutputConfiguration.getS3BucketName() +
+ LogFeederConstants.S3_PATH_SEPARATOR + resolvedPath;
inputConfig.put("path", s3CompletePath);
ArrayList<Map<String, Object>> inputConfigList = new ArrayList<>();
@@ -117,17 +116,15 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
}
}
- private void writeConfigToS3(Map<String, Object> configToWrite, String s3KeySuffix,
- S3OutputConfiguration s3OutputConfiguration) {
+ private void writeConfigToS3(Map<String, Object> configToWrite, String s3KeySuffix, S3OutputConfiguration s3OutputConfiguration) {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
String configJson = gson.toJson(configToWrite);
- String s3ResolvedKey = new S3LogPathResolver().
- getResolvedPath(getStringValue("s3_config_dir"), s3KeySuffix, s3OutputConfiguration.getCluster());
+ String s3ResolvedKey = new S3LogPathResolver().getResolvedPath(getStringValue("s3_config_dir"), s3KeySuffix,
+ s3OutputConfiguration.getCluster());
- S3Util.INSTANCE.writeIntoS3File(configJson, s3OutputConfiguration.getS3BucketName(),
- s3ResolvedKey, s3OutputConfiguration.getS3AccessKey(),
- s3OutputConfiguration.getS3SecretKey());
+ S3Util.writeIntoS3File(configJson, s3OutputConfiguration.getS3BucketName(), s3ResolvedKey,
+ s3OutputConfiguration.getS3AccessKey(), s3OutputConfiguration.getS3SecretKey());
}
private String getComponentConfigFileName(String componentName) {
@@ -136,7 +133,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
private Map<String, Object> getGlobalConfig() {
- Map<String, Object> globalConfig = LogFeeder.globalMap;
+ Map<String, Object> globalConfig = LogFeeder.globalConfigs;
if (globalConfig == null) {
globalConfig = new HashMap<>();
}
@@ -173,8 +170,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
globalConfig.put("copy_file", false);
globalConfig.put("process_file", true);
globalConfig.put("tail", false);
- Map<String, Object> addFields = (Map<String, Object>) globalConfig
- .get("add_fields");
+ Map<String, Object> addFields = (Map<String, Object>) globalConfig.get("add_fields");
if (addFields == null) {
addFields = new HashMap<>();
}
@@ -216,7 +212,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
@VisibleForTesting
protected S3Uploader createUploader(String logType) {
- S3Uploader uploader = new S3Uploader(s3OutputConfiguration, S3Util.INSTANCE, true, logType);
+ S3Uploader uploader = new S3Uploader(s3OutputConfiguration, true, logType);
uploader.startUploaderThread();
return uploader;
}
@@ -224,8 +220,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
@VisibleForTesting
protected LogSpooler createSpooler(String filePath) {
String spoolDirectory = LogFeederUtil.getLogfeederTempDir() + "/s3/service";
- logger.info(String.format("Creating spooler with spoolDirectory=%s, filePath=%s",
- spoolDirectory, filePath));
+ LOG.info(String.format("Creating spooler with spoolDirectory=%s, filePath=%s", spoolDirectory, filePath));
return new LogSpooler(spoolDirectory, new File(filePath).getName()+"-", this, this,
s3OutputConfiguration.getRolloverTimeThresholdSecs());
}
@@ -244,7 +239,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH
long currentSize = spoolFile.length();
boolean result = (currentSize >= s3OutputConfiguration.getRolloverSizeThresholdBytes());
if (result) {
- logger.info(String.format("Rolling over %s, current size %d, threshold size %d", spoolFile, currentSize,
+ LOG.info(String.format("Rolling over %s, current size %d, threshold size %d", spoolFile, currentSize,
s3OutputConfiguration.getRolloverSizeThresholdBytes()));
}
return result;
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index cd9ce4d..47f139d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -34,7 +34,8 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.logconfig.FetchConfigFromSolr;
+import org.apache.ambari.logfeeder.logconfig.LogConfigHandler;
+import org.apache.ambari.logfeeder.util.DateUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
@@ -76,7 +77,17 @@ public class OutputSolr extends Output {
private BlockingQueue<OutputData> outgoingBuffer = null;
private List<SolrWorkerThread> workerThreadList = new ArrayList<>();
-
+
+ @Override
+ protected String getStatMetricName() {
+ return "output.solr.write_logs";
+ }
+
+ @Override
+ protected String getWriteBytesMetricName() {
+ return "output.solr.write_bytes";
+ }
+
@Override
public void init() throws Exception {
super.init();
@@ -87,9 +98,6 @@ public class OutputSolr extends Output {
}
private void initParams() throws Exception {
- statMetric.metricsName = "output.solr.write_logs";
- writeBytesMetric.metricsName = "output.solr.write_bytes";
-
splitMode = getStringValue("splits_interval_mins", "none");
if (!splitMode.equalsIgnoreCase("none")) {
splitInterval = getIntValue("split_interval_mins", DEFAULT_SPLIT_INTERVAL);
@@ -204,10 +212,8 @@ public class OutputSolr extends Output {
LOG.info("Ping to Solr server is successful for worker=" + count);
} else {
LOG.warn(
- String.format(
- "Ping to Solr server failed. It would check again. worker=%d, "
- + "solrUrl=%s, zkConnectString=%s, collection=%s, response=%s",
- count, solrUrl, zkConnectString, collection, response));
+ String.format("Ping to Solr server failed. It would check again. worker=%d, solrUrl=%s, zkConnectString=%s, " +
+ "collection=%s, response=%s", count, solrUrl, zkConnectString, collection, response));
}
} catch (Throwable t) {
LOG.warn(String.format(
@@ -223,7 +229,7 @@ public class OutputSolr extends Output {
while (true) {
LOG.info("Checking if config is available");
- if (FetchConfigFromSolr.isFilterAvailable()) {
+ if (LogConfigHandler.isFilterAvailable()) {
LOG.info("Config is available");
return;
}
@@ -256,7 +262,7 @@ public class OutputSolr extends Output {
private void useActualDateIfNeeded(Map<String, Object> jsonObj) {
if (skipLogtime) {
- jsonObj.put("logtime", LogFeederUtil.getActualDateStr());
+ jsonObj.put("logtime", DateUtil.getActualDateStr());
}
}
@@ -324,7 +330,7 @@ public class OutputSolr extends Output {
private final SolrClient solrClient;
private final Collection<SolrInputDocument> localBuffer = new ArrayList<>();
- private final Map<String, InputMarker> latestInputMarkerList = new HashMap<>();
+ private final Map<String, InputMarker> latestInputMarkers = new HashMap<>();
private long localBufferBytesSize = 0;
@@ -352,17 +358,16 @@ public class OutputSolr extends Output {
}
}
- if (localBuffer.size() > 0 && ((outputData == null && isDrain())
- || (nextDispatchDuration <= 0 || localBuffer.size() >= maxBufferSize))) {
+ if (localBuffer.size() > 0 && ((outputData == null && isDrain()) ||
+ (nextDispatchDuration <= 0 || localBuffer.size() >= maxBufferSize))) {
boolean response = sendToSolr(outputData);
if( isDrain() && !response) {
//Since sending to Solr response failed and it is in draining mode, let's break;
- LOG.warn("In drain mode and sending to Solr failed. So exiting. output="
- + getShortDescription());
+ LOG.warn("In drain mode and sending to Solr failed. So exiting. output=" + getShortDescription());
break;
}
}
- if( localBuffer.size() == 0 ) {
+ if (localBuffer.size() == 0) {
//If localBuffer is empty, then reset the timer
lastDispatchTime = currTimeMS;
}
@@ -403,8 +408,7 @@ public class OutputSolr extends Output {
} catch (IOException | SolrException exception) {
// Transient error, lets block till it is available
try {
- LOG.warn("Solr is not reachable. Going to retry after "
- + RETRY_INTERVAL + " seconds. " + "output="
+ LOG.warn("Solr is not reachable. Going to retry after " + RETRY_INTERVAL + " seconds. " + "output="
+ getShortDescription(), exception);
Thread.sleep(RETRY_INTERVAL * 1000);
} catch (Throwable t) {
@@ -414,8 +418,8 @@ public class OutputSolr extends Output {
// Something unknown happened. Let's not block because of this error.
// Clear the buffer
String logMessageKey = this.getClass().getSimpleName() + "_SOLR_UPDATE_EXCEPTION";
- LogFeederUtil.logErrorMessageByInterval(logMessageKey,
- "Error sending log message to server. Dropping logs", serverException, LOG, Level.ERROR);
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Error sending log message to server. Dropping logs",
+ serverException, LOG, Level.ERROR);
resetLocalBuffer();
break;
}
@@ -447,7 +451,7 @@ public class OutputSolr extends Output {
Level.ERROR);
}
}
- latestInputMarkerList.put(outputData.inputMarker.base64FileKey, outputData.inputMarker);
+ latestInputMarkers.put(outputData.inputMarker.base64FileKey, outputData.inputMarker);
localBuffer.add(document);
}
@@ -479,9 +483,9 @@ public class OutputSolr extends Output {
LogFeederUtil.logErrorMessageByInterval(logMessageKey,
String.format("Error writing to Solr. response=%s, log=%s", response, outputData), null, LOG, Level.ERROR);
}
- statMetric.count += localBuffer.size();
- writeBytesMetric.count += localBufferBytesSize;
- for (InputMarker inputMarker : latestInputMarkerList.values()) {
+ statMetric.value += localBuffer.size();
+ writeBytesMetric.value += localBufferBytesSize;
+ for (InputMarker inputMarker : latestInputMarkers.values()) {
inputMarker.input.checkIn(inputMarker);
}
}
@@ -499,7 +503,7 @@ public class OutputSolr extends Output {
public void resetLocalBuffer() {
localBuffer.clear();
localBufferBytesSize = 0;
- latestInputMarkerList.clear();
+ latestInputMarkers.clear();
}
public boolean isDone() {
@@ -512,9 +516,7 @@ public class OutputSolr extends Output {
}
@Override
- public void copyFile(File inputFile, InputMarker inputMarker)
- throws UnsupportedOperationException {
- throw new UnsupportedOperationException(
- "copyFile method is not yet supported for output=solr");
+ public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
+ throw new UnsupportedOperationException("copyFile method is not yet supported for output=solr");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3LogPathResolver.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3LogPathResolver.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3LogPathResolver.java
index 58282e0..8c544cf 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3LogPathResolver.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3LogPathResolver.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,9 +18,9 @@
package org.apache.ambari.logfeeder.output;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.ambari.logfeeder.util.PlaceholderUtil;
-import org.apache.ambari.logfeeder.util.S3Util;
import java.util.HashMap;
@@ -40,7 +40,7 @@ public class S3LogPathResolver {
public String getResolvedPath(String baseKeyPrefix, String keySuffix, String cluster) {
HashMap<String, String> contextParam = buildContextParam(cluster);
String resolvedKeyPrefix = PlaceholderUtil.replaceVariables(baseKeyPrefix, contextParam);
- return resolvedKeyPrefix + S3Util.S3_PATH_SEPARATOR + keySuffix;
+ return resolvedKeyPrefix + LogFeederConstants.S3_PATH_SEPARATOR + keySuffix;
}
private HashMap<String, String> buildContextParam(String cluster) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3OutputConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3OutputConfiguration.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3OutputConfiguration.java
index 485b0d4..e5974c5 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3OutputConfiguration.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3OutputConfiguration.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -103,8 +103,7 @@ public class S3OutputConfiguration {
};
for (int i = 0; i < longValuedKeysToCopy.length; i++) {
- configs.put(longValuedKeysToCopy[i],
- configBlock.getLongValue(longValuedKeysToCopy[i], defaultValuesForLongValuedKeys[i]));
+ configs.put(longValuedKeysToCopy[i], configBlock.getLongValue(longValuedKeysToCopy[i], defaultValuesForLongValuedKeys[i]));
}
configs.put(ADDITIONAL_FIELDS_KEY, configBlock.getNVList(ADDITIONAL_FIELDS_KEY));
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3Uploader.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3Uploader.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3Uploader.java
index fd59c51..e95a663 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3Uploader.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/S3Uploader.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,8 +18,12 @@
package org.apache.ambari.logfeeder.output;
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.Upload;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.util.CompressionUtil;
import org.apache.ambari.logfeeder.util.S3Util;
import org.apache.log4j.Logger;
@@ -39,20 +43,18 @@ import java.util.concurrent.atomic.AtomicBoolean;
* {@link org.apache.ambari.logfeeder.input.InputFile}.
*/
public class S3Uploader implements Runnable {
+ private static final Logger LOG = Logger.getLogger(S3Uploader.class);
+
public static final String POISON_PILL = "POISON-PILL";
- private static Logger logger = Logger.getLogger(S3Uploader.class);
private final S3OutputConfiguration s3OutputConfiguration;
- private final S3Util s3UtilInstance;
private final boolean deleteOnEnd;
- private String logType;
+ private final String logType;
private final BlockingQueue<String> fileContextsToUpload;
- private AtomicBoolean stopRunningThread = new AtomicBoolean(false);
+ private final AtomicBoolean stopRunningThread = new AtomicBoolean(false);
- public S3Uploader(S3OutputConfiguration s3OutputConfiguration, S3Util s3UtilInstance, boolean deleteOnEnd,
- String logType) {
+ public S3Uploader(S3OutputConfiguration s3OutputConfiguration, boolean deleteOnEnd, String logType) {
this.s3OutputConfiguration = s3OutputConfiguration;
- this.s3UtilInstance = s3UtilInstance;
this.deleteOnEnd = deleteOnEnd;
this.logType = logType;
this.fileContextsToUpload = new LinkedBlockingQueue<>();
@@ -81,7 +83,7 @@ public class S3Uploader implements Runnable {
stopRunningThread.set(true);
boolean offerStatus = fileContextsToUpload.offer(POISON_PILL);
if (!offerStatus) {
- logger.warn("Could not add poison pill to interrupt uploader thread.");
+ LOG.warn("Could not add poison pill to interrupt uploader thread.");
}
}
@@ -92,7 +94,7 @@ public class S3Uploader implements Runnable {
void addFileForUpload(String fileToUpload) {
boolean offerStatus = fileContextsToUpload.offer(fileToUpload);
if (!offerStatus) {
- logger.error("Could not add file " + fileToUpload + " for upload.");
+ LOG.error("Could not add file " + fileToUpload + " for upload.");
}
}
@@ -102,12 +104,12 @@ public class S3Uploader implements Runnable {
try {
String fileNameToUpload = fileContextsToUpload.take();
if (POISON_PILL.equals(fileNameToUpload)) {
- logger.warn("Found poison pill while waiting for files to upload, exiting");
+ LOG.warn("Found poison pill while waiting for files to upload, exiting");
return;
}
uploadFile(new File(fileNameToUpload), logType);
} catch (InterruptedException e) {
- logger.error("Interrupted while waiting for elements from fileContextsToUpload", e);
+ LOG.error("Interrupted while waiting for elements from fileContextsToUpload", e);
return;
}
}
@@ -130,34 +132,44 @@ public class S3Uploader implements Runnable {
String compressionAlgo = s3OutputConfiguration.getCompressionAlgo();
String keySuffix = fileToUpload.getName() + "." + compressionAlgo;
- String s3Path = new S3LogPathResolver().
- getResolvedPath(s3OutputConfiguration.getS3Path()+S3Util.S3_PATH_SEPARATOR+logType,
- keySuffix, s3OutputConfiguration.getCluster());
- logger.info(String.format("keyPrefix=%s, keySuffix=%s, s3Path=%s",
- s3OutputConfiguration.getS3Path(), keySuffix, s3Path));
+ String s3Path = new S3LogPathResolver().getResolvedPath(
+ s3OutputConfiguration.getS3Path() + LogFeederConstants.S3_PATH_SEPARATOR + logType, keySuffix,
+ s3OutputConfiguration.getCluster());
+ LOG.info(String.format("keyPrefix=%s, keySuffix=%s, s3Path=%s", s3OutputConfiguration.getS3Path(), keySuffix, s3Path));
File sourceFile = createCompressedFileForUpload(fileToUpload, compressionAlgo);
- logger.info("Starting S3 upload " + sourceFile + " -> " + bucketName + ", " + s3Path);
- s3UtilInstance.uploadFileTos3(bucketName, s3Path, sourceFile, s3AccessKey,
- s3SecretKey);
+ LOG.info("Starting S3 upload " + sourceFile + " -> " + bucketName + ", " + s3Path);
+ uploadFileToS3(bucketName, s3Path, sourceFile, s3AccessKey, s3SecretKey);
// delete local compressed file
sourceFile.delete();
if (deleteOnEnd) {
- logger.info("Deleting input file as required");
+ LOG.info("Deleting input file as required");
if (!fileToUpload.delete()) {
- logger.error("Could not delete file " + fileToUpload.getAbsolutePath() + " after upload to S3");
+ LOG.error("Could not delete file " + fileToUpload.getAbsolutePath() + " after upload to S3");
}
}
return s3Path;
}
@VisibleForTesting
+ protected void uploadFileToS3(String bucketName, String s3Key, File localFile, String accessKey, String secretKey) {
+ TransferManager transferManager = S3Util.getTransferManager(accessKey, secretKey);
+ try {
+ Upload upload = transferManager.upload(bucketName, s3Key, localFile);
+ upload.waitForUploadResult();
+ } catch (AmazonClientException | InterruptedException e) {
+ LOG.error("s3 uploading failed for file :" + localFile.getAbsolutePath(), e);
+ } finally {
+ S3Util.shutdownTransferManager(transferManager);
+ }
+ }
+
+ @VisibleForTesting
protected File createCompressedFileForUpload(File fileToUpload, String compressionAlgo) {
- File outputFile = new File(fileToUpload.getParent(), fileToUpload.getName() + "_"
- + new Date().getTime() + "." + compressionAlgo);
- outputFile = CompressionUtil.compressFile(fileToUpload, outputFile,
- compressionAlgo);
+ File outputFile = new File(fileToUpload.getParent(), fileToUpload.getName() + "_" + new Date().getTime() +
+ "." + compressionAlgo);
+ outputFile = CompressionUtil.compressFile(fileToUpload, outputFile, compressionAlgo);
return outputFile;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
index fb263ba..1f13357 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpooler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -39,8 +39,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
* {@link RolloverHandler} to trigger the handling of the rolled over file.
*/
public class LogSpooler {
+
+ private static final Logger LOG = Logger.getLogger(LogSpooler.class);
public static final long TIME_BASED_ROLLOVER_DISABLED_THRESHOLD = 0;
- static private Logger logger = Logger.getLogger(LogSpooler.class);
static final String fileDateFormat = "yyyy-MM-dd-HH-mm-ss";
private String spoolDirectory;
@@ -98,7 +99,7 @@ public class LogSpooler {
private void initializeSpoolDirectory() {
File spoolDir = new File(spoolDirectory);
if (!spoolDir.exists()) {
- logger.info("Creating spool directory: " + spoolDir);
+ LOG.info("Creating spool directory: " + spoolDir);
boolean result = spoolDir.mkdirs();
if (!result) {
throw new LogSpoolerException("Could not create spool directory: " + spoolDirectory);
@@ -116,7 +117,7 @@ public class LogSpooler {
+ ", error message: " + e.getLocalizedMessage(), e);
}
currentSpoolerContext = new LogSpoolerContext(currentSpoolFile);
- logger.info("Initialized spool file at path: " + currentSpoolFile);
+ LOG.info("Initialized spool file at path: " + currentSpoolFile);
}
@VisibleForTesting
@@ -141,7 +142,7 @@ public class LogSpooler {
currentSpoolBufferedWriter.println(logEvent);
currentSpoolerContext.logEventSpooled();
if (rolloverCondition.shouldRollover(currentSpoolerContext)) {
- logger.info("Trying to rollover based on rollover condition");
+ LOG.info("Trying to rollover based on rollover condition");
tryRollover();
}
}
@@ -154,19 +155,19 @@ public class LogSpooler {
* rolled over file.
*/
public void rollover() {
- logger.info("Rollover condition detected, rolling over file: " + currentSpoolFile);
+ LOG.info("Rollover condition detected, rolling over file: " + currentSpoolFile);
currentSpoolBufferedWriter.flush();
if (currentSpoolFile.length()==0) {
- logger.info("No data in file " + currentSpoolFile + ", not doing rollover");
+ LOG.info("No data in file " + currentSpoolFile + ", not doing rollover");
} else {
currentSpoolBufferedWriter.close();
rolloverHandler.handleRollover(currentSpoolFile);
- logger.info("Invoked rollover handler with file: " + currentSpoolFile);
+ LOG.info("Invoked rollover handler with file: " + currentSpoolFile);
initializeSpoolState();
}
boolean status = rolloverInProgress.compareAndSet(true, false);
if (!status) {
- logger.error("Should have reset rollover flag!!");
+ LOG.error("Should have reset rollover flag!!");
}
}
@@ -174,7 +175,7 @@ public class LogSpooler {
if (rolloverInProgress.compareAndSet(false, true)) {
rollover();
} else {
- logger.warn("Ignoring rollover call as rollover already in progress for file " +
+ LOG.warn("Ignoring rollover call as rollover already in progress for file " +
currentSpoolFile);
}
}
@@ -197,7 +198,7 @@ public class LogSpooler {
private class LogSpoolerRolloverTimerTask extends TimerTask {
@Override
public void run() {
- logger.info("Trying rollover based on time");
+ LOG.info("Trying rollover based on time");
tryRollover();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerContext.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerContext.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerContext.java
index 084d6a2..616300f 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerContext.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerContext.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerException.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerException.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerException.java
index 1e12fb7..14bb139 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerException.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/LogSpoolerException.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverCondition.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverCondition.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverCondition.java
index 8279645..48ace11 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverCondition.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverCondition.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverHandler.java
index 11308e4..2ec2708 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverHandler.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/spool/RolloverHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AWSUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AWSUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AWSUtil.java
index 15f7594..f814a92 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AWSUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AWSUtil.java
@@ -20,62 +20,20 @@ package org.apache.ambari.logfeeder.util;
import org.apache.log4j.Logger;
-import com.amazonaws.AmazonServiceException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.services.identitymanagement.AmazonIdentityManagementClient;
-public enum AWSUtil {
- INSTANCE;
+public class AWSUtil {
private static final Logger LOG = Logger.getLogger(AWSUtil.class);
- public String getAwsUserName(String accessKey, String secretKey) {
- String username = null;
- AWSCredentials awsCredentials = createAWSCredentials(accessKey, secretKey);
- AmazonIdentityManagementClient amazonIdentityManagementClient;
- if (awsCredentials != null) {
- amazonIdentityManagementClient = new AmazonIdentityManagementClient(
- awsCredentials);
- } else {
- // create default client
- amazonIdentityManagementClient = new AmazonIdentityManagementClient();
- }
- try {
- username = amazonIdentityManagementClient.getUser().getUser()
- .getUserName();
- } catch (AmazonServiceException e) {
- if (e.getErrorCode().compareTo("AccessDenied") == 0) {
- String arn = null;
- String msg = e.getMessage();
- int arnIdx = msg.indexOf("arn:aws");
- if (arnIdx != -1) {
- int arnSpace = msg.indexOf(" ", arnIdx);
- // should be similar to "arn:aws:iam::111111111111:user/username"
- arn = msg.substring(arnIdx, arnSpace);
- }
- if (arn != null) {
- String[] arnParts = arn.split(":");
- if (arnParts != null && arnParts.length > 5) {
- username = arnParts[5];
- if (username != null) {
- username = username.replace("user/", "");
- }
- }
- }
- }
- } catch (Exception exception) {
- LOG.error(
- "Error in getting username :" + exception.getLocalizedMessage(),
- exception.getCause());
- }
- return username;
+ private AWSUtil() {
+ throw new UnsupportedOperationException();
}
- public AWSCredentials createAWSCredentials(String accessKey, String secretKey) {
+ public static AWSCredentials createAWSCredentials(String accessKey, String secretKey) {
if (accessKey != null && secretKey != null) {
LOG.debug("Creating aws client as per new accesskey and secretkey");
- AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey,
- secretKey);
+ AWSCredentials awsCredentials = new BasicAWSCredentials(accessKey, secretKey);
return awsCredentials;
} else {
return null;
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AliasUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AliasUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AliasUtil.java
index a92ba29..5049b62 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AliasUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/AliasUtil.java
@@ -21,69 +21,90 @@ package org.apache.ambari.logfeeder.util;
import java.io.File;
import java.util.HashMap;
+import org.apache.ambari.logfeeder.filter.Filter;
+import org.apache.ambari.logfeeder.input.Input;
+import org.apache.ambari.logfeeder.mapper.Mapper;
+import org.apache.ambari.logfeeder.output.Output;
+import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
public class AliasUtil {
- private static Logger logger = Logger.getLogger(AliasUtil.class);
+ private static final Logger LOG = Logger.getLogger(AliasUtil.class);
- private static AliasUtil instance = null;
+ private static final String ALIAS_CONFIG_JSON = "alias_config.json";
+ private static HashMap<String, Object> aliasMap = null;
- private static String aliasConfigJson = "alias_config.json";
-
- private HashMap<String, Object> aliasMap = null;
-
- public static enum ALIAS_TYPE {
- INPUT, FILTER, MAPPER, OUTPUT
+ static {
+ File jsonFile = FileUtil.getFileFromClasspath(ALIAS_CONFIG_JSON);
+ if (jsonFile != null) {
+ aliasMap = FileUtil.readJsonFromFile(jsonFile);
+ }
}
- public static enum ALIAS_PARAM {
- KLASS
+ public static enum AliasType {
+ INPUT, FILTER, MAPPER, OUTPUT
}
private AliasUtil() {
- init();
+ throw new UnsupportedOperationException();
}
- public static AliasUtil getInstance() {
- if (instance == null) {
- synchronized (AliasUtil.class) {
- if (instance == null) {
- instance = new AliasUtil();
- }
- }
+ public static Object getClassInstance(String key, AliasType aliasType) {
+ String classFullName = getClassFullName(key, aliasType);
+
+ Object instance = null;
+ try {
+ instance = (Object) Class.forName(classFullName).getConstructor().newInstance();
+ } catch (Exception exception) {
+ LOG.error("Unsupported class = " + classFullName, exception.getCause());
}
- return instance;
- }
- /**
- */
- private void init() {
- File jsonFile = LogFeederUtil.getFileFromClasspath(aliasConfigJson);
- if (jsonFile != null) {
- this.aliasMap = LogFeederUtil.readJsonFromFile(jsonFile);
+ if (instance != null) {
+ boolean isValid = false;
+ switch (aliasType) {
+ case FILTER:
+ isValid = Filter.class.isAssignableFrom(instance.getClass());
+ break;
+ case INPUT:
+ isValid = Input.class.isAssignableFrom(instance.getClass());
+ break;
+ case OUTPUT:
+ isValid = Output.class.isAssignableFrom(instance.getClass());
+ break;
+ case MAPPER:
+ isValid = Mapper.class.isAssignableFrom(instance.getClass());
+ break;
+ default:
+ LOG.warn("Unhandled aliasType: " + aliasType);
+ isValid = true;
+ }
+ if (!isValid) {
+ LOG.error("Not a valid class :" + classFullName + " AliasType :" + aliasType.name());
+ }
}
-
+ return instance;
}
-
- public String readAlias(String key, ALIAS_TYPE aliastype, ALIAS_PARAM aliasParam) {
- String result = key;// key as a default value;
+ private static String getClassFullName(String key, AliasType aliastype) {
+ String className = null;// key as a default value;
+
HashMap<String, String> aliasInfo = getAliasInfo(key, aliastype);
- String value = aliasInfo.get(aliasParam.name().toLowerCase());
- if (value != null && !value.isEmpty()) {
- result = value;
- logger.debug("Alias found for key :" + key + ", param :" + aliasParam.name().toLowerCase() + ", value :"
- + value + " aliastype:" + aliastype.name());
+ String value = aliasInfo.get("klass");
+ if (!StringUtils.isEmpty(value)) {
+ className = value;
+ LOG.debug("Class name found for key :" + key + ", class name :" + className + " aliastype:" + aliastype.name());
} else {
- logger.debug("Alias not found for key :" + key + ", param :" + aliasParam.name().toLowerCase());
+ LOG.debug("Class name not found for key :" + key + " aliastype:" + aliastype.name());
}
- return result;
+
+ return className;
}
@SuppressWarnings("unchecked")
- private HashMap<String, String> getAliasInfo(String key, ALIAS_TYPE aliastype) {
- HashMap<String, String> aliasInfo = null;
+ private static HashMap<String, String> getAliasInfo(String key, AliasType aliastype) {
+ HashMap<String, String> aliasInfo = new HashMap<String, String>();
+
if (aliasMap != null) {
String typeKey = aliastype.name().toLowerCase();
HashMap<String, Object> typeJson = (HashMap<String, Object>) aliasMap.get(typeKey);
@@ -91,9 +112,7 @@ public class AliasUtil {
aliasInfo = (HashMap<String, String>) typeJson.get(key);
}
}
- if (aliasInfo == null) {
- aliasInfo = new HashMap<String, String>();
- }
+
return aliasInfo;
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/CompressionUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/CompressionUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/CompressionUtil.java
index c2addbd..c460ab3 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/CompressionUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/CompressionUtil.java
@@ -37,25 +37,20 @@ public class CompressionUtil {
FileInputStream ios = null;
try {
if (!inputFile.exists()) {
- throw new IllegalArgumentException("Input File:"
- + inputFile.getAbsolutePath() + " is not exist.");
+ throw new IllegalArgumentException("Input File:" + inputFile.getAbsolutePath() + " is not exist.");
}
if (inputFile.isDirectory()) {
- throw new IllegalArgumentException("Input File:"
- + inputFile.getAbsolutePath() + " is a directory.");
+ throw new IllegalArgumentException("Input File:" + inputFile.getAbsolutePath() + " is a directory.");
}
File parent = outputFile.getParentFile();
if (parent != null && !parent.exists()) {
boolean isParentCreated = parent.mkdirs();
if (!isParentCreated) {
- throw new IllegalAccessException(
- "User does not have permission to create parent directory :"
- + parent.getAbsolutePath());
+ throw new IllegalAccessException( "User does not have permission to create parent directory :" + parent.getAbsolutePath());
}
}
- final OutputStream out = new FileOutputStream(outputFile);
- cos = new CompressorStreamFactory().createCompressorOutputStream(
- algoName, out);
+ OutputStream out = new FileOutputStream(outputFile);
+ cos = new CompressorStreamFactory().createCompressorOutputStream(algoName, out);
ios = new FileInputStream(inputFile);
IOUtils.copy(ios, cos);
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/DateUtil.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/DateUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/DateUtil.java
index 2ca9353..6321e17 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/DateUtil.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/DateUtil.java
@@ -20,12 +20,17 @@ package org.apache.ambari.logfeeder.util;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.TimeZone;
import org.apache.log4j.Logger;
public class DateUtil {
- private static final Logger logger = Logger.getLogger(DateUtil.class);
-
+ private static final Logger LOG = Logger.getLogger(DateUtil.class);
+
+ private DateUtil() {
+ throw new UnsupportedOperationException();
+ }
+
public static String dateToString(Date date, String dateFormat) {
if (date == null || dateFormat == null || dateFormat.isEmpty()) {
return "";
@@ -34,8 +39,36 @@ public class DateUtil {
SimpleDateFormat formatter = new SimpleDateFormat(dateFormat);
return formatter.format(date).toString();
} catch (Exception e) {
- logger.error("Error in coverting dateToString format :" + dateFormat, e);
+ LOG.error("Error in coverting dateToString format :" + dateFormat, e);
}
return "";
}
+
+ private final static String SOLR_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
+ private static ThreadLocal<SimpleDateFormat> dateFormatter = new ThreadLocal<SimpleDateFormat>() {
+ @Override
+ protected SimpleDateFormat initialValue() {
+ SimpleDateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT);
+ sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return sdf;
+ }
+ };
+
+ public static String getDate(String timeStampStr) {
+ try {
+ return dateFormatter.get().format(new Date(Long.parseLong(timeStampStr)));
+ } catch (Exception ex) {
+ LOG.error(ex);
+ return null;
+ }
+ }
+
+ public static String getActualDateStr() {
+ try {
+ return dateFormatter.get().format(new Date());
+ } catch (Exception ex) {
+ LOG.error(ex);
+ return null;
+ }
+ }
}
[07/10] ambari git commit: AMBARI-18246. Clean up Log Feeder (Miklos
Gergely via oleewere)
Posted by ol...@apache.org.
AMBARI-18246. Clean up Log Feeder (Miklos Gergely via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8399960d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8399960d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8399960d
Branch: refs/heads/branch-dev-logsearch
Commit: 8399960da7f013c7df7ca45622ceb0c0d1e36e71
Parents: 0a30feb
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Wed Sep 7 22:24:51 2016 +0200
Committer: oleewere <ol...@gmail.com>
Committed: Wed Sep 7 22:24:51 2016 +0200
----------------------------------------------------------------------
.../org/apache/ambari/logfeeder/LogFeeder.java | 594 +++++++------------
.../ambari/logfeeder/common/ConfigBlock.java | 71 +--
.../logfeeder/common/LogFeederConstants.java | 39 ++
.../apache/ambari/logfeeder/filter/Filter.java | 55 +-
.../ambari/logfeeder/filter/FilterGrok.java | 94 +--
.../ambari/logfeeder/filter/FilterJSON.java | 8 +-
.../ambari/logfeeder/filter/FilterKeyValue.java | 44 +-
.../logfeeder/input/AbstractInputFile.java | 319 ++++++++++
.../apache/ambari/logfeeder/input/Input.java | 313 +++++-----
.../ambari/logfeeder/input/InputFile.java | 503 ++--------------
.../ambari/logfeeder/input/InputManager.java | 379 ++++++++++++
.../ambari/logfeeder/input/InputMarker.java | 17 +-
.../apache/ambari/logfeeder/input/InputMgr.java | 451 --------------
.../ambari/logfeeder/input/InputS3File.java | 424 +------------
.../ambari/logfeeder/input/InputSimulate.java | 40 +-
.../logfeeder/input/reader/GZIPReader.java | 23 +-
.../input/reader/LogsearchReaderFactory.java | 8 +-
.../logconfig/FetchConfigFromSolr.java | 194 ------
.../logfeeder/logconfig/FilterLogData.java | 83 +++
.../logfeeder/logconfig/LogConfigFetcher.java | 168 ++++++
.../logfeeder/logconfig/LogConfigHandler.java | 189 ++++++
.../logfeeder/logconfig/LogFeederConstants.java | 34 --
.../logfeeder/logconfig/LogFeederFilter.java | 90 +++
.../logconfig/LogFeederFilterWrapper.java | 55 ++
.../logfeeder/logconfig/LogfeederScheduler.java | 59 --
.../logconfig/filter/ApplyLogFilter.java | 62 --
.../logconfig/filter/DefaultDataFilter.java | 49 --
.../logconfig/filter/FilterLogData.java | 53 --
.../apache/ambari/logfeeder/mapper/Mapper.java | 14 +-
.../ambari/logfeeder/mapper/MapperDate.java | 32 +-
.../logfeeder/mapper/MapperFieldName.java | 20 +-
.../logfeeder/mapper/MapperFieldValue.java | 31 +-
.../logfeeder/metrics/LogFeederAMSClient.java | 10 +-
.../ambari/logfeeder/metrics/MetricCount.java | 31 -
.../ambari/logfeeder/metrics/MetricData.java | 46 ++
.../logfeeder/metrics/MetricsManager.java | 168 ++++++
.../ambari/logfeeder/metrics/MetricsMgr.java | 178 ------
.../apache/ambari/logfeeder/output/Output.java | 13 +-
.../ambari/logfeeder/output/OutputData.java | 8 +-
.../ambari/logfeeder/output/OutputDevNull.java | 7 +-
.../ambari/logfeeder/output/OutputFile.java | 42 +-
.../ambari/logfeeder/output/OutputHDFSFile.java | 70 +--
.../ambari/logfeeder/output/OutputKafka.java | 58 +-
.../ambari/logfeeder/output/OutputManager.java | 250 ++++++++
.../ambari/logfeeder/output/OutputMgr.java | 263 --------
.../ambari/logfeeder/output/OutputS3File.java | 41 +-
.../ambari/logfeeder/output/OutputSolr.java | 62 +-
.../logfeeder/output/S3LogPathResolver.java | 6 +-
.../logfeeder/output/S3OutputConfiguration.java | 5 +-
.../ambari/logfeeder/output/S3Uploader.java | 64 +-
.../logfeeder/output/spool/LogSpooler.java | 23 +-
.../output/spool/LogSpoolerContext.java | 2 +-
.../output/spool/LogSpoolerException.java | 2 +-
.../output/spool/RolloverCondition.java | 2 +-
.../logfeeder/output/spool/RolloverHandler.java | 2 +-
.../apache/ambari/logfeeder/util/AWSUtil.java | 52 +-
.../apache/ambari/logfeeder/util/AliasUtil.java | 103 ++--
.../ambari/logfeeder/util/CompressionUtil.java | 15 +-
.../apache/ambari/logfeeder/util/DateUtil.java | 39 +-
.../apache/ambari/logfeeder/util/FileUtil.java | 66 ++-
.../ambari/logfeeder/util/LogFeederUtil.java | 511 +++++-----------
.../logfeeder/util/LogfeederHDFSUtil.java | 58 +-
.../ambari/logfeeder/util/PlaceholderUtil.java | 32 +-
.../apache/ambari/logfeeder/util/S3Util.java | 81 +--
.../apache/ambari/logfeeder/util/SolrUtil.java | 186 ------
.../ambari/logfeeder/view/VLogfeederFilter.java | 90 ---
.../logfeeder/view/VLogfeederFilterWrapper.java | 55 --
.../org/apache/ambari/logfeeder/AppTest.java | 116 ----
.../ambari/logfeeder/filter/FilterGrokTest.java | 55 +-
.../ambari/logfeeder/filter/FilterJSONTest.java | 41 +-
.../logfeeder/filter/FilterKeyValueTest.java | 34 +-
.../ambari/logfeeder/input/InputFileTest.java | 24 +-
.../logfeeder/input/InputManagerTest.java | 241 ++++++++
.../logconfig/LogConfigHandlerTest.java | 117 ++++
.../ambari/logfeeder/mapper/MapperDateTest.java | 17 +-
.../logfeeder/mapper/MapperFieldNameTest.java | 2 +-
.../logfeeder/mapper/MapperFieldValueTest.java | 2 +-
.../logfeeder/metrics/MetrcisManagerTest.java | 128 ++++
.../logfeeder/output/OutputKafkaTest.java | 5 +-
.../logfeeder/output/OutputManagerTest.java | 256 ++++++++
.../logfeeder/output/OutputS3FileTest.java | 17 +-
.../ambari/logfeeder/output/OutputSolrTest.java | 5 +-
.../logfeeder/output/S3LogPathResolverTest.java | 2 +-
.../ambari/logfeeder/output/S3UploaderTest.java | 42 +-
.../logfeeder/output/spool/LogSpoolerTest.java | 2 +-
.../ambari/logfeeder/util/AWSUtilTest.java | 29 -
.../logfeeder/util/PlaceholderUtilTest.java | 20 +-
.../ambari/logfeeder/util/S3UtilTest.java | 4 +-
.../src/test/resources/logfeeder.properties | 20 +
89 files changed, 3854 insertions(+), 4481 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
index 373d743..6d0f22c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeeder.java
@@ -20,14 +20,10 @@
package org.apache.ambari.logfeeder;
import java.io.BufferedInputStream;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
import java.lang.reflect.Type;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
@@ -39,18 +35,21 @@ import java.util.Set;
import org.apache.ambari.logfeeder.filter.Filter;
import org.apache.ambari.logfeeder.input.Input;
-import org.apache.ambari.logfeeder.input.InputMgr;
+import org.apache.ambari.logfeeder.input.InputManager;
import org.apache.ambari.logfeeder.input.InputSimulate;
-import org.apache.ambari.logfeeder.logconfig.LogfeederScheduler;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
-import org.apache.ambari.logfeeder.metrics.MetricsMgr;
+import org.apache.ambari.logfeeder.logconfig.LogConfigHandler;
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logfeeder.metrics.MetricsManager;
import org.apache.ambari.logfeeder.output.Output;
-import org.apache.ambari.logfeeder.output.OutputMgr;
+import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.ambari.logfeeder.util.AliasUtil;
import org.apache.ambari.logfeeder.util.FileUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logfeeder.util.AliasUtil.ALIAS_PARAM;
-import org.apache.ambari.logfeeder.util.AliasUtil.ALIAS_TYPE;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.ambari.logfeeder.util.AliasUtil.AliasType;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
@@ -58,171 +57,142 @@ import org.apache.log4j.Logger;
import com.google.gson.reflect.TypeToken;
public class LogFeeder {
- private static final Logger logger = Logger.getLogger(LogFeeder.class);
+ private static final Logger LOG = Logger.getLogger(LogFeeder.class);
private static final int LOGFEEDER_SHUTDOWN_HOOK_PRIORITY = 30;
+ private static final int CHECKPOINT_CLEAN_INTERVAL_MS = 24 * 60 * 60 * 60 * 1000; // 24 hours
- private Collection<Output> outputList = new ArrayList<Output>();
+ private OutputManager outputManager = new OutputManager();
+ private InputManager inputManager = new InputManager();
+ private MetricsManager metricsManager = new MetricsManager();
- private OutputMgr outMgr = new OutputMgr();
- private InputMgr inputMgr = new InputMgr();
- private MetricsMgr metricsMgr = new MetricsMgr();
+ public static Map<String, Object> globalConfigs = new HashMap<>();
- public static Map<String, Object> globalMap = null;
- private String[] inputParams;
-
- private List<Map<String, Object>> globalConfigList = new ArrayList<Map<String, Object>>();
- private List<Map<String, Object>> inputConfigList = new ArrayList<Map<String, Object>>();
- private List<Map<String, Object>> filterConfigList = new ArrayList<Map<String, Object>>();
- private List<Map<String, Object>> outputConfigList = new ArrayList<Map<String, Object>>();
+ private List<Map<String, Object>> inputConfigList = new ArrayList<>();
+ private List<Map<String, Object>> filterConfigList = new ArrayList<>();
+ private List<Map<String, Object>> outputConfigList = new ArrayList<>();
- private int checkPointCleanIntervalMS = 24 * 60 * 60 * 60 * 1000; // 24 hours
private long lastCheckPointCleanedMS = 0;
-
- private static boolean isLogfeederCompleted = false;
-
+ private boolean isLogfeederCompleted = false;
private Thread statLoggerThread = null;
- private LogFeeder(String[] args) {
- inputParams = args;
+ private LogFeeder() {}
+
+ public void run() {
+ try {
+ init();
+ monitor();
+ waitOnAllDaemonThreads();
+ } catch (Throwable t) {
+ LOG.fatal("Caught exception in main.", t);
+ System.exit(1);
+ }
}
private void init() throws Throwable {
+ Date startTime = new Date();
- LogFeederUtil.loadProperties("logfeeder.properties", inputParams);
-
- String configFiles = LogFeederUtil.getStringProperty("logfeeder.config.files");
- logger.info("logfeeder.config.files=" + configFiles);
+ loadConfigFiles();
+ addSimulatedInputs();
+ mergeAllConfigs();
- String[] configFileList = null;
- if (configFiles != null) {
- configFileList = configFiles.split(",");
- }
- //list of config those are there in cmd line config dir , end with .json
- String[] cmdLineConfigs = getConfigFromCmdLine();
- //merge both config
- String mergedConfigList[] = LogFeederUtil.mergeArray(configFileList,
- cmdLineConfigs);
- //mergedConfigList is null then set default conifg
- if (mergedConfigList == null || mergedConfigList.length == 0) {
- mergedConfigList = LogFeederUtil.getStringProperty("config.file",
- "config.json").split(",");
- }
- for (String configFileName : mergedConfigList) {
- logger.info("Going to load config file:" + configFileName);
- //escape space from config file path
- configFileName= configFileName.replace("\\ ", "%20");
+ LogConfigHandler.handleConfig();
+
+ outputManager.init();
+ inputManager.init();
+ metricsManager.init();
+
+ LOG.debug("==============");
+
+ Date endTime = new Date();
+ LOG.info("Took " + (endTime.getTime() - startTime.getTime()) + " ms to initialize");
+ }
+
+ private void loadConfigFiles() throws Exception {
+ List<String> configFiles = getConfigFiles();
+ for (String configFileName : configFiles) {
+ LOG.info("Going to load config file:" + configFileName);
+ configFileName = configFileName.replace("\\ ", "%20");
File configFile = new File(configFileName);
if (configFile.exists() && configFile.isFile()) {
- logger.info("Config file exists in path."
- + configFile.getAbsolutePath());
+ LOG.info("Config file exists in path." + configFile.getAbsolutePath());
loadConfigsUsingFile(configFile);
} else {
- // Let's try to load it from class loader
- logger.info("Trying to load config file from classloader: "
- + configFileName);
+ LOG.info("Trying to load config file from classloader: " + configFileName);
loadConfigsUsingClassLoader(configFileName);
- logger.info("Loaded config file from classloader: "
- + configFileName);
+ LOG.info("Loaded config file from classloader: " + configFileName);
}
}
-
- addSimulatedInputs();
-
- mergeAllConfigs();
-
- LogfeederScheduler.INSTANCE.start();
-
- outMgr.setOutputList(outputList);
- for (Output output : outputList) {
- output.init();
- }
- inputMgr.init();
- metricsMgr.init();
- logger.debug("==============");
}
- private void loadConfigsUsingClassLoader(String configFileName) throws Exception {
- BufferedInputStream fileInputStream = null;
- BufferedReader br = null;
- try {
- fileInputStream = (BufferedInputStream) this
- .getClass().getClassLoader()
- .getResourceAsStream(configFileName);
- if (fileInputStream != null) {
- br = new BufferedReader(new InputStreamReader(
- fileInputStream));
- String configData = readFile(br);
- loadConfigs(configData);
- } else {
- throw new Exception("Can't find configFile=" + configFileName);
- }
- } finally {
- if (br != null) {
- try {
- br.close();
- } catch (IOException e) {
- }
- }
+ private List<String> getConfigFiles() {
+ List<String> configFiles = new ArrayList<>();
+
+ String logfeederConfigFilesProperty = LogFeederUtil.getStringProperty("logfeeder.config.files");
+ LOG.info("logfeeder.config.files=" + logfeederConfigFilesProperty);
+ if (logfeederConfigFilesProperty != null) {
+ configFiles.addAll(Arrays.asList(logfeederConfigFilesProperty.split(",")));
+ }
- if (fileInputStream != null) {
- try {
- fileInputStream.close();
- } catch (IOException e) {
- }
+ String inputConfigDir = LogFeederUtil.getStringProperty("input_config_dir");
+ if (StringUtils.isNotEmpty(inputConfigDir)) {
+ File configDirFile = new File(inputConfigDir);
+ List<File> inputConfigFiles = FileUtil.getAllFileFromDir(configDirFile, "json", false);
+ for (File inputConfigFile : inputConfigFiles) {
+ configFiles.add(inputConfigFile.getAbsolutePath());
}
}
+
+ if (CollectionUtils.isEmpty(configFiles)) {
+ String configFileProperty = LogFeederUtil.getStringProperty("config.file", "config.json");
+ configFiles.addAll(Arrays.asList(configFileProperty.split(",")));
+ }
+
+ return configFiles;
}
- /**
- * This method loads the configurations from the given file.
- */
private void loadConfigsUsingFile(File configFile) throws Exception {
- FileInputStream fileInputStream = null;
try {
- fileInputStream = new FileInputStream(configFile);
- BufferedReader br = new BufferedReader(new InputStreamReader(
- fileInputStream));
- String configData = readFile(br);
+ String configData = FileUtils.readFileToString(configFile);
loadConfigs(configData);
} catch (Exception t) {
- logger.error("Error opening config file. configFilePath="
- + configFile.getAbsolutePath());
+ LOG.error("Error opening config file. configFilePath=" + configFile.getAbsolutePath());
throw t;
- } finally {
- if (fileInputStream != null) {
- try {
- fileInputStream.close();
- } catch (Throwable t) {
- // ignore
- }
- }
+ }
+ }
+
+ private void loadConfigsUsingClassLoader(String configFileName) throws Exception {
+ try (BufferedInputStream fis = (BufferedInputStream) this.getClass().getClassLoader().getResourceAsStream(configFileName)) {
+ String configData = IOUtils.toString(fis);
+ loadConfigs(configData);
}
}
@SuppressWarnings("unchecked")
private void loadConfigs(String configData) throws Exception {
- Type type = new TypeToken<Map<String, Object>>() {
- }.getType();
- Map<String, Object> configMap = LogFeederUtil.getGson().fromJson(
- configData, type);
+ Type type = new TypeToken<Map<String, Object>>() {}.getType();
+ Map<String, Object> configMap = LogFeederUtil.getGson().fromJson(configData, type);
// Get the globals
for (String key : configMap.keySet()) {
- if (key.equalsIgnoreCase("global")) {
- globalConfigList.add((Map<String, Object>) configMap.get(key));
- } else if (key.equalsIgnoreCase("input")) {
- List<Map<String, Object>> mapList = (List<Map<String, Object>>) configMap
- .get(key);
- inputConfigList.addAll(mapList);
- } else if (key.equalsIgnoreCase("filter")) {
- List<Map<String, Object>> mapList = (List<Map<String, Object>>) configMap
- .get(key);
- filterConfigList.addAll(mapList);
- } else if (key.equalsIgnoreCase("output")) {
- List<Map<String, Object>> mapList = (List<Map<String, Object>>) configMap
- .get(key);
- outputConfigList.addAll(mapList);
+ switch (key) {
+ case "global" :
+ globalConfigs.putAll((Map<String, Object>) configMap.get(key));
+ break;
+ case "input" :
+ List<Map<String, Object>> inputConfig = (List<Map<String, Object>>) configMap.get(key);
+ inputConfigList.addAll(inputConfig);
+ break;
+ case "filter" :
+ List<Map<String, Object>> filterConfig = (List<Map<String, Object>>) configMap.get(key);
+ filterConfigList.addAll(filterConfig);
+ break;
+ case "output" :
+ List<Map<String, Object>> outputConfig = (List<Map<String, Object>>) configMap.get(key);
+ outputConfigList.addAll(outputConfig);
+ break;
+ default :
+ LOG.warn("Unknown config key: " + key);
}
}
}
@@ -244,231 +214,175 @@ public class LogFeeder {
}
private void mergeAllConfigs() {
- globalMap = mergeConfigs(globalConfigList);
+ loadOutputs();
+ loadInputs();
+ loadFilters();
+
+ assignOutputsToInputs();
+ }
- sortBlocks(filterConfigList);
- // First loop for output
+ private void loadOutputs() {
for (Map<String, Object> map : outputConfigList) {
if (map == null) {
continue;
}
- mergeBlocks(globalMap, map);
+ mergeBlocks(globalConfigs, map);
String value = (String) map.get("destination");
- Output output;
- if (value == null || value.isEmpty()) {
- logger.error("Output block doesn't have destination element");
- continue;
- }
- String classFullName = AliasUtil.getInstance().readAlias(value, ALIAS_TYPE.OUTPUT, ALIAS_PARAM.KLASS);
- if (classFullName == null || classFullName.isEmpty()) {
- logger.error("Destination block doesn't have output element");
+ if (StringUtils.isEmpty(value)) {
+ LOG.error("Output block doesn't have destination element");
continue;
}
- output = (Output) LogFeederUtil.getClassInstance(classFullName, ALIAS_TYPE.OUTPUT);
-
+ Output output = (Output) AliasUtil.getClassInstance(value, AliasType.OUTPUT);
if (output == null) {
- logger.error("Destination Object is null");
+ LOG.error("Output object could not be found");
continue;
}
-
output.setDestination(value);
output.loadConfig(map);
- // We will only check for is_enabled out here. Down below we will
- // check whether this output is enabled for the input
- boolean isEnabled = output.getBooleanValue("is_enabled", true);
- if (isEnabled) {
- outputList.add(output);
+ // We will only check for is_enabled out here. Down below we will check whether this output is enabled for the input
+ if (output.getBooleanValue("is_enabled", true)) {
output.logConfgs(Level.INFO);
+ outputManager.add(output);
} else {
- logger.info("Output is disabled. So ignoring it. "
- + output.getShortDescription());
+ LOG.info("Output is disabled. So ignoring it. " + output.getShortDescription());
}
}
+ }
- // Second loop for input
+ private void loadInputs() {
for (Map<String, Object> map : inputConfigList) {
if (map == null) {
continue;
}
- mergeBlocks(globalMap, map);
+ mergeBlocks(globalConfigs, map);
String value = (String) map.get("source");
- Input input;
- if (value == null || value.isEmpty()) {
- logger.error("Input block doesn't have source element");
- continue;
- }
- String classFullName = AliasUtil.getInstance().readAlias(value, ALIAS_TYPE.INPUT, ALIAS_PARAM.KLASS);
- if (classFullName == null || classFullName.isEmpty()) {
- logger.error("Source block doesn't have source element");
+ if (StringUtils.isEmpty(value)) {
+ LOG.error("Input block doesn't have source element");
continue;
}
- input = (Input) LogFeederUtil.getClassInstance(classFullName, ALIAS_TYPE.INPUT);
-
+ Input input = (Input) AliasUtil.getClassInstance(value, AliasType.INPUT);
if (input == null) {
- logger.error("Source Object is null");
+ LOG.error("Input object could not be found");
continue;
}
-
input.setType(value);
input.loadConfig(map);
if (input.isEnabled()) {
- input.setOutputMgr(outMgr);
- input.setInputMgr(inputMgr);
- inputMgr.add(input);
+ input.setOutputManager(outputManager);
+ input.setInputManager(inputManager);
+ inputManager.add(input);
input.logConfgs(Level.INFO);
} else {
- logger.info("Input is disabled. So ignoring it. "
- + input.getShortDescription());
+ LOG.info("Input is disabled. So ignoring it. " + input.getShortDescription());
}
}
+ }
+
+ private void loadFilters() {
+ sortFilters();
- // Third loop is for filter, but we will have to create a filter
- // instance for each input, so it can maintain the state per input
List<Input> toRemoveInputList = new ArrayList<Input>();
- for (Input input : inputMgr.getInputList()) {
- Filter prevFilter = null;
+ for (Input input : inputManager.getInputList()) {
for (Map<String, Object> map : filterConfigList) {
if (map == null) {
continue;
}
- mergeBlocks(globalMap, map);
+ mergeBlocks(globalConfigs, map);
String value = (String) map.get("filter");
- Filter filter;
- if (value == null || value.isEmpty()) {
- logger.error("Filter block doesn't have filter element");
- continue;
- }
-
- String classFullName = AliasUtil.getInstance().readAlias(value, ALIAS_TYPE.FILTER, ALIAS_PARAM.KLASS);
- if (classFullName == null || classFullName.isEmpty()) {
- logger.error("Filter block doesn't have filter element");
+ if (StringUtils.isEmpty(value)) {
+ LOG.error("Filter block doesn't have filter element");
continue;
}
- filter = (Filter) LogFeederUtil.getClassInstance(classFullName, ALIAS_TYPE.FILTER);
-
+ Filter filter = (Filter) AliasUtil.getClassInstance(value, AliasType.FILTER);
if (filter == null) {
- logger.error("Filter Object is null");
+ LOG.error("Filter object could not be found");
continue;
}
filter.loadConfig(map);
filter.setInput(input);
if (filter.isEnabled()) {
- filter.setOutputMgr(outMgr);
- if (prevFilter == null) {
- input.setFirstFilter(filter);
- } else {
- prevFilter.setNextFilter(filter);
- }
- prevFilter = filter;
+ filter.setOutputManager(outputManager);
+ input.addFilter(filter);
filter.logConfgs(Level.INFO);
} else {
- logger.debug("Ignoring filter "
- + filter.getShortDescription() + " for input "
- + input.getShortDescription());
+ LOG.debug("Ignoring filter " + filter.getShortDescription() + " for input " + input.getShortDescription());
}
}
+
if (input.getFirstFilter() == null) {
toRemoveInputList.add(input);
}
}
- // Fourth loop is for associating valid outputs to input
- Set<Output> usedOutputSet = new HashSet<Output>();
- for (Input input : inputMgr.getInputList()) {
- for (Output output : outputList) {
- boolean ret = LogFeederUtil.isEnabled(output.getConfigs(),
- input.getConfigs());
- if (ret) {
- usedOutputSet.add(output);
- input.addOutput(output);
- }
- }
- }
- outputList = usedOutputSet;
-
for (Input toRemoveInput : toRemoveInputList) {
- logger.warn("There are no filters, we will ignore this input. "
- + toRemoveInput.getShortDescription());
- inputMgr.removeInput(toRemoveInput);
+ LOG.warn("There are no filters, we will ignore this input. " + toRemoveInput.getShortDescription());
+ inputManager.removeInput(toRemoveInput);
}
}
- private void sortBlocks(List<Map<String, Object>> blockList) {
-
- Collections.sort(blockList, new Comparator<Map<String, Object>>() {
+ private void sortFilters() {
+ Collections.sort(filterConfigList, new Comparator<Map<String, Object>>() {
@Override
public int compare(Map<String, Object> o1, Map<String, Object> o2) {
Object o1Sort = o1.get("sort_order");
Object o2Sort = o2.get("sort_order");
- if (o1Sort == null) {
- return 0;
- }
- if (o2Sort == null) {
+ if (o1Sort == null || o2Sort == null) {
return 0;
}
- int o1Value = 0;
- if (!(o1Sort instanceof Number)) {
- try {
- o1Value = (new Double(Double.parseDouble(o1Sort
- .toString()))).intValue();
- } catch (Throwable t) {
- logger.error("Value is not of type Number. class="
- + o1Sort.getClass().getName() + ", value="
- + o1Sort.toString() + ", map=" + o1.toString());
- }
- } else {
- o1Value = ((Number) o1Sort).intValue();
- }
- int o2Value = 0;
- if (!(o2Sort instanceof Integer)) {
+
+ int o1Value = parseSort(o1, o1Sort);
+ int o2Value = parseSort(o2, o2Sort);
+
+ return o1Value - o2Value;
+ }
+
+ private int parseSort(Map<String, Object> map, Object o) {
+ if (!(o instanceof Number)) {
try {
- o2Value = (new Double(Double.parseDouble(o2Sort
- .toString()))).intValue();
+ return (new Double(Double.parseDouble(o.toString()))).intValue();
} catch (Throwable t) {
- logger.error("Value is not of type Number. class="
- + o2Sort.getClass().getName() + ", value="
- + o2Sort.toString() + ", map=" + o2.toString());
+ LOG.error("Value is not of type Number. class=" + o.getClass().getName() + ", value=" + o.toString()
+ + ", map=" + map.toString());
+ return 0;
}
} else {
-
+ return ((Number) o).intValue();
}
- return o1Value - o2Value;
}
});
}
- private Map<String, Object> mergeConfigs(
- List<Map<String, Object>> configList) {
- Map<String, Object> mergedConfig = new HashMap<String, Object>();
- for (Map<String, Object> config : configList) {
- mergeBlocks(config, mergedConfig);
+ private void assignOutputsToInputs() {
+ Set<Output> usedOutputSet = new HashSet<Output>();
+ for (Input input : inputManager.getInputList()) {
+ for (Output output : outputManager.getOutputs()) {
+ if (LogFeederUtil.isEnabled(output.getConfigs(), input.getConfigs())) {
+ usedOutputSet.add(output);
+ input.addOutput(output);
+ }
+ }
}
- return mergedConfig;
+ outputManager.retainUsedOutputs(usedOutputSet);
}
- private void mergeBlocks(Map<String, Object> fromMap,
- Map<String, Object> toMap) {
- // Merge the non-string
+ @SuppressWarnings("unchecked")
+ private void mergeBlocks(Map<String, Object> fromMap, Map<String, Object> toMap) {
for (String key : fromMap.keySet()) {
Object objValue = fromMap.get(key);
if (objValue == null) {
continue;
}
if (objValue instanceof Map) {
- @SuppressWarnings("unchecked")
- Map<String, Object> globalFields = LogFeederUtil
- .cloneObject((Map<String, Object>) fromMap.get(key));
+ Map<String, Object> globalFields = LogFeederUtil.cloneObject((Map<String, Object>) objValue);
- @SuppressWarnings("unchecked")
- Map<String, Object> localFields = (Map<String, Object>) toMap
- .get(key);
+ Map<String, Object> localFields = (Map<String, Object>) toMap.get(key);
if (localFields == null) {
localFields = new HashMap<String, Object>();
toMap.put(key, localFields);
@@ -477,8 +391,7 @@ public class LogFeeder {
if (globalFields != null) {
for (String fieldKey : globalFields.keySet()) {
if (!localFields.containsKey(fieldKey)) {
- localFields.put(fieldKey,
- globalFields.get(fieldKey));
+ localFields.put(fieldKey, globalFields.get(fieldKey));
}
}
}
@@ -493,11 +406,29 @@ public class LogFeeder {
}
}
+ private class JVMShutdownHook extends Thread {
+
+ public void run() {
+ try {
+ LOG.info("Processing is shutting down.");
+
+ inputManager.close();
+ outputManager.close();
+ inputManager.checkInAll();
+
+ logStats();
+
+ LOG.info("LogSearch is exiting.");
+ } catch (Throwable t) {
+ // Ignore
+ }
+ }
+ }
+
private void monitor() throws Exception {
- inputMgr.monitor();
+ inputManager.monitor();
JVMShutdownHook logfeederJVMHook = new JVMShutdownHook();
- ShutdownHookManager.get().addShutdownHook(logfeederJVMHook,
- LOGFEEDER_SHUTDOWN_HOOK_PRIORITY);
+ ShutdownHookManager.get().addShutdownHook(logfeederJVMHook, LOGFEEDER_SHUTDOWN_HOOK_PRIORITY);
statLoggerThread = new Thread("statLogger") {
@@ -512,17 +443,14 @@ public class LogFeeder {
try {
logStats();
} catch (Throwable t) {
- logger.error(
- "LogStats: Caught exception while logging stats.",
- t);
+ LOG.error("LogStats: Caught exception while logging stats.", t);
}
- if (System.currentTimeMillis() > (lastCheckPointCleanedMS + checkPointCleanIntervalMS)) {
+ if (System.currentTimeMillis() > (lastCheckPointCleanedMS + CHECKPOINT_CLEAN_INTERVAL_MS)) {
lastCheckPointCleanedMS = System.currentTimeMillis();
- inputMgr.cleanCheckPointFiles();
+ inputManager.cleanCheckPointFiles();
}
- // logfeeder is stopped then break the loop
if (isLogfeederCompleted) {
break;
}
@@ -536,84 +464,20 @@ public class LogFeeder {
}
private void logStats() {
- inputMgr.logStats();
- outMgr.logStats();
-
- if (metricsMgr.isMetricsEnabled()) {
- List<MetricCount> metricsList = new ArrayList<MetricCount>();
- inputMgr.addMetricsContainers(metricsList);
- outMgr.addMetricsContainers(metricsList);
- metricsMgr.useMetrics(metricsList);
- }
- }
-
- private String readFile(BufferedReader br) throws Exception {
- try {
- StringBuilder sb = new StringBuilder();
- String line = br.readLine();
- while (line != null) {
- sb.append(line);
- line = br.readLine();
- }
- return sb.toString();
- } catch (Exception t) {
- logger.error("Error loading properties file.", t);
- throw t;
- }
- }
-
- public Collection<Output> getOutputList() {
- return outputList;
- }
-
- public OutputMgr getOutMgr() {
- return outMgr;
- }
-
- public static void main(String[] args) {
- LogFeeder logFeeder = new LogFeeder(args);
- logFeeder.run();
- }
-
- public void run() {
- try {
- Date startTime = new Date();
- this.init();
- Date endTime = new Date();
- logger.info("Took " + (endTime.getTime() - startTime.getTime())
- + " ms to initialize");
- this.monitor();
- //wait for all background thread before stop main thread
- this.waitOnAllDaemonThreads();
- } catch (Throwable t) {
- logger.fatal("Caught exception in main.", t);
- System.exit(1);
+ inputManager.logStats();
+ outputManager.logStats();
+
+ if (metricsManager.isMetricsEnabled()) {
+ List<MetricData> metricsList = new ArrayList<MetricData>();
+ inputManager.addMetricsContainers(metricsList);
+ outputManager.addMetricsContainers(metricsList);
+ metricsManager.useMetrics(metricsList);
}
}
- private class JVMShutdownHook extends Thread {
-
- public void run() {
- try {
- logger.info("Processing is shutting down.");
-
- inputMgr.close();
- outMgr.close();
- inputMgr.checkInAll();
-
- logStats();
-
- logger.info("LogSearch is exiting.");
- } catch (Throwable t) {
- // Ignore
- }
- }
- }
-
private void waitOnAllDaemonThreads() {
- String foreground = LogFeederUtil.getStringProperty("foreground");
- if (foreground != null && foreground.equalsIgnoreCase("true")) {
- inputMgr.waitOnAllInputs();
+ if ("true".equals(LogFeederUtil.getStringProperty("foreground"))) {
+ inputManager.waitOnAllInputs();
isLogfeederCompleted = true;
if (statLoggerThread != null) {
try {
@@ -624,24 +488,16 @@ public class LogFeeder {
}
}
}
-
- private String[] getConfigFromCmdLine() {
- String inputConfigDir = LogFeederUtil.getStringProperty("input_config_dir");
- if (inputConfigDir != null && !inputConfigDir.isEmpty()) {
- String[] searchFileWithExtensions = new String[] { "json" };
- File configDirFile = new File(inputConfigDir);
- List<File> configFiles = FileUtil.getAllFileFromDir(configDirFile,
- searchFileWithExtensions, false);
- if (configFiles != null && configFiles.size() > 0) {
- String configPaths[] = new String[configFiles.size()];
- for (int index = 0; index < configFiles.size(); index++) {
- File configFile = configFiles.get(index);
- String configFilePath = configFile.getAbsolutePath();
- configPaths[index] = configFilePath;
- }
- return configPaths;
- }
+
+ public static void main(String[] args) {
+ try {
+ LogFeederUtil.loadProperties("logfeeder.properties", args);
+ } catch (Throwable t) {
+ LOG.warn("Could not load logfeeder properites");
+ System.exit(1);
}
- return new String[0];
+
+ LogFeeder logFeeder = new LogFeeder();
+ logFeeder.run();
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigBlock.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigBlock.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigBlock.java
index 287982f..47ddc51 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigBlock.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigBlock.java
@@ -23,27 +23,27 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
+import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.apache.log4j.Priority;
public abstract class ConfigBlock {
- static private Logger logger = Logger.getLogger(ConfigBlock.class);
+ private static final Logger LOG = Logger.getLogger(ConfigBlock.class);
private boolean drain = false;
protected Map<String, Object> configs;
protected Map<String, String> contextFields = new HashMap<String, String>();
- public MetricCount statMetric = new MetricCount();
-
- /**
- *
- */
+ public MetricData statMetric = new MetricData(getStatMetricName(), false);
+ protected String getStatMetricName() {
+ return null;
+ }
+
public ConfigBlock() {
- super();
}
/**
@@ -58,10 +58,7 @@ public abstract class ConfigBlock {
return this.getClass().getSimpleName();
}
- /**
- * @param metricsList
- */
- public void addMetricsContainers(List<MetricCount> metricsList) {
+ public void addMetricsContainers(List<MetricData> metricsList) {
metricsList.add(statMetric);
}
@@ -89,25 +86,21 @@ public abstract class ConfigBlock {
boolean isEnabled = getBooleanValue("is_enabled", true);
if (isEnabled) {
// Let's check for static conditions
- Map<String, Object> conditions = (Map<String, Object>) configs
- .get("conditions");
+ Map<String, Object> conditions = (Map<String, Object>) configs.get("conditions");
boolean allow = true;
- if (conditions != null && conditions.size() > 0) {
+ if (MapUtils.isNotEmpty(conditions)) {
allow = false;
for (String conditionType : conditions.keySet()) {
if (conditionType.equalsIgnoreCase("fields")) {
- Map<String, Object> fields = (Map<String, Object>) conditions
- .get("fields");
+ Map<String, Object> fields = (Map<String, Object>) conditions.get("fields");
for (String fieldName : fields.keySet()) {
Object values = fields.get(fieldName);
if (values instanceof String) {
- allow = isFieldConditionMatch(fieldName,
- (String) values);
+ allow = isFieldConditionMatch(fieldName, (String) values);
} else {
List<String> listValues = (List<String>) values;
for (String stringValue : listValues) {
- allow = isFieldConditionMatch(fieldName,
- stringValue);
+ allow = isFieldConditionMatch(fieldName, stringValue);
if (allow) {
break;
}
@@ -135,8 +128,7 @@ public abstract class ConfigBlock {
allow = true;
} else {
@SuppressWarnings("unchecked")
- Map<String, Object> addFields = (Map<String, Object>) configs
- .get("add_fields");
+ Map<String, Object> addFields = (Map<String, Object>) configs.get("add_fields");
if (addFields != null && addFields.get(fieldName) != null) {
String addFieldValue = (String) addFields.get(fieldName);
if (stringValue.equalsIgnoreCase(addFieldValue)) {
@@ -184,12 +176,7 @@ public abstract class ConfigBlock {
String strValue = getStringValue(key);
boolean retValue = defaultValue;
if (!StringUtils.isEmpty(strValue)) {
- if (strValue.equalsIgnoreCase("true")
- || strValue.equalsIgnoreCase("yes")) {
- retValue = true;
- } else {
- retValue = false;
- }
+ retValue = (strValue.equalsIgnoreCase("true") || strValue.equalsIgnoreCase("yes"));
}
return retValue;
}
@@ -201,8 +188,7 @@ public abstract class ConfigBlock {
try {
retValue = Integer.parseInt(strValue);
} catch (Throwable t) {
- logger.error("Error parsing integer value. key=" + key
- + ", value=" + strValue);
+ LOG.error("Error parsing integer value. key=" + key + ", value=" + strValue);
}
}
return retValue;
@@ -215,8 +201,7 @@ public abstract class ConfigBlock {
try {
retValue = Long.parseLong(strValue);
} catch (Throwable t) {
- logger.error("Error parsing long value. key=" + key + ", value="
- + strValue);
+ LOG.error("Error parsing long value. key=" + key + ", value=" + strValue);
}
}
return retValue;
@@ -227,29 +212,27 @@ public abstract class ConfigBlock {
}
public void incrementStat(int count) {
- statMetric.count += count;
+ statMetric.value += count;
}
- public void logStatForMetric(MetricCount metric, String prefixStr) {
- LogFeederUtil.logStatForMetric(metric, prefixStr, ", key="
- + getShortDescription());
+ public void logStatForMetric(MetricData metric, String prefixStr) {
+ LogFeederUtil.logStatForMetric(metric, prefixStr, ", key=" + getShortDescription());
}
- synchronized public void logStat() {
+ public synchronized void logStat() {
logStatForMetric(statMetric, "Stat");
}
public boolean logConfgs(Priority level) {
- if (level.toInt() == Priority.INFO_INT && !logger.isInfoEnabled()) {
+ if (level.toInt() == Priority.INFO_INT && !LOG.isInfoEnabled()) {
return false;
}
- if (level.toInt() == Priority.DEBUG_INT && !logger.isDebugEnabled()) {
+ if (level.toInt() == Priority.DEBUG_INT && !LOG.isDebugEnabled()) {
return false;
}
- logger.log(level, "Printing configuration Block="
- + getShortDescription());
- logger.log(level, "configs=" + configs);
- logger.log(level, "contextFields=" + contextFields);
+ LOG.log(level, "Printing configuration Block=" + getShortDescription());
+ LOG.log(level, "configs=" + configs);
+ LOG.log(level, "contextFields=" + contextFields);
return true;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
new file mode 100644
index 0000000..d1e7fba
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.common;
+
+public class LogFeederConstants {
+
+ public static final String ALL = "all";
+ public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config";
+ public static final String LOG_LEVEL_UNKNOWN = "UNKNOWN";
+
+ // solr fields
+ public static final String SOLR_LEVEL = "level";
+ public static final String SOLR_COMPONENT = "type";
+ public static final String SOLR_HOST = "host";
+
+ // UserConfig Constants History
+ public static final String VALUES = "jsons";
+ public static final String ROW_TYPE = "rowtype";
+
+ // S3 Constants
+ public static final String S3_PATH_START_WITH = "s3://";
+ public static final String S3_PATH_SEPARATOR = "/";
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
index ab371f1..684f3c4 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/Filter.java
@@ -29,21 +29,19 @@ import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.Input;
import org.apache.ambari.logfeeder.input.InputMarker;
import org.apache.ambari.logfeeder.mapper.Mapper;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
-import org.apache.ambari.logfeeder.output.OutputMgr;
+import org.apache.ambari.logfeeder.metrics.MetricData;
+import org.apache.ambari.logfeeder.output.OutputManager;
import org.apache.ambari.logfeeder.util.AliasUtil;
-import org.apache.ambari.logfeeder.util.LogFeederUtil;
-import org.apache.ambari.logfeeder.util.AliasUtil.ALIAS_PARAM;
-import org.apache.ambari.logfeeder.util.AliasUtil.ALIAS_TYPE;
+import org.apache.ambari.logfeeder.util.AliasUtil.AliasType;
import org.apache.log4j.Logger;
import org.apache.log4j.Priority;
public abstract class Filter extends ConfigBlock {
- private static final Logger logger = Logger.getLogger(Filter.class);
+ private static final Logger LOG = Logger.getLogger(Filter.class);
protected Input input;
private Filter nextFilter = null;
- private OutputMgr outputMgr;
+ private OutputManager outputManager;
private Map<String, List<Mapper>> postFieldValueMappers = new HashMap<String, List<Mapper>>();
@@ -74,15 +72,12 @@ public abstract class Filter extends ConfigBlock {
}
for (Map<String, Object> mapObject : mapList) {
for (String mapClassCode : mapObject.keySet()) {
- Mapper mapper = getMapper(mapClassCode);
+ Mapper mapper = (Mapper) AliasUtil.getClassInstance(mapClassCode, AliasType.MAPPER);
if (mapper == null) {
break;
}
- if (mapper.init(getInput().getShortDescription(),
- fieldName, mapClassCode,
- mapObject.get(mapClassCode))) {
- List<Mapper> fieldMapList = postFieldValueMappers
- .get(fieldName);
+ if (mapper.init(getInput().getShortDescription(), fieldName, mapClassCode, mapObject.get(mapClassCode))) {
+ List<Mapper> fieldMapList = postFieldValueMappers.get(fieldName);
if (fieldMapList == null) {
fieldMapList = new ArrayList<Mapper>();
postFieldValueMappers.put(fieldName, fieldMapList);
@@ -94,17 +89,8 @@ public abstract class Filter extends ConfigBlock {
}
}
- private Mapper getMapper(String mapClassCode) {
- String classFullName = AliasUtil.getInstance().readAlias(mapClassCode, ALIAS_TYPE.MAPPER, ALIAS_PARAM.KLASS);
- if (classFullName != null && !classFullName.isEmpty()) {
- Mapper mapper = (Mapper) LogFeederUtil.getClassInstance(classFullName, ALIAS_TYPE.MAPPER);
- return mapper;
- }
- return null;
- }
-
- public void setOutputMgr(OutputMgr outputMgr) {
- this.outputMgr = outputMgr;
+ public void setOutputManager(OutputManager outputManager) {
+ this.outputManager = outputManager;
}
public Filter getNextFilter() {
@@ -131,25 +117,23 @@ public abstract class Filter extends ConfigBlock {
if (nextFilter != null) {
nextFilter.apply(inputStr, inputMarker);
} else {
- outputMgr.write(inputStr, inputMarker);
+ outputManager.write(inputStr, inputMarker);
}
}
public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) throws LogfeederException {
- if (postFieldValueMappers.size() > 0) {
- for (String fieldName : postFieldValueMappers.keySet()) {
- Object value = jsonObj.get(fieldName);
- if (value != null) {
- for (Mapper mapper : postFieldValueMappers.get(fieldName)) {
- value = mapper.apply(jsonObj, value);
- }
+ for (String fieldName : postFieldValueMappers.keySet()) {
+ Object value = jsonObj.get(fieldName);
+ if (value != null) {
+ for (Mapper mapper : postFieldValueMappers.get(fieldName)) {
+ value = mapper.apply(jsonObj, value);
}
}
}
if (nextFilter != null) {
nextFilter.apply(jsonObj, inputMarker);
} else {
- outputMgr.write(jsonObj, inputMarker);
+ outputManager.write(jsonObj, inputMarker);
}
}
@@ -193,16 +177,15 @@ public abstract class Filter extends ConfigBlock {
if (!super.logConfgs(level)) {
return false;
}
- logger.log(level, "input=" + input.getShortDescription());
+ LOG.log(level, "input=" + input.getShortDescription());
return true;
}
@Override
- public void addMetricsContainers(List<MetricCount> metricsList) {
+ public void addMetricsContainers(List<MetricData> metricsList) {
super.addMetricsContainers(metricsList);
if (nextFilter != null) {
nextFilter.addMetricsContainers(metricsList);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
index 372c208..7e2da70 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java
@@ -36,7 +36,7 @@ import oi.thekraken.grok.api.exception.GrokException;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
+import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
@@ -45,7 +45,7 @@ import org.apache.log4j.Logger;
import com.google.gson.reflect.TypeToken;
public class FilterGrok extends Filter {
- static private Logger logger = Logger.getLogger(FilterGrok.class);
+ private static final Logger LOG = Logger.getLogger(FilterGrok.class);
private static final String GROK_PATTERN_FILE = "grok-patterns";
@@ -68,25 +68,23 @@ public class FilterGrok extends Filter {
private Type jsonType = new TypeToken<Map<String, String>>() {}.getType();
- private MetricCount grokErrorMetric = new MetricCount();
+ private MetricData grokErrorMetric = new MetricData("filter.error.grok", false);
@Override
public void init() throws Exception {
super.init();
try {
- grokErrorMetric.metricsName = "filter.error.grok";
messagePattern = escapePattern(getStringValue("message_pattern"));
multilinePattern = escapePattern(getStringValue("multiline_pattern"));
sourceField = getStringValue("source_field");
removeSourceField = getBooleanValue("remove_source_field",
removeSourceField);
- logger.info("init() done. grokPattern=" + messagePattern
- + ", multilinePattern=" + multilinePattern + ", "
- + getShortDescription());
+ LOG.info("init() done. grokPattern=" + messagePattern + ", multilinePattern=" + multilinePattern + ", " +
+ getShortDescription());
if (StringUtils.isEmpty(messagePattern)) {
- logger.error("message_pattern is not set for filter.");
+ LOG.error("message_pattern is not set for filter.");
return;
}
extractNamedParams(messagePattern, namedParamList);
@@ -102,9 +100,7 @@ public class FilterGrok extends Filter {
grokMultiline.compile(multilinePattern);
}
} catch (Throwable t) {
- logger.fatal(
- "Caught exception while initializing Grok. multilinePattern="
- + multilinePattern + ", messagePattern="
+ LOG.fatal("Caught exception while initializing Grok. multilinePattern=" + multilinePattern + ", messagePattern="
+ messagePattern, t);
grokMessage = null;
grokMultiline = null;
@@ -123,9 +119,10 @@ public class FilterGrok extends Filter {
}
private void extractNamedParams(String patternStr, Set<String> paramList) {
- String grokRegEx = "%\\{" + "(?<name>" + "(?<pattern>[A-z0-9]+)"
- + "(?::(?<subname>[A-z0-9_:]+))?" + ")" + "(?:=(?<definition>"
- + "(?:" + "(?:[^{}]+|\\.+)+" + ")+" + ")" + ")?" + "\\}";
+ String grokRegEx = "%\\{" +
+ "(?<name>" + "(?<pattern>[A-z0-9]+)" + "(?::(?<subname>[A-z0-9_:]+))?" + ")" +
+ "(?:=(?<definition>" + "(?:" + "(?:[^{}]+|\\.+)+" + ")+" + ")" + ")?" +
+ "\\}";
Pattern pattern = Pattern.compile(grokRegEx);
java.util.regex.Matcher matcher = pattern.matcher(patternStr);
@@ -139,28 +136,23 @@ public class FilterGrok extends Filter {
private boolean loadPatterns(Grok grok) {
InputStreamReader grokPatternsReader = null;
- logger.info("Loading pattern file " + GROK_PATTERN_FILE);
+ LOG.info("Loading pattern file " + GROK_PATTERN_FILE);
try {
- BufferedInputStream fileInputStream = (BufferedInputStream) this
- .getClass().getClassLoader()
- .getResourceAsStream(GROK_PATTERN_FILE);
+ BufferedInputStream fileInputStream =
+ (BufferedInputStream) this.getClass().getClassLoader().getResourceAsStream(GROK_PATTERN_FILE);
if (fileInputStream == null) {
- logger.fatal("Couldn't load grok-patterns file "
- + GROK_PATTERN_FILE + ". Things will not work");
+ LOG.fatal("Couldn't load grok-patterns file " + GROK_PATTERN_FILE + ". Things will not work");
return false;
}
grokPatternsReader = new InputStreamReader(fileInputStream);
} catch (Throwable t) {
- logger.fatal("Error reading grok-patterns file " + GROK_PATTERN_FILE
- + " from classpath. Grok filtering will not work.", t);
+ LOG.fatal("Error reading grok-patterns file " + GROK_PATTERN_FILE + " from classpath. Grok filtering will not work.", t);
return false;
}
try {
grok.addPatternFromReader(grokPatternsReader);
} catch (GrokException e) {
- logger.fatal(
- "Error loading patterns from grok-patterns reader for file "
- + GROK_PATTERN_FILE, e);
+ LOG.fatal("Error loading patterns from grok-patterns reader for file " + GROK_PATTERN_FILE, e);
return false;
}
@@ -177,8 +169,7 @@ public class FilterGrok extends Filter {
String jsonStr = grokMultiline.capture(inputStr);
if (!"{}".equals(jsonStr)) {
if (strBuff != null) {
- Map<String, Object> jsonObj = Collections
- .synchronizedMap(new HashMap<String, Object>());
+ Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>());
try {
applyMessage(strBuff.toString(), jsonObj, currMultilineJsonStr);
} finally {
@@ -192,15 +183,13 @@ public class FilterGrok extends Filter {
if (strBuff == null) {
strBuff = new StringBuilder();
} else {
- strBuff.append('\r');
- strBuff.append('\n');
+ strBuff.append("\r\n");
}
strBuff.append(inputStr);
savedInputMarker = inputMarker;
} else {
savedInputMarker = inputMarker;
- Map<String, Object> jsonObj = Collections
- .synchronizedMap(new HashMap<String, Object>());
+ Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>());
applyMessage(inputStr, jsonObj, null);
}
}
@@ -216,14 +205,8 @@ public class FilterGrok extends Filter {
}
}
- /**
- * @param inputStr
- * @param jsonObj
- * @throws LogfeederException
- */
- private void applyMessage(String inputStr, Map<String, Object> jsonObj,
- String multilineJsonStr) throws LogfeederException {
- String jsonStr = grokParse(inputStr);
+ private void applyMessage(String inputStr, Map<String, Object> jsonObj, String multilineJsonStr) throws LogfeederException {
+ String jsonStr = grokMessage.capture(inputStr);
boolean parseError = false;
if ("{}".equals(jsonStr)) {
@@ -239,8 +222,7 @@ public class FilterGrok extends Filter {
if (parseError) {
jsonStr = multilineJsonStr;
}
- Map<String, String> jsonSrc = LogFeederUtil.getGson().fromJson(jsonStr,
- jsonType);
+ Map<String, String> jsonSrc = LogFeederUtil.getGson().fromJson(jsonStr, jsonType);
for (String namedParam : namedParamList) {
if (jsonSrc.get(namedParam) != null) {
jsonObj.put(namedParam, jsonSrc.get(namedParam));
@@ -260,37 +242,26 @@ public class FilterGrok extends Filter {
}
}
super.apply(jsonObj, savedInputMarker);
- statMetric.count++;
- }
-
- public String grokParse(String inputStr) {
- String jsonStr = grokMessage.capture(inputStr);
- return jsonStr;
+ statMetric.value++;
}
private void logParseError(String inputStr) {
- grokErrorMetric.count++;
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_PARSEERROR";
+ grokErrorMetric.value++;
+ String logMessageKey = this.getClass().getSimpleName() + "_PARSEERROR";
int inputStrLength = inputStr != null ? inputStr.length() : 0;
- LogFeederUtil.logErrorMessageByInterval(
- LOG_MESSAGE_KEY,
- "Error parsing string. length=" + inputStrLength
- + ", input=" + input.getShortDescription()
- + ". First upto 100 characters="
- + LogFeederUtil.subString(inputStr, 100), null, logger,
- Level.WARN);
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Error parsing string. length=" + inputStrLength + ", input=" +
+ input.getShortDescription() + ". First upto 100 characters=" + StringUtils.abbreviate(inputStr, 100), null, LOG,
+ Level.WARN);
}
@Override
public void flush() {
if (strBuff != null) {
- Map<String, Object> jsonObj = Collections
- .synchronizedMap(new HashMap<String, Object>());
+ Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>());
try {
applyMessage(strBuff.toString(), jsonObj, currMultilineJsonStr);
} catch (LogfeederException e) {
- logger.error(e.getLocalizedMessage(), e.getCause());
+ LOG.error(e.getLocalizedMessage(), e.getCause());
}
strBuff = null;
savedInputMarker = null;
@@ -304,7 +275,7 @@ public class FilterGrok extends Filter {
}
@Override
- public void addMetricsContainers(List<MetricCount> metricsList) {
+ public void addMetricsContainers(List<MetricData> metricsList) {
super.addMetricsContainers(metricsList);
metricsList.add(grokErrorMetric);
}
@@ -314,5 +285,4 @@ public class FilterGrok extends Filter {
super.logStat();
logStatForMetric(grokErrorMetric, "Stat: Grok Errors");
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
index 2954106..ba63c61 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterJSON.java
@@ -22,12 +22,13 @@ import java.util.Map;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
+import org.apache.ambari.logfeeder.util.DateUtil;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.log4j.Logger;
public class FilterJSON extends Filter {
- private static final Logger logger = Logger.getLogger(FilterJSON.class);
+ private static final Logger LOG = Logger.getLogger(FilterJSON.class);
@Override
public void apply(String inputStr, InputMarker inputMarker) throws LogfeederException {
@@ -35,7 +36,7 @@ public class FilterJSON extends Filter {
try {
jsonMap = LogFeederUtil.toJSONObject(inputStr);
} catch (Exception e) {
- logger.error(e.getLocalizedMessage());
+ LOG.error(e.getLocalizedMessage());
throw new LogfeederException("Json parsing failed for inputstr = " + inputStr ,e.getCause());
}
Double lineNumberD = (Double) jsonMap.get("line_number");
@@ -45,10 +46,9 @@ public class FilterJSON extends Filter {
}
String timeStampStr = (String) jsonMap.get("logtime");
if (timeStampStr != null && !timeStampStr.isEmpty()) {
- String logtime = LogFeederUtil.getDate(timeStampStr);
+ String logtime = DateUtil.getDate(timeStampStr);
jsonMap.put("logtime", logtime);
}
super.apply(jsonMap, inputMarker);
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
index 7adb468..c9c3f2c 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
@@ -25,38 +25,35 @@ import java.util.StringTokenizer;
import org.apache.ambari.logfeeder.common.LogfeederException;
import org.apache.ambari.logfeeder.input.InputMarker;
-import org.apache.ambari.logfeeder.metrics.MetricCount;
+import org.apache.ambari.logfeeder.metrics.MetricData;
import org.apache.ambari.logfeeder.util.LogFeederUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
public class FilterKeyValue extends Filter {
- private static final Logger logger = Logger.getLogger(FilterKeyValue.class);
+ private static final Logger LOG = Logger.getLogger(FilterKeyValue.class);
private String sourceField = null;
private String valueSplit = "=";
private String fieldSplit = "\t";
- private MetricCount errorMetric = new MetricCount();
+ private MetricData errorMetric = new MetricData("filter.error.keyvalue", false);
@Override
public void init() throws Exception {
super.init();
- errorMetric.metricsName = "filter.error.keyvalue";
sourceField = getStringValue("source_field");
valueSplit = getStringValue("value_split", valueSplit);
fieldSplit = getStringValue("field_split", fieldSplit);
- logger.info("init() done. source_field=" + sourceField
- + ", value_split=" + valueSplit + ", " + ", field_split="
- + fieldSplit + ", " + getShortDescription());
+ LOG.info("init() done. source_field=" + sourceField + ", value_split=" + valueSplit + ", " + ", field_split=" +
+ fieldSplit + ", " + getShortDescription());
if (StringUtils.isEmpty(sourceField)) {
- logger.fatal("source_field is not set for filter. This filter will not be applied");
+ LOG.fatal("source_field is not set for filter. This filter will not be applied");
return;
}
-
}
@Override
@@ -71,40 +68,30 @@ public class FilterKeyValue extends Filter {
}
Object valueObj = jsonObj.get(sourceField);
if (valueObj != null) {
- StringTokenizer fieldTokenizer = new StringTokenizer(
- valueObj.toString(), fieldSplit);
+ StringTokenizer fieldTokenizer = new StringTokenizer(valueObj.toString(), fieldSplit);
while (fieldTokenizer.hasMoreTokens()) {
String nv = fieldTokenizer.nextToken();
- StringTokenizer nvTokenizer = new StringTokenizer(nv,
- valueSplit);
+ StringTokenizer nvTokenizer = new StringTokenizer(nv, valueSplit);
while (nvTokenizer.hasMoreTokens()) {
String name = nvTokenizer.nextToken();
if (nvTokenizer.hasMoreTokens()) {
String value = nvTokenizer.nextToken();
jsonObj.put(name, value);
} else {
- logParseError("name=" + name + ", pair=" + nv
- + ", field=" + sourceField + ", field_value="
- + valueObj);
+ logParseError("name=" + name + ", pair=" + nv + ", field=" + sourceField + ", field_value=" + valueObj);
}
}
}
}
super.apply(jsonObj, inputMarker);
- statMetric.count++;
+ statMetric.value++;
}
private void logParseError(String inputStr) {
- errorMetric.count++;
- final String LOG_MESSAGE_KEY = this.getClass().getSimpleName()
- + "_PARSEERROR";
- LogFeederUtil
- .logErrorMessageByInterval(
- LOG_MESSAGE_KEY,
- "Error parsing string. length=" + inputStr.length()
- + ", input=" + input.getShortDescription()
- + ". First upto 100 characters="
- + LogFeederUtil.subString(inputStr, 100), null, logger,
+ errorMetric.value++;
+ String logMessageKey = this.getClass().getSimpleName() + "_PARSEERROR";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Error parsing string. length=" + inputStr.length() + ", input=" +
+ input.getShortDescription() + ". First upto 100 characters=" + StringUtils.abbreviate(inputStr, 100), null, LOG,
Level.ERROR);
}
@@ -114,9 +101,8 @@ public class FilterKeyValue extends Filter {
}
@Override
- public void addMetricsContainers(List<MetricCount> metricsList) {
+ public void addMetricsContainers(List<MetricData> metricsList) {
super.addMetricsContainers(metricsList);
metricsList.add(errorMetric);
}
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8399960d/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
new file mode 100644
index 0000000..41a1fa5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/AbstractInputFile.java
@@ -0,0 +1,319 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.input;
+
+import java.io.BufferedReader;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.logfeeder.util.LogFeederUtil;
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+public abstract class AbstractInputFile extends Input {
+ protected static final Logger LOG = Logger.getLogger(AbstractInputFile.class);
+
+ private static final int DEFAULT_CHECKPOINT_INTERVAL_MS = 5 * 1000;
+
+ protected File[] logFiles;
+ protected String logPath;
+ protected Object fileKey;
+ protected String base64FileKey;
+
+ protected boolean isReady;
+ private boolean isStartFromBegining = true;
+
+ private String checkPointExtension;
+ private File checkPointFile;
+ private RandomAccessFile checkPointWriter;
+ private long lastCheckPointTimeMS;
+ private int checkPointIntervalMS;
+ private Map<String, Object> jsonCheckPoint;
+ private InputMarker lastCheckPointInputMarker;
+
+ @Override
+ protected String getStatMetricName() {
+ return "input.files.read_lines";
+ }
+
+ @Override
+ protected String getReadBytesMetricName() {
+ return "input.files.read_bytes";
+ }
+
+ @Override
+ public void init() throws Exception {
+ LOG.info("init() called");
+
+ checkPointExtension = LogFeederUtil.getStringProperty("logfeeder.checkpoint.extension", InputManager.DEFAULT_CHECKPOINT_EXTENSION);
+
+ // Let's close the file and set it to true after we start monitoring it
+ setClosed(true);
+ logPath = getStringValue("path");
+ tail = getBooleanValue("tail", tail);
+ checkPointIntervalMS = getIntValue("checkpoint.interval.ms", DEFAULT_CHECKPOINT_INTERVAL_MS);
+
+ if (StringUtils.isEmpty(logPath)) {
+ LOG.error("path is empty for file input. " + getShortDescription());
+ return;
+ }
+
+ String startPosition = getStringValue("start_position");
+ if (StringUtils.isEmpty(startPosition) || startPosition.equalsIgnoreCase("beginning") ||
+ startPosition.equalsIgnoreCase("begining") || !tail) {
+ isStartFromBegining = true;
+ }
+
+ setFilePath(logPath);
+ boolean isFileReady = isReady();
+
+ LOG.info("File to monitor " + logPath + ", tail=" + tail + ", isReady=" + isFileReady);
+
+ super.init();
+ }
+
+ protected void processFile(File logPathFile) throws FileNotFoundException, IOException {
+ LOG.info("Monitoring logPath=" + logPath + ", logPathFile=" + logPathFile);
+ BufferedReader br = null;
+ checkPointFile = null;
+ checkPointWriter = null;
+ jsonCheckPoint = null;
+
+ int lineCount = 0;
+ try {
+ setFilePath(logPathFile.getAbsolutePath());
+
+ br = openLogFile(logPathFile);
+
+ boolean resume = isStartFromBegining;
+ int resumeFromLineNumber = getResumeFromLineNumber();
+ if (resumeFromLineNumber > 0) {
+ resume = false;
+ }
+
+ setClosed(false);
+ int sleepStep = 2;
+ int sleepIteration = 0;
+ while (true) {
+ try {
+ if (isDrain()) {
+ break;
+ }
+
+ String line = br.readLine();
+ if (line == null) {
+ if (!resume) {
+ resume = true;
+ }
+ sleepIteration++;
+ if (sleepIteration == 2) {
+ flush();
+ if (!tail) {
+ LOG.info("End of file. Done with filePath=" + logPathFile.getAbsolutePath() + ", lineCount=" + lineCount);
+ break;
+ }
+ } else if (sleepIteration > 4) {
+ Object newFileKey = getFileKey(logPathFile);
+ if (newFileKey != null && (fileKey == null || !newFileKey.equals(fileKey))) {
+ LOG.info("File key is different. Marking this input file for rollover. oldKey=" + fileKey + ", newKey=" +
+ newFileKey + ". " + getShortDescription());
+
+ try {
+ LOG.info("File is rolled over. Closing current open file." + getShortDescription() + ", lineCount=" +
+ lineCount);
+ br.close();
+ } catch (Exception ex) {
+ LOG.error("Error closing file" + getShortDescription(), ex);
+ break;
+ }
+
+ try {
+ LOG.info("Opening new rolled over file." + getShortDescription());
+ br = openLogFile(logPathFile);
+ lineCount = 0;
+ } catch (Exception ex) {
+ LOG.error("Error opening rolled over file. " + getShortDescription(), ex);
+ LOG.info("Added input to not ready list." + getShortDescription());
+ isReady = false;
+ inputManager.addToNotReady(this);
+ break;
+ }
+ LOG.info("File is successfully rolled over. " + getShortDescription());
+ continue;
+ }
+ }
+ try {
+ Thread.sleep(sleepStep * 1000);
+ sleepStep = Math.min(sleepStep * 2, 10);
+ } catch (InterruptedException e) {
+ LOG.info("Thread interrupted." + getShortDescription());
+ }
+ } else {
+ lineCount++;
+ sleepStep = 1;
+ sleepIteration = 0;
+
+ if (!resume && lineCount > resumeFromLineNumber) {
+ LOG.info("Resuming to read from last line. lineCount=" + lineCount + ", input=" + getShortDescription());
+ resume = true;
+ }
+ if (resume) {
+ InputMarker marker = new InputMarker(this, base64FileKey, lineCount);
+ outputLine(line, marker);
+ }
+ }
+ } catch (Throwable t) {
+ String logMessageKey = this.getClass().getSimpleName() + "_READ_LOOP_EXCEPTION";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Caught exception in read loop. lineNumber=" + lineCount +
+ ", input=" + getShortDescription(), t, LOG, Level.ERROR);
+ }
+ }
+ } finally {
+ if (br != null) {
+ LOG.info("Closing reader." + getShortDescription() + ", lineCount=" + lineCount);
+ try {
+ br.close();
+ } catch (Throwable t) {
+ // ignore
+ }
+ }
+ }
+ }
+
+ protected abstract BufferedReader openLogFile(File logFile) throws IOException;
+
+ protected abstract Object getFileKey(File logFile);
+
+ private int getResumeFromLineNumber() {
+ int resumeFromLineNumber = 0;
+
+ if (tail) {
+ try {
+ LOG.info("Checking existing checkpoint file. " + getShortDescription());
+
+ String checkPointFileName = base64FileKey + checkPointExtension;
+ File checkPointFolder = inputManager.getCheckPointFolderFile();
+ checkPointFile = new File(checkPointFolder, checkPointFileName);
+ checkPointWriter = new RandomAccessFile(checkPointFile, "rw");
+
+ try {
+ int contentSize = checkPointWriter.readInt();
+ byte b[] = new byte[contentSize];
+ int readSize = checkPointWriter.read(b, 0, contentSize);
+ if (readSize != contentSize) {
+ LOG.error("Couldn't read expected number of bytes from checkpoint file. expected=" + contentSize + ", read=" +
+ readSize + ", checkPointFile=" + checkPointFile + ", input=" + getShortDescription());
+ } else {
+ String jsonCheckPointStr = new String(b, 0, readSize);
+ jsonCheckPoint = LogFeederUtil.toJSONObject(jsonCheckPointStr);
+
+ resumeFromLineNumber = LogFeederUtil.objectToInt(jsonCheckPoint.get("line_number"), 0, "line_number");
+
+ LOG.info("CheckPoint. checkPointFile=" + checkPointFile + ", json=" + jsonCheckPointStr +
+ ", resumeFromLineNumber=" + resumeFromLineNumber);
+ }
+ } catch (EOFException eofEx) {
+ LOG.info("EOFException. Will reset checkpoint file " + checkPointFile.getAbsolutePath() + " for " +
+ getShortDescription());
+ }
+ if (jsonCheckPoint == null) {
+ // This seems to be first time, so creating the initial checkPoint object
+ jsonCheckPoint = new HashMap<String, Object>();
+ jsonCheckPoint.put("file_path", filePath);
+ jsonCheckPoint.put("file_key", base64FileKey);
+ }
+
+ } catch (Throwable t) {
+ LOG.error("Error while configuring checkpoint file. Will reset file. checkPointFile=" + checkPointFile, t);
+ }
+ }
+
+ return resumeFromLineNumber;
+ }
+
+ @Override
+ public synchronized void checkIn(InputMarker inputMarker) {
+ if (checkPointWriter != null) {
+ try {
+ int lineNumber = LogFeederUtil.objectToInt(jsonCheckPoint.get("line_number"), 0, "line_number");
+ if (lineNumber > inputMarker.lineNumber) {
+ // Already wrote higher line number for this input
+ return;
+ }
+ // If interval is greater than last checkPoint time, then write
+ long currMS = System.currentTimeMillis();
+ if (!isClosed() && (currMS - lastCheckPointTimeMS) < checkPointIntervalMS) {
+ // Let's save this one so we can update the check point file on flush
+ lastCheckPointInputMarker = inputMarker;
+ return;
+ }
+ lastCheckPointTimeMS = currMS;
+
+ jsonCheckPoint.put("line_number", "" + new Integer(inputMarker.lineNumber));
+ jsonCheckPoint.put("last_write_time_ms", "" + new Long(currMS));
+ jsonCheckPoint.put("last_write_time_date", new Date());
+
+ String jsonStr = LogFeederUtil.getGson().toJson(jsonCheckPoint);
+
+ // Let's rewind
+ checkPointWriter.seek(0);
+ checkPointWriter.writeInt(jsonStr.length());
+ checkPointWriter.write(jsonStr.getBytes());
+
+ if (isClosed()) {
+ String logMessageKey = this.getClass().getSimpleName() + "_FINAL_CHECKIN";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Wrote final checkPoint, input=" + getShortDescription() +
+ ", checkPointFile=" + checkPointFile.getAbsolutePath() + ", checkPoint=" + jsonStr, null, LOG, Level.INFO);
+ }
+ } catch (Throwable t) {
+ String logMessageKey = this.getClass().getSimpleName() + "_CHECKIN_EXCEPTION";
+ LogFeederUtil.logErrorMessageByInterval(logMessageKey, "Caught exception checkIn. , input=" + getShortDescription(), t,
+ LOG, Level.ERROR);
+ }
+ }
+ }
+
+ @Override
+ public void lastCheckIn() {
+ if (lastCheckPointInputMarker != null) {
+ checkIn(lastCheckPointInputMarker);
+ }
+ }
+
+ @Override
+ public void close() {
+ super.close();
+ LOG.info("close() calling checkPoint checkIn(). " + getShortDescription());
+ lastCheckIn();
+ }
+
+ @Override
+ public String getShortDescription() {
+ return "input:source=" + getStringValue("source") + ", path=" +
+ (!ArrayUtils.isEmpty(logFiles) ? logFiles[0].getAbsolutePath() : logPath);
+ }
+}
[08/10] ambari git commit: AMBARI-18310. Refactor logsearch portal
side code - Part 2 (oleewere)
Posted by ol...@apache.org.
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java
deleted file mode 100644
index 53e2ca2..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/metrics/SolrMetricsLoader.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.solr.metrics;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.TreeMap;
-
-import javax.management.MalformedObjectNameException;
-
-import org.apache.ambari.logsearch.common.PropertiesHelper;
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient;
-import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
-import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class SolrMetricsLoader extends TimerTask {
- private static final Logger LOG = LoggerFactory.getLogger(SolrMetricsLoader.class);
-
- private static final int RETRY = 3;
- private static final int MAX_METRIC_SIZE= 1000;
-
- private final String solrHost;
- private final SolrJmxAdapter solrJmxAdapter;
- private final SolrAmsClient solrAmsClient;
-
- private final TimelineMetrics metrics = new TimelineMetrics();
-
- public SolrMetricsLoader(String solrHost, int solrJmxPort, String collectorHost) throws IOException {
- this.solrHost = solrHost;
- this.solrJmxAdapter = new SolrJmxAdapter(solrHost, solrJmxPort);
- this.solrAmsClient = new SolrAmsClient(collectorHost);
-
- solrJmxAdapter.connect();
- }
-
- @Override
- public void run() {
- LOG.info("Loading Solr Metrics for the host " + solrHost);
-
- addCpuUsageMetric();
- addHeapMemoryUsageMetric();
- addIndexSizeMetric();
-
- emitMetrics();
- removeOverTheLimitMetrics();
- }
-
- private void addCpuUsageMetric() {
- Exception lastException = null;
- for (int retries = 0; retries < RETRY; retries++) {
- try {
- double processCpuLoad = solrJmxAdapter.getProcessCpuLoad();
- addMetric("logsearch.solr.cpu.usage", "Float", processCpuLoad);
- return;
- } catch (MalformedObjectNameException e) {
- lastException = e;
- try {
- solrJmxAdapter.reConnect();
- } catch (IOException e1) {
- }
- }
- }
-
- LOG.info("Could not load solr cpu usage metric, last exception:", lastException);
- }
-
- private void addHeapMemoryUsageMetric() {
- Exception lastException = null;
- for (int retries = 0; retries < RETRY; retries++) {
- try {
- Map<String, Long> memoryData = solrJmxAdapter.getMemoryData();
- addMetric("jvm.JvmMetrics.MemHeapUsedM", "Long", memoryData.get("heapMemoryUsed").doubleValue() / 1024 / 1024);
- addMetric("jvm.JvmMetrics.MemHeapCommittedM", "Long", memoryData.get("heapMemoryCommitted").doubleValue() / 1024 / 1024);
- addMetric("jvm.JvmMetrics.MemHeapMaxM", "Long", memoryData.get("heapMemoryMax").doubleValue() / 1024 / 1024);
- addMetric("jvm.JvmMetrics.MemNonHeapUsedM", "Long", memoryData.get("nonHeapMemoryUsed").doubleValue() / 1024 / 1024);
- addMetric("jvm.JvmMetrics.MemNonHeapCommittedM", "Long", memoryData.get("nonHeapMemoryCommitted").doubleValue() / 1024 / 1024);
- addMetric("jvm.JvmMetrics.MemNonHeapMaxM", "Long", memoryData.get("nonHeapMemoryMax").doubleValue() / 1024 / 1024);
- return;
- } catch (MalformedObjectNameException e) {
- lastException = e;
- try {
- solrJmxAdapter.reConnect();
- } catch (IOException e1) {
- }
- }
- }
-
- LOG.info("Could not load solr heap memory usage metric, last exception:", lastException);
- }
-
- private void addIndexSizeMetric() {
- Exception lastException = null;
- for (int retries = 0; retries < RETRY; retries++) {
- try {
- double indexSize = solrJmxAdapter.getIndexSize();
- addMetric("logsearch.solr.index.size", "Long", indexSize / 1024 / 1024 / 1024);
- return;
- } catch (Exception e) {
- lastException = e;
- try {
- solrJmxAdapter.reConnect();
- } catch (IOException e1) {
- }
- }
- }
-
- LOG.info("Could not load solr index size metric, last exception:", lastException);
- }
-
- private void addMetric(String metricName, String type, Double value) {
- Long currMS = System.currentTimeMillis();
-
- TimelineMetric metric = new TimelineMetric();
- metric.setMetricName(metricName);
- metric.setHostName(solrHost);
- metric.setAppId("infra-solr");
- metric.setStartTime(currMS);
- metric.setType(type);
- metric.setTimestamp(currMS);
- metric.getMetricValues().put(currMS, value);
-
- metrics.addOrMergeTimelineMetric(metric);
- }
-
- private void emitMetrics() {
- Exception lastException = null;
- for (int retries = 0; retries < RETRY; retries++) {
- try {
- if (solrAmsClient.emitMetrics(metrics)) {
- metrics.getMetrics().clear();
- return;
- }
- } catch (Exception e) {
- lastException = e;
- }
- }
-
- LOG.info("Could not emit metrics, last exception:", lastException);
- }
-
- private void removeOverTheLimitMetrics() {
- for (TimelineMetric metric : metrics.getMetrics()) {
- TreeMap<Long, Double> metricValues = metric.getMetricValues();
- while (metricValues.size() > MAX_METRIC_SIZE) {
- metricValues.remove(metricValues.firstKey());
- }
- }
- }
-
- public static void startSolrMetricsLoaderTasks() {
- try {
- String collectorHosts = PropertiesHelper.getProperty("logsearch.solr.metrics.collector.hosts");
- if (StringUtils.isEmpty(collectorHosts)) {
- LOG.warn("No Ambari Metrics service is available, no Solr metrics will be loaded!");
- return;
- }
-
- int solrJmxPort = PropertiesHelper.getIntProperty("logsearch.solr.jmx.port");
-
- String zkConnectString = PropertiesHelper.getProperty("logsearch.solr.zk_connect_string");
- AmbariSolrCloudClient ambariSolrCloudClient = new AmbariSolrCloudClientBuilder()
- .withZkConnectString(zkConnectString)
- .build();
-
- Collection<String> solrHosts = ambariSolrCloudClient.getSolrHosts();
- for (String solrHost : solrHosts) {
- SolrMetricsLoader sml = new SolrMetricsLoader(solrHost, solrJmxPort, collectorHosts);
- Timer timer = new Timer("Solr Metrics Loader - " + solrHost, true);
- timer.scheduleAtFixedRate(sml, 0, 10000);
- }
- } catch (Exception e) {
- LOG.warn("Could not start solr metric loader tasks", e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java
index a3f59f7..0cf05a6 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/solr/model/SolrAuditLogData.java
@@ -69,7 +69,7 @@ public class SolrAuditLogData extends SolrCommonLogData implements AuditLogData
private String repo;
@Field("repoType")
- private String repoType;
+ private Integer repoType;
@Field("reqData")
private String requestData;
@@ -188,12 +188,12 @@ public class SolrAuditLogData extends SolrCommonLogData implements AuditLogData
}
@Override
- public String getRepoType() {
+ public Integer getRepoType() {
return repoType;
}
@Override
- public void setRepoType(String repoType) {
+ public void setRepoType(Integer repoType) {
this.repoType = repoType;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/authenticate/LogsearchLogoutSuccessHandler.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/authenticate/LogsearchLogoutSuccessHandler.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/authenticate/LogsearchLogoutSuccessHandler.java
index 5da1c97..c20e383 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/authenticate/LogsearchLogoutSuccessHandler.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/authenticate/LogsearchLogoutSuccessHandler.java
@@ -36,6 +36,6 @@ public class LogsearchLogoutSuccessHandler extends SimpleUrlLogoutSuccessHandler
public void onLogoutSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication)
throws IOException, ServletException {
logger.debug("LogsearchLogoutSuccessHandler ::: onLogoutSuccess");
- response.sendRedirect("/login.html");
+ response.sendRedirect("/index.html");
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuthenticationEntryPoint.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuthenticationEntryPoint.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuthenticationEntryPoint.java
index 1b24c06..1831697 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuthenticationEntryPoint.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchAuthenticationEntryPoint.java
@@ -28,7 +28,7 @@ import org.apache.log4j.Logger;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.web.authentication.LoginUrlAuthenticationEntryPoint;
-class LogsearchAuthenticationEntryPoint extends LoginUrlAuthenticationEntryPoint {
+public class LogsearchAuthenticationEntryPoint extends LoginUrlAuthenticationEntryPoint {
private static final Logger logger = Logger.getLogger(LogsearchAuthenticationEntryPoint.class);
public LogsearchAuthenticationEntryPoint(String loginFormUrl) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java
index 29fd5b2..8cd435b 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchKRBAuthenticationFilter.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHa
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.springframework.security.web.authentication.WebAuthenticationDetails;
+import org.springframework.stereotype.Component;
public class LogsearchKRBAuthenticationFilter extends LogsearchKrbFilter {
private static final Logger logger = LoggerFactory.getLogger(LogsearchKRBAuthenticationFilter.class);
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchSecurityContextFormationFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchSecurityContextFormationFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchSecurityContextFormationFilter.java
index a34fbd0..1320278 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchSecurityContextFormationFilter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchSecurityContextFormationFilter.java
@@ -38,6 +38,7 @@ import org.apache.log4j.Logger;
import org.springframework.security.authentication.AnonymousAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.stereotype.Component;
import org.springframework.web.filter.GenericFilterBean;
public class LogsearchSecurityContextFormationFilter extends GenericFilterBean {
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUsernamePasswordAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUsernamePasswordAuthenticationFilter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUsernamePasswordAuthenticationFilter.java
index 2c83001..85688a2 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUsernamePasswordAuthenticationFilter.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/filters/LogsearchUsernamePasswordAuthenticationFilter.java
@@ -20,11 +20,13 @@ package org.apache.ambari.logsearch.web.filters;
import java.io.IOException;
+import javax.inject.Inject;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.log4j.Logger;
+import org.springframework.security.authentication.AuthenticationManager;
import org.springframework.security.core.AuthenticationException;
import org.springframework.security.web.authentication.RememberMeServices;
import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listeners/SpringEventListener.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listeners/SpringEventListener.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listeners/SpringEventListener.java
deleted file mode 100644
index fc6a594..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/listeners/SpringEventListener.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.ambari.logsearch.web.listeners;
-
-import org.apache.log4j.Logger;
-import org.springframework.context.ApplicationListener;
-import org.springframework.security.authentication.event.AbstractAuthenticationEvent;
-
-public class SpringEventListener implements ApplicationListener<AbstractAuthenticationEvent> {
- private static final Logger logger = Logger.getLogger(SpringEventListener.class);
-
- @Override
- public void onApplicationEvent(AbstractAuthenticationEvent event) {
- logger.trace(" Inside onApplicationEvent SpringEventListener");
- }
-}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAbstractAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAbstractAuthenticationProvider.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAbstractAuthenticationProvider.java
index 0c102c3..eab33a1 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAbstractAuthenticationProvider.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAbstractAuthenticationProvider.java
@@ -65,14 +65,4 @@ public abstract class LogsearchAbstractAuthenticationProvider implements Authent
return isEnable;
}
- /**
- * Check authentication provider is enable or disable
- *
- * @return boolean
- */
- public boolean isEnable() {
- // default is disabled
- return false;
- }
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAuthenticationProvider.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAuthenticationProvider.java
index 3534818..05104b4 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAuthenticationProvider.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchAuthenticationProvider.java
@@ -26,7 +26,6 @@ import org.apache.log4j.Logger;
import org.springframework.security.authentication.UsernamePasswordAuthenticationToken;
import org.springframework.security.core.Authentication;
import org.springframework.security.core.AuthenticationException;
-import org.springframework.security.core.userdetails.UserDetailsService;
import org.springframework.security.web.authentication.WebAuthenticationDetails;
import org.springframework.stereotype.Component;
@@ -41,19 +40,19 @@ public class LogsearchAuthenticationProvider extends
.getLogger("org.apache.ambari.logsearch.audit");
@Inject
- UserDao userDao;
+ private UserDao userDao;
@Inject
- LogsearchLdapAuthenticationProvider ldapAuthenticationProvider;
+ private LogsearchLdapAuthenticationProvider ldapAuthenticationProvider;
@Inject
- LogsearchFileAuthenticationProvider fileAuthenticationProvider;
+ private LogsearchFileAuthenticationProvider fileAuthenticationProvider;
@Inject
- LogsearchSimpleAuthenticationProvider simpleAuthenticationProvider;
+ private LogsearchSimpleAuthenticationProvider simpleAuthenticationProvider;
@Inject
- LogsearchExternalServerAuthenticationProvider externalServerAuthenticationProvider;
+ private LogsearchExternalServerAuthenticationProvider externalServerAuthenticationProvider;
@Override
public Authentication authenticate(Authentication authentication)
@@ -128,7 +127,7 @@ public class LogsearchAuthenticationProvider extends
authentication = fileAuthenticationProvider.authenticate(authentication);
} else if (authMethod.equals(AUTH_METHOD.SIMPLE)) {
authentication = simpleAuthenticationProvider.authenticate(authentication);
- }else if (authMethod.equals(AUTH_METHOD.EXTERNAL_AUTH)) {
+ } else if (authMethod.equals(AUTH_METHOD.EXTERNAL_AUTH)) {
authentication = externalServerAuthenticationProvider.authenticate(authentication);
} else {
logger.error("Invalid authentication method :" + authMethod.name());
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchExternalServerAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchExternalServerAuthenticationProvider.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchExternalServerAuthenticationProvider.java
index a06a381..d398bdc 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchExternalServerAuthenticationProvider.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchExternalServerAuthenticationProvider.java
@@ -21,7 +21,6 @@ package org.apache.ambari.logsearch.web.security;
import java.util.ArrayList;
import java.util.List;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
import org.apache.ambari.logsearch.common.ExternalServerClient;
@@ -114,7 +113,7 @@ public class LogsearchExternalServerAuthenticationProvider extends
@Override
public Authentication authenticate(Authentication authentication)
throws AuthenticationException {
- if (!this.isEnable()) {
+ if (!authConfig.isAuthExternalEnabled()) {
LOG.debug("external server auth is disabled.");
return authentication;
}
@@ -173,13 +172,4 @@ public class LogsearchExternalServerAuthenticationProvider extends
}
return false;
}
-
- /**
- * Return true/false based on EXTERNAL_AUTH authentication method is
- * enabled/disabled return boolean
- */
- @Override
- public boolean isEnable() {
- return isEnable(AUTH_METHOD.EXTERNAL_AUTH);
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchFileAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchFileAuthenticationProvider.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchFileAuthenticationProvider.java
index 180de31..9662266 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchFileAuthenticationProvider.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchFileAuthenticationProvider.java
@@ -20,6 +20,7 @@ package org.apache.ambari.logsearch.web.security;
import java.util.Collection;
+import org.apache.ambari.logsearch.conf.AuthConfig;
import org.apache.ambari.logsearch.util.CommonUtil;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
@@ -41,11 +42,14 @@ public class LogsearchFileAuthenticationProvider extends LogsearchAbstractAuthen
private static Logger logger = Logger.getLogger(LogsearchFileAuthenticationProvider.class);
@Inject
+ private AuthConfig authConfig;
+
+ @Inject
private UserDetailsService userDetailsService;
@Override
public Authentication authenticate(Authentication authentication) throws AuthenticationException {
- if (!this.isEnable()) {
+ if (!authConfig.isAuthFileEnabled()) {
logger.debug("File auth is disabled.");
return authentication;
}
@@ -80,9 +84,4 @@ public class LogsearchFileAuthenticationProvider extends LogsearchAbstractAuthen
authentication = new UsernamePasswordAuthenticationToken(username, encPassword, authorities);
return authentication;
}
-
- @Override
- public boolean isEnable() {
- return isEnable(AUTH_METHOD.FILE);
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapAuthenticationProvider.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapAuthenticationProvider.java
index 742cd9b..f6c7df0 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapAuthenticationProvider.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapAuthenticationProvider.java
@@ -20,6 +20,7 @@ package org.apache.ambari.logsearch.web.security;
import java.util.List;
+import org.apache.ambari.logsearch.conf.AuthConfig;
import org.apache.log4j.Logger;
import org.springframework.ldap.CommunicationException;
import org.springframework.ldap.core.support.LdapContextSource;
@@ -31,6 +32,9 @@ import org.springframework.security.ldap.authentication.LdapAuthenticationProvid
import org.springframework.security.ldap.search.FilterBasedLdapUserSearch;
import org.springframework.stereotype.Component;
+import javax.annotation.PostConstruct;
+import javax.inject.Inject;
+
@Component
public class LogsearchLdapAuthenticationProvider extends
LogsearchAbstractAuthenticationProvider {
@@ -42,9 +46,16 @@ public class LogsearchLdapAuthenticationProvider extends
private static LdapAuthenticationProvider ldapAuthProvider = null;
private String logStatement = "";
+ @Inject
+ private AuthConfig authConfig;
+
public LogsearchLdapAuthenticationProvider() {
+ }
+
+ @PostConstruct
+ public void postConstruct() {
logger.debug("Creating object of ldap auth provider ");
- if (this.isEnable()) {
+ if (authConfig.isAuthLdapEnabled()) {
ldapAuthProvider = loadLdapAuthenticationProvider();
} else {
logger.info("Ldap auth is disabled");
@@ -54,7 +65,7 @@ public class LogsearchLdapAuthenticationProvider extends
@Override
public Authentication authenticate(Authentication authentication)
throws AuthenticationException {
- if (!this.isEnable()) {
+ if (!authConfig.isAuthLdapEnabled()) {
logger.debug("Ldap auth is disabled");
return authentication;
}
@@ -98,7 +109,7 @@ public class LogsearchLdapAuthenticationProvider extends
*
* @return corresponding LDAP authentication provider
*/
- LdapAuthenticationProvider loadLdapAuthenticationProvider() {
+ private LdapAuthenticationProvider loadLdapAuthenticationProvider() {
if (reloadLdapServerProperties()) {
logger.info("LDAP Properties changed - rebuilding Context");
LdapContextSource springSecurityContextSource = new LdapContextSource();
@@ -173,9 +184,4 @@ public class LogsearchLdapAuthenticationProvider extends
return false;
}
- @Override
- public boolean isEnable() {
- return isEnable(AUTH_METHOD.LDAP);
- }
-
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapBindAuthenticator.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapBindAuthenticator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapBindAuthenticator.java
index f9207b1..10f7507 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapBindAuthenticator.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchLdapBindAuthenticator.java
@@ -40,69 +40,7 @@ public class LogsearchLdapBindAuthenticator extends BindAuthenticator {
@Override
public DirContextOperations authenticate(Authentication authentication) {
-
- DirContextOperations user = super.authenticate(authentication);
-
- return setAmbariAdminAttr(user);
- }
-
- /**
- * Checks whether user is a member of ambari administrators group in LDAP.
- * If yes, sets user's ambari_admin attribute to true
- *
- * @param user
- * @return
- */
- private DirContextOperations setAmbariAdminAttr(DirContextOperations user) {
- String baseDn = ldapServerProperties.getBaseDN().toLowerCase();
- String groupBase = ldapServerProperties.getGroupBase().toLowerCase();
- String groupObjectClass = ldapServerProperties.getGroupObjectClass();
- String groupMembershipAttr = ldapServerProperties
- .getGroupMembershipAttr();
- String adminGroupMappingRules = ldapServerProperties
- .getAdminGroupMappingRules();
- final String groupNamingAttribute = ldapServerProperties
- .getGroupNamingAttr();
- String groupSearchFilter = ldapServerProperties.getGroupSearchFilter();
-
- // If groupBase is set incorrectly or isn't set - search in BaseDn
- int indexOfBaseDn = groupBase.indexOf(baseDn);
- groupBase = indexOfBaseDn <= 0 ? "" : groupBase.substring(0,
- indexOfBaseDn - 1);
-
- StringBuilder filterBuilder = new StringBuilder();
-
- filterBuilder.append("(&(");
- filterBuilder.append(groupMembershipAttr);
- filterBuilder.append("=");
- filterBuilder.append(user.getNameInNamespace());// DN
-
- if ((groupSearchFilter == null) || groupSearchFilter.equals("")) {
- // If groupSearchFilter is not specified, build it from other
- // authorization
- // group properties
- filterBuilder.append(")(objectclass=");
- filterBuilder.append(groupObjectClass);
- filterBuilder.append(")(|");
- String[] adminGroupMappingRegexs = adminGroupMappingRules
- .split(",");
- for (String adminGroupMappingRegex : adminGroupMappingRegexs) {
- filterBuilder.append("(");
- filterBuilder.append(groupNamingAttribute);
- filterBuilder.append("=");
- filterBuilder.append(adminGroupMappingRegex);
- filterBuilder.append(")");
- }
- filterBuilder.append(")");
- } else {
- filterBuilder.append(")");
- filterBuilder.append(groupSearchFilter);
- }
- filterBuilder.append(")");
-
- logger.info("filter=" + filterBuilder);
- // TODO: Filter is not used anywhere
- return user;
+ return super.authenticate(authentication);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchSimpleAuthenticationProvider.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchSimpleAuthenticationProvider.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchSimpleAuthenticationProvider.java
index ec2516c..5dc1c1f 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchSimpleAuthenticationProvider.java
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/web/security/LogsearchSimpleAuthenticationProvider.java
@@ -18,6 +18,7 @@
*/
package org.apache.ambari.logsearch.web.security;
+import org.apache.ambari.logsearch.conf.AuthConfig;
import org.apache.ambari.logsearch.web.model.User;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
@@ -28,14 +29,19 @@ import org.springframework.security.core.Authentication;
import org.springframework.security.core.AuthenticationException;
import org.springframework.stereotype.Component;
+import javax.inject.Inject;
+
@Component
public class LogsearchSimpleAuthenticationProvider extends LogsearchAbstractAuthenticationProvider {
private static Logger logger = Logger.getLogger(LogsearchSimpleAuthenticationProvider.class);
+ @Inject
+ private AuthConfig authConfig;
+
@Override
public Authentication authenticate(Authentication authentication) throws AuthenticationException {
- if (!this.isEnable()) {
+ if (!authConfig.isAuthSimpleEnabled()) {
logger.debug("Simple auth is disabled");
return authentication;
}
@@ -64,9 +70,4 @@ public class LogsearchSimpleAuthenticationProvider extends LogsearchAbstractAuth
return false;
}
}
-
- @Override
- public boolean isEnable() {
- return this.isEnable(AUTH_METHOD.SIMPLE);
- }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml
index ad96558..60b09cd 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/log4j.xml
@@ -57,7 +57,7 @@
<appender name="rolling_file_json"
class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
- <param name="file" value="logs/logsearch-app.json" />
+ <param name="file" value="target/logs/logsearch-app.json" />
<param name="Threshold" value="info" />
<param name="append" value="true" />
<param name="maxFileSize" value="10MB" />
@@ -67,7 +67,7 @@
<appender name="audit_rolling_file_json"
class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
- <param name="file" value="logs/logsearch-audit.json" />
+ <param name="file" value="target/logs/logsearch-audit.json" />
<param name="Threshold" value="info" />
<param name="append" value="true" />
<param name="maxFileSize" value="10MB" />
@@ -77,7 +77,7 @@
<appender name="performance_analyzer_json"
class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender">
- <param name="file" value="logs/logsearch-performance.json" />
+ <param name="file" value="target/logs/logsearch-performance.json" />
<param name="Threshold" value="info" />
<param name="append" value="true" />
<param name="maxFileSize" value="10MB" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
deleted file mode 100644
index 9961cd0..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/security-applicationContext.xml
+++ /dev/null
@@ -1,83 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<beans:beans xmlns="http://www.springframework.org/schema/security"
-xmlns:beans="http://www.springframework.org/schema/beans"
-xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-xmlns:security="http://www.springframework.org/schema/security"
-xsi:schemaLocation="http://www.springframework.org/schema/beans
-http://www.springframework.org/schema/beans/spring-beans-4.2.xsd
-http://www.springframework.org/schema/security
-http://www.springframework.org/schema/security/spring-security-4.0.xsd">
-
- <security:http pattern="/login.html" security="none" />
- <security:http pattern="/styles/**" security="none" />
- <security:http pattern="/fonts/**" security="none" />
- <security:http pattern="/scripts/**" security="none" />
- <security:http pattern="/libs/**" security="none" />
- <security:http pattern="/images/**" security="none" />
- <security:http pattern="/templates/**" security="none" />
- <security:http pattern="/favicon.ico" security="none" />
- <security:http pattern="/api/v1/public/**" security="none" />
- <security:http pattern="/api/v1/swagger.json" security="none"/>
- <security:http pattern="/api/v1/swagger.yaml" security="none"/>
-
- <security:http disable-url-rewriting="true" use-expressions="true" create-session="always" entry-point-ref="authenticationProcessingFilterEntryPoint">
- <csrf disabled="true"/>
- <security:session-management session-fixation-protection="newSession" />
- <intercept-url pattern="/**" access="isAuthenticated()"/>
- <security:custom-filter ref="krbAuthenticationFilter" after="FIRST" />
- <security:custom-filter position="FORM_LOGIN_FILTER" ref="logsearchUsernamePasswordAuthenticationFilter"/>
- <security:custom-filter position="LAST" ref="userContextFormationFilter"/>
- <security:logout delete-cookies="JSESSIONID" logout-url="/logout.html" success-handler-ref="logsearchLogoutSuccessHandler" />
- <http-basic entry-point-ref="authenticationProcessingFilterEntryPoint"/>
- </security:http>
-
- <beans:bean id="userContextFormationFilter" class="org.apache.ambari.logsearch.web.filters.LogsearchSecurityContextFormationFilter"/>
-
- <beans:bean id="krbAuthenticationFilter" class="org.apache.ambari.logsearch.web.filters.LogsearchKRBAuthenticationFilter"/>
-
- <beans:bean id="logsearchUsernamePasswordAuthenticationFilter" class="org.apache.ambari.logsearch.web.filters.LogsearchUsernamePasswordAuthenticationFilter">
- <beans:property name="authenticationManager" ref="authenticationManager"/>
- <beans:property name="authenticationSuccessHandler" ref="authSuccessHandler"/>
- <beans:property name="authenticationFailureHandler" ref="authFailureHandler"/>
- </beans:bean>
-
- <beans:bean id="authenticationProcessingFilterEntryPoint" class="org.apache.ambari.logsearch.web.filters.LogsearchAuthenticationEntryPoint">
- <beans:constructor-arg index="0" value="/login.html"/>
- <beans:property name="forceHttps" value="false"/>
- </beans:bean>
-
- <beans:bean id="authSuccessHandler" class="org.apache.ambari.logsearch.web.authenticate.LogsearchAuthSuccessHandler">
- </beans:bean>
-
- <beans:bean id="authFailureHandler" class="org.apache.ambari.logsearch.web.authenticate.LogsearchAuthFailureHandler">
- </beans:bean>
-
- <beans:bean id="logsearchLogoutSuccessHandler" class="org.apache.ambari.logsearch.web.authenticate.LogsearchLogoutSuccessHandler">
- </beans:bean>
-
- <beans:bean id="logsearchAuthenticationProvider" class="org.apache.ambari.logsearch.web.security.LogsearchAuthenticationProvider" >
- </beans:bean>
-
- <security:authentication-manager alias="authenticationManager">
- <security:authentication-provider ref="logsearchAuthenticationProvider"/>
- </security:authentication-manager>
-
- <beans:bean id="securityEventListener" class ="org.apache.ambari.logsearch.web.listeners.SpringEventListener"/>
-
-</beans:beans>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
deleted file mode 100755
index 0f1beec..0000000
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://java.sun.com/xml/ns/javaee" xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd" id="WebApp_ID" version="3.0">
-
- <listener>
- <listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
- </listener>
- <context-param>
- <param-name>contextClass</param-name>
- <param-value>
- org.springframework.web.context.support.AnnotationConfigWebApplicationContext
- </param-value>
- </context-param>
- <context-param>
- <param-name>contextConfigLocation</param-name>
- <param-value>org.apache.ambari.logsearch.conf.ApplicationConfig</param-value>
- </context-param>
-
- <listener>
- <listener-class>org.springframework.web.context.request.RequestContextListener</listener-class>
- </listener>
- <filter>
- <filter-name>springSecurityFilterChain</filter-name>
- <filter-class>org.springframework.web.filter.DelegatingFilterProxy</filter-class>
- </filter>
-
- <filter-mapping>
- <filter-name>springSecurityFilterChain</filter-name>
- <url-pattern>/*</url-pattern>
- </filter-mapping>
-
- <!-- Servlet mapping for REST -->
- <servlet>
- <servlet-name>REST service</servlet-name>
- <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>jersey.config.server.provider.packages</param-name>
- <param-value>org.apache.ambari.logsearch.rest,io.swagger.jaxrs.listing</param-value>
- </init-param>
- <load-on-startup>1</load-on-startup>
- </servlet>
- <servlet-mapping>
- <servlet-name>REST service</servlet-name>
- <url-pattern>/api/v1/*</url-pattern>
- </servlet-mapping>
-
-</web-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
index b588a2d..f930ec9 100644
--- a/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
+++ b/ambari-logsearch/docker/test-config/logfeeder/logfeeder.properties
@@ -21,7 +21,8 @@ logfeeder.config.files=shipper-conf/global.config.json,\
shipper-conf/input.config-logsearch.json,\
shipper-conf/input.config-hst.json,\
shipper-conf/input.config-system_message.json,\
- shipper-conf/input.config-secure_log.json
+ shipper-conf/input.config-secure_log.json,\
+ shipper-conf/input.config-hdfs.json
logfeeder.log.filter.enable=true
logfeeder.solr.config.interval=5
logfeeder.solr.core.config.name=history
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json
new file mode 100644
index 0000000..d975b0d
--- /dev/null
+++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hdfs.json
@@ -0,0 +1,172 @@
+{
+ "input": [
+ {
+ "type": "hdfs_audit",
+ "rowtype": "audit",
+ "is_enabled": "true",
+ "add_fields": {
+ "logType": "HDFSAudit",
+ "enforcer": "hadoop-acl",
+ "repoType": "1",
+ "repo": "hdfs"
+ },
+ "path": "/root/test-logs/hdfs-audit/hdfs-audit.log"
+ }
+ ],
+ "filter": [
+ {
+ "filter":"grok",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n",
+ "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})",
+ "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+ "post_map_values":{
+ "evtTime":{
+ "map_date":{
+ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"keyvalue",
+ "sort_order":1,
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "source_field":"log_message",
+ "value_split":"=",
+ "field_split":"\t",
+ "post_map_values":{
+ "src":{
+ "map_fieldname":{
+ "new_fieldname":"resource"
+ }
+
+ },
+ "ip":{
+ "map_fieldname":{
+ "new_fieldname":"cliIP"
+ }
+
+ },
+ "allowed":[
+ {
+ "map_fieldvalue":{
+ "pre_value":"true",
+ "post_value":"1"
+ }
+
+ },
+ {
+ "map_fieldvalue":{
+ "pre_value":"false",
+ "post_value":"0"
+ }
+
+ },
+ {
+ "map_fieldname":{
+ "new_fieldname":"result"
+ }
+
+ }
+
+ ],
+ "cmd":{
+ "map_fieldname":{
+ "new_fieldname":"action"
+ }
+
+ },
+ "proto":{
+ "map_fieldname":{
+ "new_fieldname":"cliType"
+ }
+
+ },
+ "callerContext":{
+ "map_fieldname":{
+ "new_fieldname":"req_caller_id"
+ }
+
+ }
+
+ }
+
+ },
+ {
+ "filter":"grok",
+ "sort_order":2,
+ "source_field":"ugi",
+ "remove_source_field":"false",
+ "conditions":{
+ "fields":{
+ "type":[
+ "hdfs_audit"
+ ]
+
+ }
+
+ },
+ "message_pattern":"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}",
+ "post_map_values":{
+ "user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "x_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "p_user":{
+ "map_fieldname":{
+ "new_fieldname":"reqUser"
+ }
+
+ },
+ "k_user":{
+ "map_fieldname":{
+ "new_fieldname":"proxyUsers"
+ }
+
+ },
+ "p_authType":{
+ "map_fieldname":{
+ "new_fieldname":"authType"
+ }
+
+ },
+ "k_authType":{
+ "map_fieldname":{
+ "new_fieldname":"proxyAuthType"
+ }
+
+ }
+
+ }
+
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/d89101ec/ambari-logsearch/docker/test-logs/hdfs-audit/hdfs-audit.log
----------------------------------------------------------------------
diff --git a/ambari-logsearch/docker/test-logs/hdfs-audit/hdfs-audit.log b/ambari-logsearch/docker/test-logs/hdfs-audit/hdfs-audit.log
new file mode 100644
index 0000000..a646cc4
--- /dev/null
+++ b/ambari-logsearch/docker/test-logs/hdfs-audit/hdfs-audit.log
@@ -0,0 +1,4 @@
+2016-03-18 10:00:47,252 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/192.168.64.102 cmd=getfileinfo src=/ats/active dst=null perm=null proto=rpc callerContext=HIVE_QUERY_ID:ambari-qa_20160317200111_223b3079-4a2d-431c-920f-6ba37ed63e9f
+2016-03-18 10:00:48,939 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/192.168.64.102 cmd=delete src=/tmp/hive/ambari-qa/resource1 dst=null perm=null proto=rpc
+2016-03-18 10:00:49,242 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/192.168.64.102 cmd=getfileinfo src=/tmp/hive/ambari-qa/resource2 dst=null perm=null proto=rpc
+2016-03-18 10:00:49,277 INFO FSNamesystem.audit: allowed=true ugi=ambari-qa (auth:SIMPLE) ip=/192.168.64.102 cmd=getfileinfo src=/tmp/hive/ambari-qa/resource2 dst=null perm=null proto=rpc