You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mg...@apache.org on 2017/07/10 12:51:33 UTC
ambari git commit: AMBARI-21423 Add REST end point for the
documentation of the Log Feeder shipper properties (mgergely)
Repository: ambari
Updated Branches:
refs/heads/trunk c0882898d -> 15dd999ff
AMBARI-21423 Add REST end point for the documentation of the Log Feeder shipper properties (mgergely)
Change-Id: If6d1b66c3a1f74b118ae60a7edc26624d49fb7e6
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/15dd999f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/15dd999f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/15dd999f
Branch: refs/heads/trunk
Commit: 15dd999fff99fb80bc65ddfc94513e890a6efdef
Parents: c088289
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Mon Jul 10 14:51:23 2017 +0200
Committer: Miklos Gergely <mg...@hortonworks.com>
Committed: Mon Jul 10 14:51:23 2017 +0200
----------------------------------------------------------------------
.../api/ShipperConfigElementDescription.java | 59 +++++++++++
.../api/ShipperConfigTypeDescription.java | 44 ++++++++
.../model/inputconfig/impl/ConditionsImpl.java | 13 +++
.../model/inputconfig/impl/FieldsImpl.java | 14 +++
.../inputconfig/impl/FilterDescriptorImpl.java | 51 ++++++++++
.../impl/FilterGrokDescriptorImpl.java | 24 +++++
.../impl/FilterKeyValueDescriptorImpl.java | 28 +++++
.../model/inputconfig/impl/InputConfigImpl.java | 18 ++++
.../inputconfig/impl/InputDescriptorImpl.java | 101 +++++++++++++++++++
.../impl/InputFileBaseDescriptorImpl.java | 27 +++++
.../impl/InputS3FileDescriptorImpl.java | 16 +++
.../impl/MapAnonymizeDescriptorImpl.java | 21 +++-
.../inputconfig/impl/MapDateDescriptorImpl.java | 20 +++-
.../impl/MapFieldCopyDescriptorImpl.java | 14 ++-
.../impl/MapFieldDescriptorImpl.java | 33 ++++++
.../impl/MapFieldNameDescriptorImpl.java | 14 ++-
.../impl/MapFieldValueDescriptorImpl.java | 20 +++-
.../inputconfig/impl/PostMapValuesAdapter.java | 2 +-
.../ambari-logsearch-logfeeder/docs/filter.md | 4 +-
.../ambari-logsearch-logfeeder/docs/input.md | 10 +-
.../docs/postMapValues.md | 2 +-
.../ambari/logfeeder/filter/FilterJSONTest.java | 12 ++-
.../common/ShipperConfigDescriptionStorage.java | 67 ++++++++++++
.../ambari/logsearch/doc/DocConstants.java | 1 +
.../ambari/logsearch/manager/InfoManager.java | 9 ++
.../response/ShipperConfigDescriptionData.java | 52 ++++++++++
.../ambari/logsearch/rest/InfoResource.java | 10 ++
27 files changed, 667 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java
new file mode 100644
index 0000000..d65bf8e
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigElementDescription.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.api;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker for the shipper configuration properties.
+ * Can be used to generate documentation about the shipper configs.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.FIELD})
+public @interface ShipperConfigElementDescription {
+
+ /**
+ * The path of the json element.
+ */
+ String path();
+
+ /**
+ * The type of the json element.
+ */
+ String type();
+
+ /**
+ * Describe what the json element is used for.
+ */
+ String description();
+
+ /**
+ * An example value for the element, if applicable.
+ */
+ String[] examples() default {};
+
+ /**
+ * Default value of the json element, if applicable.
+ */
+ String defaultValue() default "";
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java
new file mode 100644
index 0000000..1c112d8
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/ShipperConfigTypeDescription.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.api;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marker for the shipper configuration types.
+ * Can be used to generate documentation about the shipper configs.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface ShipperConfigTypeDescription {
+
+ /**
+ * The name of the element type.
+ */
+ String name();
+
+ /**
+ * The description of the json element.
+ */
+ String description();
+
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
index 8bbff8f..2ba472c 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/ConditionsImpl.java
@@ -19,11 +19,24 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.Conditions;
import com.google.gson.annotations.Expose;
+@ShipperConfigTypeDescription(
+ name = "Conditions",
+ description = "Describes the conditions that should be met in order to match a filter to an input element.\n" +
+ "\n" +
+ "It has the following attributes:"
+)
public class ConditionsImpl implements Conditions {
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/conditions/fields",
+ type = "json object",
+ description = "The fields in the input element of which's value should be met."
+ )
@Expose
private FieldsImpl fields;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
index 68cd0e2..32a0348 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FieldsImpl.java
@@ -21,11 +21,25 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
import java.util.Set;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.Fields;
import com.google.gson.annotations.Expose;
+@ShipperConfigTypeDescription(
+ name = "Fields",
+ description = "Describes a the fields which's value should be met in order to match a filter to an input element.\n" +
+ "\n" +
+ "It has the following attributes:"
+ )
public class FieldsImpl implements Fields {
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/conditions/fields/type",
+ type = "list of strings",
+ description = "The acceptable values for the type field in the input element.",
+ examples = {"ambari_server", "\"spark_jobhistory_server\", \"spark_thriftserver\", \"livy_server\""}
+ )
@Expose
private Set<String> type;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
index 4e11715..eb9d38c 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterDescriptorImpl.java
@@ -22,35 +22,86 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
import java.util.List;
import java.util.Map;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
import org.apache.ambari.logsearch.config.api.model.inputconfig.PostMapValues;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+@ShipperConfigTypeDescription(
+ name = "Filter",
+ description = "The filter element in the [input configuration](inputConfig.md) contains a list of filter descriptions, each describing one filter applied on an input.\n" +
+ "\n" +
+ "The general elements in the json are the following:"
+)
public abstract class FilterDescriptorImpl implements FilterDescriptor {
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/filter",
+ type = "string",
+ description = "The type of the filter.",
+ examples = {"grok", "keyvalue", "json"}
+ )
@Expose
private String filter;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/conditions",
+ type = "json object",
+ description = "The conditions of which input to filter."
+ )
@Expose
private ConditionsImpl conditions;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/sort_order",
+ type = "integer",
+ description = "Describes the order in which the filters should be applied.",
+ examples = {"1", "3"}
+ )
@Expose
@SerializedName("sort_order")
private Integer sortOrder;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/source_field",
+ type = "integer",
+ description = "The source of the filter, must be set for keyvalue filters.",
+ examples = {"field_further_to_filter"},
+ defaultValue = "log_message"
+ )
@Expose
@SerializedName("source_field")
private String sourceField;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/remove_source_field",
+ type = "boolean",
+ description = "Remove the source field after the filter is applied.",
+ examples = {"true", "false"},
+ defaultValue = "false"
+ )
@Expose
@SerializedName("remove_source_field")
private Boolean removeSourceField;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values",
+ type = "dictionary string to list of json objects",
+ description = "Mappings done after the filtering provided it's result."
+ )
@Expose
@SerializedName("post_map_values")
private Map<String, List<PostMapValuesImpl>> postMapValues;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/is_enabled",
+ type = "boolean",
+ description = "A flag to show if the filter should be used.",
+ examples = {"true", "false"},
+ defaultValue = "true"
+ )
@Expose
@SerializedName("is_enabled")
private Boolean isEnabled;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
index 995f76b..e140df0 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterGrokDescriptorImpl.java
@@ -19,20 +19,44 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+@ShipperConfigTypeDescription(
+ name = "Grok Filter",
+ description = "Grok filters have the following additional parameters:"
+)
public class FilterGrokDescriptorImpl extends FilterDescriptorImpl implements FilterGrokDescriptor {
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/log4j_format",
+ type = "string",
+ description = "The log4j pattern of the log, not used, it is only there for documentation.",
+ examples = {"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n"}
+ )
@Expose
@SerializedName("log4j_format")
private String log4jFormat;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/multiline_pattern",
+ type = "string",
+ description = "The grok pattern that shows that the line is not a log line on it's own but the part of a multi line entry.",
+ examples = {"^(%{TIMESTAMP_ISO8601:logtime})"}
+ )
@Expose
@SerializedName("multiline_pattern")
private String multilinePattern;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/message_pattern",
+ type = "string",
+ description = "The grok pattern to use to parse the log entry.",
+ examples = {"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}"}
+ )
@Expose
@SerializedName("message_pattern")
private String messagePattern;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
index 8e89990..1c782c5 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/FilterKeyValueDescriptorImpl.java
@@ -19,20 +19,48 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+@ShipperConfigTypeDescription(
+ name = "Key-value Filter",
+ description = "value_borders is only used if it is specified, and value_split is not.\n" +
+ "\n" +
+ "Key-value filters have the following additional parameters:"
+)
public class FilterKeyValueDescriptorImpl extends FilterDescriptorImpl implements FilterKeyValueDescriptor {
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/field_split",
+ type = "string",
+ description = "The string that splits the key-value pairs.",
+ examples = {" ", ","},
+ defaultValue = "\\t"
+ )
@Expose
@SerializedName("field_split")
private String fieldSplit;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/value_split",
+ type = "string",
+ description = "The string that separates keys from values.",
+ examples = {":", "->"},
+ defaultValue = "="
+ )
@Expose
@SerializedName("value_split")
private String valueSplit;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/value_borders",
+ type = "string",
+ description = "The borders around the value, must be 2 characters long, first before it, second after it.",
+ examples = {"()", "[]", "{}"}
+ )
@Expose
@SerializedName("value_borders")
private String valueBorders;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
index a4eba8e..6ce634f 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputConfigImpl.java
@@ -21,16 +21,34 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
import java.util.List;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
import com.google.gson.annotations.Expose;
+@ShipperConfigTypeDescription(
+ name = "Input Config",
+ description = "The input configurations are stored in json files. Each of them are describing the processing of the log files of a service.\n" +
+ "\n" +
+ "The json contains two elements:"
+)
public class InputConfigImpl implements InputConfig {
+ @ShipperConfigElementDescription(
+ path = "/input",
+ type = "list of json objects",
+ description = "A list of input descriptions"
+ )
@Expose
private List<InputDescriptorImpl> input;
+ @ShipperConfigElementDescription(
+ path = "/filter",
+ type = "list of json objects",
+ description = "A list of filter descriptions"
+ )
@Expose
private List<FilterDescriptorImpl> filter;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
index 54b4b9b..cec16c8 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
@@ -21,59 +21,160 @@ package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
import java.util.Map;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+@ShipperConfigTypeDescription(
+ name = "Input",
+ description = "The input element in the input configuration contains a list of input descriptions, each describing one source of input.\n" +
+ "\n" +
+ "The general elements in the json are the following:"
+)
public abstract class InputDescriptorImpl implements InputDescriptor {
+ @ShipperConfigElementDescription(
+ path = "/input/[]/type",
+ type = "string",
+ description = "The log id for this source.",
+ examples = {"zookeeper", "ambari_server"}
+ )
@Expose
private String type;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/rowtype",
+ type = "string",
+ description = "The type of the row.",
+ examples = {"service", "audit"}
+ )
@Expose
private String rowtype;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/path",
+ type = "string",
+ description = "The path of the source, may contain '*' characters too.",
+ examples = {"/var/log/ambari-logsearch-logfeeder/logsearch-logfeeder.json", "/var/log/zookeeper/zookeeper*.log"}
+ )
@Expose
private String path;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/add_fields",
+ type = "dictionary",
+ description = "The element contains field_name: field_value pairs which will be added to each rows data.",
+ examples = {"\"cluster\":\"cluster_name\""}
+ )
@Expose
@SerializedName("add_fields")
private Map<String, String> addFields;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/source",
+ type = "dictionary",
+ description = "The type of the input source.",
+ examples = {"file", "s3_file"}
+ )
@Expose
private String source;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/tail",
+ type = "boolean",
+ description = "The input should check for only the latest file matching the pattern, not all of them.",
+ examples = {"true", "false"},
+ defaultValue = "true"
+ )
@Expose
private Boolean tail;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/gen_event_md5",
+ type = "boolean",
+ description = "Generate an event_md5 field for each row by creating a hash of the row data.",
+ examples = {"true", "false"},
+ defaultValue = "true"
+ )
@Expose
@SerializedName("gen_event_md5")
private Boolean genEventMd5;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/use_event_md5_as_id",
+ type = "boolean",
+ description = "Generate an id for each row by creating a hash of the row data.",
+ examples = {"true", "false"},
+ defaultValue = "false"
+ )
@Expose
@SerializedName("use_event_md5_as_id")
private Boolean useEventMd5AsId;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/cache_enabled",
+ type = "boolean",
+ description = "Allows the input to use a cache to filter out duplications.",
+ examples = {"true", "false"},
+ defaultValue = "false"
+ )
@Expose
@SerializedName("cache_enabled")
private Boolean cacheEnabled;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/cache_key_field",
+ type = "string",
+ description = "Specifies the field for which to use the cache to find duplications of.",
+ examples = {"some_field_prone_to_repeating_value"},
+ defaultValue = "log_message"
+ )
@Expose
@SerializedName("cache_key_field")
private String cacheKeyField;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/cache_last_dedup_enabled",
+ type = "boolean",
+ description = "Allow to filter out entries which are same as the most recent one irrelevant of it's time.",
+ examples = {"true", "false"},
+ defaultValue = "false"
+ )
@Expose
@SerializedName("cache_last_dedup_enabled")
private Boolean cacheLastDedupEnabled;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/cache_size",
+ type = "integer",
+ description = "The number of entries to store in the cache.",
+ examples = {"50"},
+ defaultValue = "100"
+ )
@Expose
@SerializedName("cache_size")
private Integer cacheSize;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/cache_dedup_interval",
+ type = "integer",
+ description = "The maximum interval in ms which may pass between two identical log messages to filter the latter out.",
+ examples = {"500"},
+ defaultValue = "1000"
+ )
@Expose
@SerializedName("cache_dedup_interval")
private Long cacheDedupInterval;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/is_enabled",
+ type = "boolean",
+ description = "A flag to show if the input should be used.",
+ examples = {"true", "false"},
+ defaultValue = "true"
+ )
@Expose
@SerializedName("is_enabled")
private Boolean isEnabled;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
index 51c7ec8..8281daa 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputFileBaseDescriptorImpl.java
@@ -19,20 +19,47 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputFileBaseDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+@ShipperConfigTypeDescription(
+ name = "File Input",
+ description = "File inputs have some additional parameters:"
+)
public class InputFileBaseDescriptorImpl extends InputDescriptorImpl implements InputFileBaseDescriptor {
+ @ShipperConfigElementDescription(
+ path = "/input/[]/checkpoint_interval_ms",
+ type = "integer",
+ description = "The time interval in ms when the checkpoint file should be updated.",
+ examples = {"10000"},
+ defaultValue = "5000"
+ )
@Expose
@SerializedName("checkpoint_interval_ms")
private Integer checkpointIntervalMs;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/process_file",
+ type = "boolean",
+ description = "Should the file be processed.",
+ examples = {"true", "false"},
+ defaultValue = "true"
+ )
@Expose
@SerializedName("process_file")
private Boolean processFile;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/copy_file",
+ type = "boolean",
+ description = "Should the file be copied (only if not processed).",
+ examples = {"true", "false"},
+ defaultValue = "false"
+ )
@Expose
@SerializedName("copy_file")
private Boolean copyFile;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
index 277a57c..19f52d3 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputS3FileDescriptorImpl.java
@@ -19,16 +19,32 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
+@ShipperConfigTypeDescription(
+ name = "S3 File Input",
+ description = "S3 file inputs have the following parameters in addition to the general file parameters:"
+)
public class InputS3FileDescriptorImpl extends InputFileBaseDescriptorImpl implements InputS3FileDescriptor {
+ @ShipperConfigElementDescription(
+ path = "/input/[]/s3_access_key",
+ type = "string",
+ description = "The access key used for AWS credentials."
+ )
@Expose
@SerializedName("s3_access_key")
private String s3AccessKey;
+ @ShipperConfigElementDescription(
+ path = "/input/[]/s3_secret_key",
+ type = "string",
+ description = "The secret key used for AWS credentials."
+ )
@Expose
@SerializedName("s3_secret_key")
private String s3SecretKey;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
index 5fdbbab..8c128de 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapAnonymizeDescriptorImpl.java
@@ -19,20 +19,39 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.MapAnonymizeDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
-public class MapAnonymizeDescriptorImpl implements MapAnonymizeDescriptor {
+@ShipperConfigTypeDescription(
+ name = "Map Anonymize",
+ description = "The name of the mapping element should be map_anonymize. The value json element should contain the following parameter:"
+)
+public class MapAnonymizeDescriptorImpl extends MapFieldDescriptorImpl implements MapAnonymizeDescriptor {
@Override
public String getJsonName() {
return "map_anonymize";
}
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_anonymize/pattern",
+ type = "string",
+ description = "The pattern to use to identify parts to anonymize. The parts to hide should be marked with the \"<hide>\" string.",
+ examples = {"Some secret is here: <hide>, and another one is here: <hide>"}
+ )
@Expose
private String pattern;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_anonymize/hide_char",
+ type = "string",
+ description = "The character to hide with",
+ defaultValue = "*",
+ examples = {"X", "-"}
+ )
@Expose
@SerializedName("hide_char")
private Character hideChar;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
index 2e54e7a..feec4b6 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapDateDescriptorImpl.java
@@ -19,21 +19,39 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.MapDateDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
-public class MapDateDescriptorImpl implements MapDateDescriptor {
+@ShipperConfigTypeDescription(
+ name = "Map Date",
+ description = "The name of the mapping element should be map_date. The value json element may contain the following parameters:"
+)
+public class MapDateDescriptorImpl extends MapFieldDescriptorImpl implements MapDateDescriptor {
@Override
public String getJsonName() {
return "map_date";
}
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_date/src_date_pattern",
+ type = "string",
+ description = "If it is specified than the mapper converts from this format to the target, and also adds missing year",
+ examples = {"MMM dd HH:mm:ss"}
+ )
@Expose
@SerializedName("src_date_pattern")
private String sourceDatePattern;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_date/target_date_pattern",
+ type = "string",
+ description = "If 'epoch' then the field is parsed as seconds from 1970, otherwise the content used as pattern",
+ examples = {"yyyy-MM-dd HH:mm:ss,SSS", "epoch"}
+ )
@Expose
@SerializedName("target_date_pattern")
private String targetDatePattern;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
index 4a8d746..e7b8fdf 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldCopyDescriptorImpl.java
@@ -19,17 +19,29 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldCopyDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
-public class MapFieldCopyDescriptorImpl implements MapFieldCopyDescriptor {
+@ShipperConfigTypeDescription(
+ name = "Map Copy",
+ description = "The name of the mapping element should be map_copy. The value json element should contain the following parameter:"
+)
+public class MapFieldCopyDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldCopyDescriptor {
@Override
public String getJsonName() {
return "map_fieldcopy";
}
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_copy/copy_name",
+ type = "string",
+ description = "The name of the copied field",
+ examples = {"new_name"}
+ )
@Expose
@SerializedName("copy_name")
private String copyName;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
new file mode 100644
index 0000000..101e0d4
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldDescriptorImpl.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
+import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldDescriptor;
+
+@ShipperConfigTypeDescription(
+ name = "Post Map Values",
+ description = "The Post Map Values element in the [filter](filter.md) field names as keys, the values are lists of sets of " +
+ "post map values, each describing one mapping done on a field named before obtained after filtering.\n" +
+ "\n" +
+ "Currently there are four kind of mappings are supported:"
+ )
+public abstract class MapFieldDescriptorImpl implements MapFieldDescriptor {
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
index bd32018..e1b71e6 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldNameDescriptorImpl.java
@@ -19,17 +19,29 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldNameDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
-public class MapFieldNameDescriptorImpl implements MapFieldNameDescriptor {
+@ShipperConfigTypeDescription(
+ name = "Map Field Name",
+ description = "The name of the mapping element should be map_fieldname. The value json element should contain the following parameter:"
+)
+public class MapFieldNameDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldNameDescriptor {
@Override
public String getJsonName() {
return "map_fieldname";
}
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldname/new_field_name",
+ type = "string",
+ description = "The name of the renamed field",
+ examples = {"new_name"}
+ )
@Expose
@SerializedName("new_field_name")
private String newFieldName;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
index 599e152..a80a994 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapFieldValueDescriptorImpl.java
@@ -19,21 +19,39 @@
package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
import org.apache.ambari.logsearch.config.api.model.inputconfig.MapFieldValueDescriptor;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
-public class MapFieldValueDescriptorImpl implements MapFieldValueDescriptor {
+@ShipperConfigTypeDescription(
+ name = "Map Field Value",
+ description = "The name of the mapping element should be map_fieldvalue. The value json element should contain the following parameter:"
+)
+public class MapFieldValueDescriptorImpl extends MapFieldDescriptorImpl implements MapFieldValueDescriptor {
@Override
public String getJsonName() {
return "map_fieldvalue";
}
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/pre_value",
+ type = "string",
+ description = "The value that the field must match (ignoring case) to be mapped",
+ examples = {"old_value"}
+ )
@Expose
@SerializedName("pre_value")
private String preValue;
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_fieldvalue/post_value",
+ type = "string",
+ description = "The value to which the field is modified to",
+ examples = {"new_value"}
+ )
@Expose
@SerializedName("post_value")
private String postValue;
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
index 3c21fd8..e3f9886 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
@@ -95,7 +95,7 @@ public class PostMapValuesAdapter implements JsonDeserializer<List<PostMapValues
private JsonElement createMapperObject(PostMapValuesImpl postMapValues, JsonSerializationContext context) {
JsonObject jsonObject = new JsonObject();
for (MapFieldDescriptor m : postMapValues.getMappers()) {
- jsonObject.add(((MapFieldDescriptor)m).getJsonName(), context.serialize(m));
+ jsonObject.add(((MapFieldDescriptorImpl)m).getJsonName(), context.serialize(m));
}
return jsonObject;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
index 129279b..d825290 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/filter.md
@@ -48,6 +48,8 @@ Grok filters have the following additional parameters:
## Key-value Filter
+value\_borders is only used if it is specified, and value\_split is not.
+
Key-value filters have the following additional parameters:
| Field | Description | Default |
@@ -56,4 +58,4 @@ Key-value filters have the following additional parameters:
| value\_split | The string that separates keys from values | "=" |
| value\_borders | The borders around the value, must be 2 characters long, first before it, second after it | - |
-If value\_borders is only used if it is specified, and value\_split is not.
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
index 661eeb8..1a9ce8d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/input.md
@@ -20,20 +20,18 @@ limitations under the License.
# Input
The input element in the [input configuration](inputConfig.md) contains a list of input descriptions, each describing one source
-of input.
-
-The general elements in the json are the following:
+of input. The general elements in the json are the following:
| Field | Description | Default |
|-----------------------------|-------------------------------------------------------------------------------------------------------|--------------|
-| type | The type of the input source, currently file and s3_file are supported | - |
+| type | The log id for this source | - |
| rowtype | The type of the row, can be service / audit | - |
| path | The path of the source, may contain '*' characters too | - |
| add\_fields | The element contains field\_name: field\_value pairs which will be added to each rows data | - |
+| source | The type of the input source, currently file and s3_file are supported | - |
| tail | The input should check for only the latest file matching the pattern, not all of them | true |
| gen\_event\_md5 | Generate an event\_md5 field for each row by creating a hash of the row data | true |
| use\_event\_md5\_as\_id | Generate an id for each row by creating a hash of the row data | false |
-| start\_position | Should the parsing start from the beginning | beginning |
| cache\_enabled | Allows the input to use a cache to filter out duplications | true |
| cache\_key\_field | Specifies the field for which to use the cache to find duplications of | log\_message |
| cache\_last\_dedup\_enabled | Allow to filter out entries which are same as the most recent one irrelevant of it's time | false |
@@ -44,7 +42,7 @@ The general elements in the json are the following:
## File Input
-File inputs have the following parameters too:
+File inputs have some additional parameters:
| Field | Description | Default |
|--------------------------|--------------------------------------------------------------------|---------|
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
index 7ec439a..bc219df 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/docs/postMapValues.md
@@ -67,4 +67,4 @@ The name of the mapping element should be map\_anonymize. The value json element
| Field | Description |
|------------|-----------------------------------------------------------------------------------------------------------------|
| pattern | The pattern to use to identify parts to anonymize. The parts to hide should be marked with the "<hide>" string. |
-| hide\_char | The character to hide with, if it is not specified then the default is 'X' |
+| hide\_char | The character to hide with, if it is not specified then the default is '*' |
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
index 7abf177..acc3d4d 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java
@@ -131,13 +131,15 @@ public class FilterJSONTest {
@Test
public void testJSONFilterCode_invalidJson() throws Exception {
LOG.info("testJSONFilterCode_invalidJson()");
+
init(new FilterJsonDescriptorImpl());
- String inputStr="invalid json";
+
+ String inputStr = "invalid json";
try{
- filterJson.apply(inputStr,new InputMarker(null, null, 0));
- fail("Expected LogFeederException was not occured");
- }catch(LogFeederException logFeederException){
- assertEquals("Json parsing failed for inputstr = "+inputStr, logFeederException.getLocalizedMessage());
+ filterJson.apply(inputStr,new InputMarker(null, null, 0));
+ fail("Expected LogFeederException was not occured");
+ } catch(LogFeederException logFeederException) {
+ assertEquals("Json parsing failed for inputstr = " + inputStr, logFeederException.getLocalizedMessage());
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
new file mode 100644
index 0000000..7d4bc2c
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/ShipperConfigDescriptionStorage.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.common;
+
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
+import org.reflections.Reflections;
+import org.reflections.scanners.FieldAnnotationsScanner;
+
+import javax.annotation.PostConstruct;
+import javax.inject.Named;
+import java.lang.reflect.Field;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+@Named
+public class ShipperConfigDescriptionStorage {
+
+ private static final String SHIPPER_CONFIG_PACKAGE = "org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl";
+
+ private final List<ShipperConfigDescriptionData> shipperConfigDescription = new ArrayList<>();
+
+ @PostConstruct
+ public void postConstruct() {
+ Thread loadShipperConfigDescriptionThread = new Thread("load_shipper_config_description") {
+ @Override
+ public void run() {
+ fillShipperConfigDescriptions();
+ }
+ };
+ loadShipperConfigDescriptionThread.setDaemon(true);
+ loadShipperConfigDescriptionThread.start();
+ }
+
+ public List<ShipperConfigDescriptionData> getShipperConfigDescription() {
+ return shipperConfigDescription;
+ }
+
+ private void fillShipperConfigDescriptions() {
+ Reflections reflections = new Reflections(SHIPPER_CONFIG_PACKAGE, new FieldAnnotationsScanner());
+ Set<Field> fields = reflections.getFieldsAnnotatedWith(ShipperConfigElementDescription.class);
+ for (Field field : fields) {
+ ShipperConfigElementDescription description = field.getAnnotation(ShipperConfigElementDescription.class);
+ shipperConfigDescription.add(new ShipperConfigDescriptionData(description.path(), description.description(),
+ description.examples(), description.defaultValue()));
+ }
+
+ shipperConfigDescription.sort((o1, o2) -> o1.getPath().compareTo(o2.getPath()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
index 6d1382d..da0a8bb 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java
@@ -104,6 +104,7 @@ public class DocConstants {
public static final String GET_AUTH_DETAILS_OD = "Get authentication details.";
public static final String GET_ALL_PROPERTIES_INFO_OD = "List all available properties for Log Search and Log Feeder";
public static final String GET_LOGSEARCH_PROPERTIES_INFO_OD = "List all available properties for Log Search property file (e.g: logsearch.properties/logfeeder.properties)";
+ public static final String GET_ALL_SHIPPER_CONFIG_INFO_OD = "List all available shipper configuration element";
}
public class EventHistoryDescriptions {
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
index f6d0449..2f63492 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/InfoManager.java
@@ -25,7 +25,9 @@ import java.util.Map;
import org.apache.ambari.logsearch.conf.AuthPropsConfig;
import org.apache.ambari.logsearch.common.PropertyDescriptionStorage;
+import org.apache.ambari.logsearch.common.ShipperConfigDescriptionStorage;
import org.apache.ambari.logsearch.model.response.PropertyDescriptionData;
+import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
import javax.inject.Inject;
import javax.inject.Named;
@@ -39,6 +41,9 @@ public class InfoManager extends JsonManagerBase {
@Inject
private PropertyDescriptionStorage propertyDescriptionStore;
+ @Inject
+ private ShipperConfigDescriptionStorage shipperConfigDescriptionStore;
+
public Map<String, Boolean> getAuthMap() {
Map<String, Boolean> authMap = new HashMap<>();
authMap.put("external", authPropsConfig.isAuthExternalEnabled());
@@ -56,4 +61,8 @@ public class InfoManager extends JsonManagerBase {
public List<PropertyDescriptionData> getLogSearchPropertyDescriptions(String propertiesFile) {
return getPropertyDescriptions().get(propertiesFile);
}
+
+ public List<ShipperConfigDescriptionData> getLogSearchShipperConfigDescription() {
+ return shipperConfigDescriptionStore.getShipperConfigDescription();
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java
new file mode 100644
index 0000000..91f7420
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ShipperConfigDescriptionData.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.model.response;
+
+public class ShipperConfigDescriptionData {
+ private final String path;
+
+ private final String description;
+
+ private final String[] examples;
+
+ private final String defaultValue;
+
+ public ShipperConfigDescriptionData(String path, String description, String[] examples, String defaultValue) {
+ this.path = path;
+ this.description = description;
+ this.examples = examples;
+ this.defaultValue = defaultValue;
+ }
+
+ public String getPath() {
+ return path;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public String[] getExamples() {
+ return examples;
+ }
+
+ public String getDefaultValue() {
+ return defaultValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/15dd999f/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
index 6ea0bab..e49be90 100644
--- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
+++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/InfoResource.java
@@ -29,12 +29,14 @@ import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.apache.ambari.logsearch.manager.InfoManager;
import org.apache.ambari.logsearch.model.response.PropertyDescriptionData;
+import org.apache.ambari.logsearch.model.response.ShipperConfigDescriptionData;
import org.springframework.context.annotation.Scope;
import java.util.List;
import java.util.Map;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_PROPERTIES_INFO_OD;
+import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_ALL_SHIPPER_CONFIG_INFO_OD;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_LOGSEARCH_PROPERTIES_INFO_OD;
import static org.apache.ambari.logsearch.doc.DocConstants.PublicOperationDescriptions.GET_AUTH_DETAILS_OD;
@@ -70,4 +72,12 @@ public class InfoResource {
public List<PropertyDescriptionData> getPropertyFileDescription(@PathParam("propertyFile") String propertyFile) {
return infoManager.getLogSearchPropertyDescriptions(propertyFile);
}
+
+ @GET
+ @Path("/shipperconfig")
+ @Produces({"application/json"})
+ @ApiOperation(GET_ALL_SHIPPER_CONFIG_INFO_OD)
+ public List<ShipperConfigDescriptionData> getShipperConfigDescription() {
+ return infoManager.getLogSearchShipperConfigDescription();
+ }
}