You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/01/10 14:22:06 UTC
[ambari] branch trunk updated: AMBARI-22750. Add custom input /
mapper descriptor to config api
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 635ee1d AMBARI-22750. Add custom input / mapper descriptor to config api
635ee1d is described below
commit 635ee1d9898e1bf6f9335c123ccef62fd9a6ce31
Author: Oliver Szabo <ol...@gmail.com>
AuthorDate: Wed Jan 10 11:05:35 2018 +0100
AMBARI-22750. Add custom input / mapper descriptor to config api
---
...{InputDescriptor.java => CustomDescriptor.java} | 31 ++-------
...tDescriptor.java => InputCustomDescriptor.java} | 32 +--------
.../api/model/inputconfig/InputDescriptor.java | 2 +
...putDescriptor.java => MapCustomDescriptor.java} | 32 +--------
.../model/inputconfig/impl/InputAdapter.java | 4 +-
.../impl/InputCustomDescriptorImpl.java | 71 +++++++++++++++++++
.../inputconfig/impl/InputDescriptorImpl.java | 11 +++
.../inputconfig/impl/MapCustomDescriptorImpl.java | 81 ++++++++++++++++++++++
.../inputconfig/impl/PostMapValuesAdapter.java | 12 ++--
9 files changed, 182 insertions(+), 94 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/CustomDescriptor.java
similarity index 68%
copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
copy to ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/CustomDescriptor.java
index 82e9504..481fb23 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/CustomDescriptor.java
@@ -16,37 +16,16 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.ambari.logsearch.config.api.model.inputconfig;
import java.util.Map;
-public interface InputDescriptor {
- String getType();
-
- String getRowtype();
-
- String getPath();
-
- Map<String, String> getAddFields();
-
- String getSource();
-
- Boolean isTail();
-
- Boolean isGenEventMd5();
-
- Boolean isUseEventMd5AsId();
-
- Boolean isCacheEnabled();
-
- String getCacheKeyField();
-
- Boolean getCacheLastDedupEnabled();
+public interface CustomDescriptor {
+ Map<String, Object> getProperties();
- Integer getCacheSize();
+ void setProperties(Map<String, Object> properties);
- Long getCacheDedupInterval();
+ String getMapperClassName();
- Boolean isEnabled();
+ void setMapperClassName(String className);
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputCustomDescriptor.java
similarity index 66%
copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
copy to ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputCustomDescriptor.java
index 82e9504..54c5935 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputCustomDescriptor.java
@@ -16,37 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.ambari.logsearch.config.api.model.inputconfig;
-import java.util.Map;
-
-public interface InputDescriptor {
- String getType();
-
- String getRowtype();
-
- String getPath();
-
- Map<String, String> getAddFields();
-
- String getSource();
-
- Boolean isTail();
-
- Boolean isGenEventMd5();
-
- Boolean isUseEventMd5AsId();
-
- Boolean isCacheEnabled();
-
- String getCacheKeyField();
-
- Boolean getCacheLastDedupEnabled();
-
- Integer getCacheSize();
-
- Long getCacheDedupInterval();
-
- Boolean isEnabled();
+public interface InputCustomDescriptor extends InputDescriptor, CustomDescriptor {
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
index 82e9504..2ad1fac 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
@@ -49,4 +49,6 @@ public interface InputDescriptor {
Long getCacheDedupInterval();
Boolean isEnabled();
+
+ Map<String, Object> getAllProperties();
}
diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapCustomDescriptor.java
similarity index 66%
copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
copy to ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapCustomDescriptor.java
index 82e9504..a6d4a42 100644
--- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java
+++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/MapCustomDescriptor.java
@@ -16,37 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-
package org.apache.ambari.logsearch.config.api.model.inputconfig;
-import java.util.Map;
-
-public interface InputDescriptor {
- String getType();
-
- String getRowtype();
-
- String getPath();
-
- Map<String, String> getAddFields();
-
- String getSource();
-
- Boolean isTail();
-
- Boolean isGenEventMd5();
-
- Boolean isUseEventMd5AsId();
-
- Boolean isCacheEnabled();
-
- String getCacheKeyField();
-
- Boolean getCacheLastDedupEnabled();
-
- Integer getCacheSize();
-
- Long getCacheDedupInterval();
-
- Boolean isEnabled();
+public interface MapCustomDescriptor extends MapFieldDescriptor, CustomDescriptor {
}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java
index 86741c6..5fcaf31 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputAdapter.java
@@ -51,7 +51,9 @@ public class InputAdapter implements JsonDeserializer<InputDescriptorImpl> {
return (InputDescriptorImpl)context.deserialize(json, InputFileDescriptorImpl.class);
case "s3_file":
return (InputDescriptorImpl)context.deserialize(json, InputS3FileDescriptorImpl.class);
- default:
+ case "custom":
+ return (InputDescriptorImpl)context.deserialize(json, InputCustomDescriptorImpl.class);
+ default:
throw new IllegalArgumentException("Unknown input type: " + json.getAsJsonObject().get("source").getAsString());
}
}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputCustomDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputCustomDescriptorImpl.java
new file mode 100644
index 0000000..41128c5
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputCustomDescriptorImpl.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+
+import com.google.gson.annotations.Expose;
+import com.google.gson.annotations.SerializedName;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.model.inputconfig.InputCustomDescriptor;
+
+import java.util.Map;
+
+public class InputCustomDescriptorImpl extends InputDescriptorImpl implements InputCustomDescriptor {
+
+ @ShipperConfigElementDescription(
+ path = "/input/[]/properties",
+ type = "map",
+ description = "Custom key value pairs",
+ examples = {"{k1 : v1, k2: v2}"},
+ defaultValue = ""
+ )
+ @Expose
+ @SerializedName("properties")
+ private Map<String, Object> properties;
+
+ @ShipperConfigElementDescription(
+ path = "/input/[]/class_name",
+ type = "string",
+ description = "Custom class which implements an input type",
+ examples = {"org.example.MyInputSource"},
+ defaultValue = ""
+ )
+ @Expose
+ @SerializedName("class")
+ private String mapperClassName;
+
+ @Override
+ public Map<String, Object> getProperties() {
+ return this.properties;
+ }
+
+ @Override
+ public String getMapperClassName() {
+ return this.mapperClassName;
+ }
+
+ @Override
+ public void setProperties(Map<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public void setMapperClassName(String className) {
+ this.mapperClassName = className;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
index cec16c8..765bf83 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java
@@ -179,6 +179,8 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
@SerializedName("is_enabled")
private Boolean isEnabled;
+ private Map<String, Object> allProperties;
+
public String getType() {
return type;
}
@@ -290,4 +292,13 @@ public abstract class InputDescriptorImpl implements InputDescriptor {
public void setIsEnabled(Boolean isEnabled) {
this.isEnabled = isEnabled;
}
+
+ @Override
+ public Map<String, Object> getAllProperties() {
+ return allProperties;
+ }
+
+ public void setAllProperties(Map<String, Object> allProperties) {
+ this.allProperties = allProperties;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapCustomDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapCustomDescriptorImpl.java
new file mode 100644
index 0000000..0cb04fb
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/MapCustomDescriptorImpl.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl;
+
+import com.google.gson.annotations.Expose;
+import com.google.gson.annotations.SerializedName;
+import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription;
+import org.apache.ambari.logsearch.config.api.ShipperConfigTypeDescription;
+import org.apache.ambari.logsearch.config.api.model.inputconfig.MapCustomDescriptor;
+
+import java.util.Map;
+
+@ShipperConfigTypeDescription(
+ name = "Map Custom",
+ description = "The name of the mapping element should be map_custom. The value json element may contain the following parameters:"
+)
+public class MapCustomDescriptorImpl implements MapCustomDescriptor {
+
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_custom/properties",
+ type = "map",
+ description = "Custom key value pairs",
+ examples = {"{k1 : v1, k2: v2}"},
+ defaultValue = ""
+ )
+ @Expose
+ @SerializedName("properties")
+ private Map<String, Object> properties;
+
+ @ShipperConfigElementDescription(
+ path = "/filter/[]/post_map_values/{field_name}/[]/map_custom/class_name",
+ type = "string",
+ description = "Custom class which implements a mapper type",
+ examples = {"org.example.MyMapper"},
+ defaultValue = ""
+ )
+ @Expose
+ @SerializedName("class")
+ private String mapperClassName;
+
+ @Override
+ public Map<String, Object> getProperties() {
+ return this.properties;
+ }
+
+ @Override
+ public String getMapperClassName() {
+ return this.mapperClassName;
+ }
+
+ @Override
+ public String getJsonName() {
+ return "map_custom";
+ }
+
+ @Override
+ public void setProperties(Map<String, Object> properties) {
+ this.properties = properties;
+ }
+
+ @Override
+ public void setMapperClassName(String className) {
+ this.mapperClassName = className;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
index e3f9886..6781066 100644
--- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
+++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/PostMapValuesAdapter.java
@@ -55,20 +55,22 @@ public class PostMapValuesAdapter implements JsonDeserializer<List<PostMapValues
for (Map.Entry<String, JsonElement> m : e.getAsJsonObject().entrySet()) {
switch (m.getKey()) {
case "map_date":
- mappers.add((MapDateDescriptorImpl)context.deserialize(m.getValue(), MapDateDescriptorImpl.class));
+ mappers.add(context.deserialize(m.getValue(), MapDateDescriptorImpl.class));
break;
case "map_fieldcopy":
- mappers.add((MapFieldCopyDescriptorImpl)context.deserialize(m.getValue(), MapFieldCopyDescriptorImpl.class));
+ mappers.add(context.deserialize(m.getValue(), MapFieldCopyDescriptorImpl.class));
break;
case "map_fieldname":
- mappers.add((MapFieldNameDescriptorImpl)context.deserialize(m.getValue(), MapFieldNameDescriptorImpl.class));
+ mappers.add(context.deserialize(m.getValue(), MapFieldNameDescriptorImpl.class));
break;
case "map_fieldvalue":
- mappers.add((MapFieldValueDescriptorImpl)context.deserialize(m.getValue(), MapFieldValueDescriptorImpl.class));
+ mappers.add(context.deserialize(m.getValue(), MapFieldValueDescriptorImpl.class));
break;
case "map_anonymize":
- mappers.add((MapAnonymizeDescriptorImpl)context.deserialize(m.getValue(), MapAnonymizeDescriptorImpl.class));
+ mappers.add(context.deserialize(m.getValue(), MapAnonymizeDescriptorImpl.class));
break;
+ case "map_custom":
+ mappers.add(context.deserialize(m.getValue(), MapCustomDescriptorImpl.class));
default:
System.out.println("Unknown key: " + m.getKey());
}
--
To stop receiving notification emails like this one, please contact
['"commits@ambari.apache.org" <co...@ambari.apache.org>'].