You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/07/25 13:57:16 UTC
[ambari] branch branch-2.7 updated: AMBARI-24353. Log Feeder:
generate solr id from specific fields. (#1882)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new f82827b AMBARI-24353. Log Feeder: generate solr id from specific fields. (#1882)
f82827b is described below
commit f82827b98faa9746abb9ea5feca8e390778843e4
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Wed Jul 25 15:57:13 2018 +0200
AMBARI-24353. Log Feeder: generate solr id from specific fields. (#1882)
---
.../ambari/logfeeder/plugin/output/Output.java | 8 ++
.../ambari/logfeeder/common/IdGeneratorHelper.java | 62 ++++++++++++++
.../ambari/logfeeder/output/OutputManagerImpl.java | 9 +-
.../apache/ambari/logfeeder/output/OutputSolr.java | 16 ++--
.../logfeeder/common/IdGeneratorHelperTest.java | 98 ++++++++++++++++++++++
5 files changed, 183 insertions(+), 10 deletions(-)
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java
index f369680..13e5ad8 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/output/Output.java
@@ -30,6 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -79,6 +80,13 @@ public abstract class Output<PROP_TYPE extends LogFeederProperties, INPUT_MARKER
this.destination = destination;
}
+ /**
+ * Get the list of fields that will be used for ID generation of log entries.
+ */
+ public List<String> getIdFields() {
+ return new ArrayList<>();
+ }
+
public boolean isClosed() {
return isClosed;
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/IdGeneratorHelper.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/IdGeneratorHelper.java
new file mode 100644
index 0000000..7430573
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/IdGeneratorHelper.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.common;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * Helper class to generete UUID (random or based on specific fields)
+ */
+public class IdGeneratorHelper {
+
+ private IdGeneratorHelper() {
+ }
+
+ /**
+ * Generate UUID based on fields (or just randomly)
+ * @param data object map which can contain the field key-value pairs
+ * @param idFields field names that used for generating uuid
+ * @return generated UUID string
+ */
+ public static String generateUUID(Map<String, Object> data, List<String> idFields) {
+ String uuid = null;
+ if (CollectionUtils.isNotEmpty(idFields)) {
+ final StringBuilder sb = new StringBuilder();
+ for (String idField : idFields) {
+ if (data.containsKey(idField)) {
+ sb.append(data.get(idField).toString());
+ }
+ }
+ String concatId = sb.toString();
+ if (StringUtils.isNotEmpty(concatId)) {
+ uuid = UUID.nameUUIDFromBytes(concatId.getBytes()).toString();
+ } else {
+ uuid = UUID.randomUUID().toString();
+ }
+ } else {
+ uuid = UUID.randomUUID().toString();
+ }
+ return uuid;
+ }
+}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
index 09951cc..390a770 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java
@@ -20,10 +20,10 @@ package org.apache.ambari.logfeeder.output;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.hash.Hashing;
+import org.apache.ambari.logfeeder.common.IdGeneratorHelper;
import org.apache.ambari.logfeeder.common.LogFeederConstants;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.loglevelfilter.LogLevelFilterHandler;
-import org.apache.ambari.logfeeder.input.InputFile;
import org.apache.ambari.logfeeder.plugin.common.MetricData;
import org.apache.ambari.logfeeder.plugin.input.Input;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
@@ -42,7 +42,6 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
-import java.util.UUID;
public class OutputManagerImpl extends OutputManager {
private static final Logger LOG = Logger.getLogger(OutputManagerImpl.class);
@@ -127,9 +126,6 @@ public class OutputManagerImpl extends OutputManager {
}
jsonObj.put("seq_num", new Long(docCounter++));
- if (jsonObj.get("id") == null) {
- jsonObj.put("id", UUID.randomUUID().toString());
- }
if (jsonObj.get("event_count") == null) {
jsonObj.put("event_count", new Integer(1));
}
@@ -154,6 +150,9 @@ public class OutputManagerImpl extends OutputManager {
List<? extends Output> outputList = input.getOutputList();
for (Output output : outputList) {
try {
+ if (jsonObj.get("id") == null) {
+ jsonObj.put("id", IdGeneratorHelper.generateUUID(jsonObj, output.getIdFields()));
+ }
output.write(jsonObj, inputMarker);
} catch (Exception e) {
LOG.error("Error writing. to " + output.getShortDescription(), e);
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
index 6b27553..278a7f5 100644
--- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java
@@ -19,6 +19,7 @@
package org.apache.ambari.logfeeder.output;
+import org.apache.ambari.logfeeder.common.IdGeneratorHelper;
import org.apache.ambari.logfeeder.common.LogFeederSolrClientFactory;
import org.apache.ambari.logfeeder.conf.LogFeederProps;
import org.apache.ambari.logfeeder.plugin.input.InputMarker;
@@ -45,13 +46,10 @@ import java.io.IOException;
import java.net.MalformedURLException;
import java.util.ArrayList;
import java.util.Calendar;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.Collection;;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -85,6 +83,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> {
private boolean implicitRouting = false;
private int lastSlotByMin = -1;
private boolean skipLogtime = false;
+ private List<String> idFields = new ArrayList<>();
private BlockingQueue<OutputData> outgoingBuffer = null;
private List<SolrWorkerThread> workerThreadList = new ArrayList<>();
@@ -134,6 +133,8 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> {
solrUrls = solrUrlsList.toArray(new String[0]);
}
+ idFields = getListValue("id_fields", new ArrayList<>());
+
skipLogtime = getBooleanValue("skip_logtime", DEFAULT_SKIP_LOGTIME);
maxIntervalMS = getIntValue("idle_flush_time_ms", DEFAULT_MAX_INTERVAL_MS);
@@ -409,7 +410,7 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> {
outputData = outgoingBuffer.poll(nextDispatchDuration, TimeUnit.MILLISECONDS);
}
if (outputData != null && outputData.jsonObj.get("id") == null) {
- outputData.jsonObj.put("id", UUID.randomUUID().toString());
+ outputData.jsonObj.put("id", IdGeneratorHelper.generateUUID(outputData.jsonObj, idFields));
}
return outputData;
}
@@ -501,4 +502,9 @@ public class OutputSolr extends Output<LogFeederProps, InputMarker> {
public void copyFile(File inputFile, InputMarker inputMarker) throws UnsupportedOperationException {
throw new UnsupportedOperationException("copyFile method is not yet supported for output=solr");
}
+
+ @Override
+ public List<String> getIdFields() {
+ return idFields;
+ }
}
diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/common/IdGeneratorHelperTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/common/IdGeneratorHelperTest.java
new file mode 100644
index 0000000..ae43ac0
--- /dev/null
+++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/common/IdGeneratorHelperTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.logfeeder.common;
+
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class IdGeneratorHelperTest {
+
+ @Test
+ public void testGenerateRandomUUID() {
+ // GIVEN
+ Map<String, Object> fieldKeyMap = new HashMap<>();
+ List<String> fields = new ArrayList<>();
+ // WHEN
+ String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap, fields);
+ String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap, fields);
+ // THEN
+ assertFalse(uuid1.equals(uuid2));
+ }
+
+ @Test
+ public void testUUIDFromFields() {
+ // GIVEN
+ Map<String, Object> fieldKeyMap1 = new HashMap<>();
+ fieldKeyMap1.put("one-field", "1");
+ Map<String, Object> fieldKeyMap2 = new HashMap<>();
+ fieldKeyMap2.put("one-field", "1");
+ List<String> fields = new ArrayList<>();
+ fields.add("one-field");
+ // WHEN
+ String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap1, fields);
+ String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap2, fields);
+ // THEN
+ assertTrue(uuid1.equals(uuid2));
+ }
+
+ @Test
+ public void testUUIDFromFieldsWithMultipleFields() {
+ // GIVEN
+ Map<String, Object> fieldKeyMap1 = new HashMap<>();
+ fieldKeyMap1.put("one-field", "1");
+ fieldKeyMap1.put("two-field", "2");
+ Map<String, Object> fieldKeyMap2 = new HashMap<>();
+ fieldKeyMap2.put("one-field", "1");
+ fieldKeyMap2.put("two-field", "2");
+ List<String> fields = new ArrayList<>();
+ fields.add("one-field");
+ fields.add("two-field");
+ // WHEN
+ String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap1, fields);
+ String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap2, fields);
+ // THEN
+ assertTrue(uuid1.equals(uuid2));
+ }
+
+ @Test
+ public void testUUIDFromFieldsDifferentNumberOfFields() {
+ // GIVEN
+ Map<String, Object> fieldKeyMap1 = new HashMap<>();
+ fieldKeyMap1.put("one-field", "1");
+ Map<String, Object> fieldKeyMap2 = new HashMap<>();
+ fieldKeyMap2.put("one-field", "1");
+ fieldKeyMap2.put("two-field", "2");
+ List<String> fields = new ArrayList<>();
+ fields.add("one-field");
+ fields.add("two-field");
+ // WHEN
+ String uuid1 = IdGeneratorHelper.generateUUID(fieldKeyMap1, fields);
+ String uuid2 = IdGeneratorHelper.generateUUID(fieldKeyMap2, fields);
+ // THEN
+ assertFalse(uuid1.equals(uuid2));
+ }
+
+}