You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@eagle.apache.org by ra...@apache.org on 2016/09/27 15:44:52 UTC
incubator-eagle git commit: [EAGLE-566] add dedup fields into
publishment compare consideration
Repository: incubator-eagle
Updated Branches:
refs/heads/master 45ff4b8f5 -> 16c17067a
[EAGLE-566] add dedup fields into publishment compare consideration
Author: Li, Garrett
Reviewer: ralphsu
This closes #454
Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/16c17067
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/16c17067
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/16c17067
Branch: refs/heads/master
Commit: 16c17067a8d75ae065c492be61f29078dee5c7b1
Parents: 45ff4b8
Author: Xiancheng Li <xi...@ebay.com>
Authored: Tue Sep 27 13:26:11 2016 +0800
Committer: Ralph, Su <su...@gmail.com>
Committed: Tue Sep 27 08:44:35 2016 -0700
----------------------------------------------------------------------
.../coordinator/OverrideDeduplicatorSpec.java | 18 ++++++++++++++++++
.../alert/engine/coordinator/Publishment.java | 10 ++++++----
.../alert/engine/publisher/dedup/DedupCache.java | 10 +++++++++-
.../publisher/dedup/DedupEventsStoreFactory.java | 8 +++++---
.../publisher/dedup/MongoDedupEventsStore.java | 8 ++++++--
.../publisher/impl/DefaultDeduplicator.java | 4 ++--
6 files changed, 46 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/16c17067/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/OverrideDeduplicatorSpec.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/OverrideDeduplicatorSpec.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/OverrideDeduplicatorSpec.java
index 0a65ee6..cad41f3 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/OverrideDeduplicatorSpec.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/OverrideDeduplicatorSpec.java
@@ -19,7 +19,10 @@ package org.apache.eagle.alert.engine.coordinator;
import java.io.Serializable;
import java.util.Map;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.google.common.base.Objects;
@JsonIgnoreProperties(ignoreUnknown = true)
public class OverrideDeduplicatorSpec implements Serializable {
@@ -45,4 +48,19 @@ public class OverrideDeduplicatorSpec implements Serializable {
this.properties = properties;
}
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof OverrideDeduplicatorSpec) {
+ OverrideDeduplicatorSpec spec = (OverrideDeduplicatorSpec) obj;
+ return Objects.equal(className, spec.getClassName())
+ && Objects.equal(properties, spec.getProperties());
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().append(className).append(properties).build();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/16c17067/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
index 8055144..bb00291 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-common/src/main/java/org/apache/eagle/alert/engine/coordinator/Publishment.java
@@ -16,14 +16,13 @@
*/
package org.apache.eagle.alert.engine.coordinator;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
import java.util.List;
import java.util.Map;
import java.util.Objects;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-
-import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
-
/**
* @since Apr 11, 2016.
*/
@@ -119,6 +118,9 @@ public class Publishment {
Publishment p = (Publishment) obj;
return (Objects.equals(name, p.getName()) && Objects.equals(type, p.getType())
&& Objects.equals(dedupIntervalMin, p.getDedupIntervalMin())
+ && Objects.equals(dedupFields, p.getDedupFields())
+ && Objects.equals(dedupStateField, p.getDedupStateField())
+ && Objects.equals(overrideDeduplicator, p.getOverrideDeduplicator())
&& Objects.equals(policyIds, p.getPolicyIds()) && properties.equals(p.getProperties()));
}
return false;
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/16c17067/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupCache.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupCache.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupCache.java
index 51029c1..fc2d6e6 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupCache.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupCache.java
@@ -200,8 +200,16 @@ public class DedupCache {
} else {
DedupValue dedupValue = dedupValues.getLast();
dedupValue.setCount(dedupValue.getCount() + 1);
- LOG.info("Update count for dedup key {}, value {} and count {}", eventEniq,
+ String updateMsg = String.format(
+ "Update count for dedup key {}, value %s and count %s", eventEniq,
dedupValue.getStateFieldValue(), dedupValue.getCount());
+ if (dedupValue.getCount() > 0 && dedupValue.getCount() % 100 == 0) {
+ LOG.info(updateMsg);
+ DedupEventsStore accessor = DedupEventsStoreFactory.getStore(type, this.config);
+ accessor.add(eventEniq, dedupValues);
+ } else if (LOG.isDebugEnabled()) {
+ LOG.debug(updateMsg);
+ }
return dedupValue;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/16c17067/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupEventsStoreFactory.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupEventsStoreFactory.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupEventsStoreFactory.java
index 9e67f66..0a3b206 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupEventsStoreFactory.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/DedupEventsStoreFactory.java
@@ -28,6 +28,8 @@ public class DedupEventsStoreFactory {
private static DedupEventsStore customizedStore;
+ private static MongoDedupEventsStore accessor;
+
public static void customizeStore(DedupEventsStore store) {
customizedStore = store;
}
@@ -36,13 +38,13 @@ public class DedupEventsStoreFactory {
if (customizedStore != null) {
return customizedStore;
}
- DedupEventsStore accessor = null;
switch (type) {
case Mongo:
- accessor = new MongoDedupEventsStore(config);
+ if (accessor == null) {
+ accessor = new MongoDedupEventsStore(config);
+ }
break;
case ElasticSearch:
- break;
default:
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/16c17067/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/MongoDedupEventsStore.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/MongoDedupEventsStore.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/MongoDedupEventsStore.java
index a536360..46a01b0 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/MongoDedupEventsStore.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/dedup/MongoDedupEventsStore.java
@@ -74,8 +74,12 @@ public class MongoDedupEventsStore implements DedupEventsStore {
public MongoDedupEventsStore(Config config) {
this.config = config;
this.connection = this.config.getString("connection");
- this.client = new MongoClient(new MongoClientURI(this.connection));
- init();
+ try {
+ this.client = new MongoClient(new MongoClientURI(this.connection));
+ init();
+ } catch (Throwable t) {
+ LOG.error(String.format("initialize mongodb %s client failed", this.connection), t);
+ }
}
private void init() {
http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/16c17067/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
----------------------------------------------------------------------
diff --git a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
index 0e79fca..d284da5 100644
--- a/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
+++ b/eagle-core/eagle-alert-parent/eagle-alert/alert-engine/src/main/java/org/apache/eagle/alert/engine/publisher/impl/DefaultDeduplicator.java
@@ -117,8 +117,8 @@ public class DefaultDeduplicator implements AlertDeduplicator {
event.getPolicyId(), event.getCreatedTime(), customFieldValues), stateFiledValue);
if (outputEvents != null && outputEvents.size() > 0) {
return outputEvents;
- } else if (LOG.isDebugEnabled()) {
- LOG.debug("Alert event is skipped because it's duplicated: {}", event.toString());
+ } else if (LOG.isInfoEnabled()) {
+ LOG.info("Alert event is skipped because it's duplicated: {}", event.toString());
}
return null;
}