You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ak...@apache.org on 2019/03/11 16:35:50 UTC
[incubator-pinot] branch master updated: [TE] Update watermarks
when detectionNames are updated in subscription groups (#3945)
This is an automated email from the ASF dual-hosted git repository.
akshayrai09 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new f697214 [TE] Update watermarks when detectionNames are updated in subscription groups (#3945)
f697214 is described below
commit f697214ee239a1dadf533e72abdec0047ffc02c6
Author: Akshay Rai <ak...@gmail.com>
AuthorDate: Mon Mar 11 09:35:39 2019 -0700
[TE] Update watermarks when detectionNames are updated in subscription groups (#3945)
---
.../thirdeye/detection/yaml/YamlResource.java | 18 ++++++++
.../thirdeye/detection/yaml/YamlResourceTest.java | 54 +++++++++++++---------
.../detection/yaml/alertconfig/alert-config-5.yaml | 40 ++++++++++++++++
3 files changed, 89 insertions(+), 23 deletions(-)
diff --git a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
index 28adeb5..5a95c02 100644
--- a/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
+++ b/thirdeye/thirdeye-pinot/src/main/java/org/apache/pinot/thirdeye/detection/yaml/YamlResource.java
@@ -77,6 +77,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.Yaml;
+import static org.apache.pinot.thirdeye.detection.yaml.YamlDetectionAlertConfigTranslator.*;
+
@Path("/yaml")
@Api(tags = {Constants.YAML_TAG})
@@ -491,6 +493,22 @@ public class YamlResource {
DetectionAlertConfigDTO updatedAlertConfig = updateDetectionAlertConfig(oldAlertConfig, newAlertConfig);
updatedAlertConfig.setYaml(yamlAlertConfig);
+ // Update watermarks to reflect changes to detectionName list in subscription config
+ Map<Long, Long> currentVectorClocks = updatedAlertConfig.getVectorClocks();
+ Map<Long, Long> updatedVectorClocks = new HashMap<>();
+ Map<String, Object> properties = updatedAlertConfig.getProperties();
+ long currentTimestamp = System.currentTimeMillis();
+ if (properties.get(PROP_DETECTION_CONFIG_IDS) != null) {
+ for (long detectionId : ConfigUtils.getLongs(properties.get(PROP_DETECTION_CONFIG_IDS))) {
+ if (currentVectorClocks != null && currentVectorClocks.keySet().contains(detectionId)) {
+ updatedVectorClocks.put(detectionId, currentVectorClocks.get(detectionId));
+ } else {
+ updatedVectorClocks.put(detectionId, currentTimestamp);
+ }
+ }
+ }
+ updatedAlertConfig.setVectorClocks(updatedVectorClocks);
+
// Validate before updating the config
subscriptionValidator.validateUpdatedConfig(updatedAlertConfig, oldAlertConfig);
diff --git a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
index 9561fcf..da07db9 100644
--- a/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
+++ b/thirdeye/thirdeye-pinot/src/test/java/org/apache/pinot/thirdeye/detection/yaml/YamlResourceTest.java
@@ -8,13 +8,9 @@ import org.apache.pinot.thirdeye.datalayer.dto.DetectionConfigDTO;
import org.apache.pinot.thirdeye.datasource.DAORegistry;
import org.apache.pinot.thirdeye.detection.annotation.registry.DetectionAlertRegistry;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.testng.Assert;
-import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -24,6 +20,8 @@ public class YamlResourceTest {
private DAOTestBase testDAOProvider;
private YamlResource yamlResource;
private DAORegistry daoRegistry;
+ private static long alertId1;
+ private static long alertId2;
@BeforeMethod
public void beforeClass() {
@@ -31,9 +29,12 @@ public class YamlResourceTest {
this.yamlResource = new YamlResource();
this.daoRegistry = DAORegistry.getInstance();
DetectionConfigManager detectionDAO = this.daoRegistry.getDetectionConfigManager();
- DetectionConfigDTO config = new DetectionConfigDTO();
- config.setName("test_detection_1");
- detectionDAO.save(config);
+ DetectionConfigDTO config1 = new DetectionConfigDTO();
+ config1.setName("test_detection_1");
+ alertId1 = detectionDAO.save(config1);
+ DetectionConfigDTO config2 = new DetectionConfigDTO();
+ config2.setName("test_detection_2");
+ alertId2 = detectionDAO.save(config2);
DetectionAlertRegistry.getInstance().registerAlertScheme("EMAIL", "EmailClass");
DetectionAlertRegistry.getInstance().registerAlertScheme("IRIS", "IrisClass");
@@ -114,22 +115,30 @@ public class YamlResourceTest {
Assert.assertNotNull(detection);
Assert.assertEquals(detection.getName(), "Subscription Group Name");
} catch (Exception e) {
- Assert.fail("Exception should not be thrown for valid yaml");
+ Assert.fail("Exception should not be thrown for valid yaml. Message = " + e);
}
}
@Test
public void testUpdateDetectionAlertConfig() throws IOException {
- DetectionAlertConfigDTO oldAlertDTO = new DetectionAlertConfigDTO();
- oldAlertDTO.setName("Subscription Group Name");
- oldAlertDTO.setApplication("Random Application");
- long oldId = daoRegistry.getDetectionAlertConfigManager().save(oldAlertDTO);
+ ApplicationDTO request = new ApplicationDTO();
+ request.setApplication("test_application");
+ request.setRecipients("abc@abc.in");
+ daoRegistry.getApplicationDAO().save(request);
+
+ String validYaml = IOUtils.toString(this.getClass().getResourceAsStream("alertconfig/alert-config-4.yaml"));
+ long oldId = -1;
+ try {
+ oldId = this.yamlResource.createSubscriptionGroup(validYaml);
+ } catch (Exception e) {
+ Assert.fail("Exception should not be thrown for valid yaml. Message = " + e);
+ }
DetectionAlertConfigDTO alertDTO;
try {
this.yamlResource.updateSubscriptionGroup(-1, "");
- Assert.fail("Exception not thrown on empty yaml");
+ Assert.fail("Exception not thrown when the subscription group doesn't exist");
} catch (Exception e) {
Assert.assertEquals(e.getMessage(), "Cannot find subscription group -1");
}
@@ -150,7 +159,7 @@ public class YamlResourceTest {
Assert.assertEquals(e.getMessage(), "Could not parse as map: application:test:application");
}
- String noSubscriptGroupYaml = "application: test_application";
+ String noSubscriptGroupYaml = "application: test_app";
try {
this.yamlResource.updateSubscriptionGroup(oldId, noSubscriptGroupYaml);
Assert.fail("Exception not thrown on empty yaml");
@@ -166,20 +175,19 @@ public class YamlResourceTest {
Assert.assertEquals(e.getMessage(), "Application field cannot be left empty");
}
- ApplicationDTO request = new ApplicationDTO();
- request.setApplication("test_application");
- request.setRecipients("abc@abc.in");
- daoRegistry.getApplicationDAO().save(request);
-
- String validYaml = IOUtils.toString(this.getClass().getResourceAsStream("alertconfig/alert-config-3.yaml"));
+ String validYaml2 = IOUtils.toString(this.getClass().getResourceAsStream("alertconfig/alert-config-5.yaml"));
try {
- this.yamlResource.updateSubscriptionGroup(oldId, validYaml);
+ this.yamlResource.updateSubscriptionGroup(oldId, validYaml2);
alertDTO = daoRegistry.getDetectionAlertConfigManager().findById(oldId);
Assert.assertNotNull(alertDTO);
- Assert.assertEquals(alertDTO.getName(), "test_group");
+ Assert.assertEquals(alertDTO.getName(), "Subscription Group Name");
Assert.assertEquals(alertDTO.getApplication(), "test_application");
+
+ // Verify if the vector clock is updated with the updated detection
+ Assert.assertEquals(alertDTO.getVectorClocks().keySet().size(), 1);
+ Assert.assertEquals(alertDTO.getVectorClocks().keySet().toArray()[0], alertId2);
} catch (Exception e) {
- Assert.fail("Exception should not be thrown for valid yaml" + e.getMessage());
+ Assert.fail("Exception should not be thrown for valid yaml. Message = " + e);
}
}
}
diff --git a/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/alertconfig/alert-config-5.yaml b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/alertconfig/alert-config-5.yaml
new file mode 100644
index 0000000..ffa241c
--- /dev/null
+++ b/thirdeye/thirdeye-pinot/src/test/resources/org/apache/pinot/thirdeye/detection/yaml/alertconfig/alert-config-5.yaml
@@ -0,0 +1,40 @@
+subscriptionGroupName: "Subscription Group Name"
+cron: "0 0/5 * 1/1 * ? *"
+application: "test_application"
+active: true
+subscribedDetections:
+ - test_detection_2
+
+type: DIMENSIONAL_ALERTER_PIPELINE
+dimensionRecipients:
+ "android":
+ - "android-thirdeye@thirdeye.com"
+ "ios":
+ - "ios-thirdeye@thirdeye.com"
+dimension: app_name
+
+fromAddress: thirdeye-dev@linkedin.com
+
+recipients:
+ to:
+ - "thirdeye@thirdeye.com"
+ cc:
+ - "thirdeye-developers@thirdeye.com"
+
+alertSchemes:
+- type: EMAIL
+- type: IRIS
+ params:
+ plan: thirdye_test_plan
+
+alertSuppressors:
+- type: TIME_WINDOW
+ params:
+ windowStartTime: 1542888000000
+ windowEndTime: 1543215600000
+ isThresholdApplied: true
+ expectedChange: -0.25
+ acceptableDeviation: 0.35
+
+referenceLinks:
+ "Oncall Runbook": "test_url"
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org