You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2021/12/07 08:51:51 UTC
[incubator-streampipes] branch dev updated (ed465e7 -> f45d9a3)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git.
from ed465e7 [hotfix] Retry failing cypress tests once
new 19430b7 [hotfix] Fix timestamp format for data lake csv export
new f45d9a3 [STREAMPIPES-449] Add test for rate limit processor
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../dataexplorer/DataLakeManagementV4.java | 56 ++++++++++++----------
.../filters-jvm/rateLimit1/description.json | 25 ++++++++++
.../filters-jvm/rateLimit1/expected.csv | 3 ++
.../filters-jvm/rateLimit1/input.csv | 12 +++++
ui/cypress/support/utils/StaticPropertyUtils.ts | 2 +-
.../static-alternatives.component.html | 4 +-
.../static-one-of-input.component.html | 2 +-
7 files changed, 75 insertions(+), 29 deletions(-)
create mode 100644 ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/description.json
create mode 100644 ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/expected.csv
create mode 100644 ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/input.csv
[incubator-streampipes] 02/02: [STREAMPIPES-449] Add test for rate limit processor
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit f45d9a3d11ee53363f904ba0f27f4302af288e91
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Tue Dec 7 09:51:39 2021 +0100
[STREAMPIPES-449] Add test for rate limit processor
---
.../filters-jvm/rateLimit1/description.json | 25 ++++++++++++++++++++++
.../filters-jvm/rateLimit1/expected.csv | 3 +++
.../filters-jvm/rateLimit1/input.csv | 12 +++++++++++
ui/cypress/support/utils/StaticPropertyUtils.ts | 2 +-
.../static-alternatives.component.html | 4 ++--
.../static-one-of-input.component.html | 2 +-
6 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/description.json b/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/description.json
new file mode 100644
index 0000000..bf2d851
--- /dev/null
+++ b/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/description.json
@@ -0,0 +1,25 @@
+{
+ "name": "rate_limit",
+ "config": [
+ {
+ "type": "radio",
+ "selector": "grouping-enabled",
+ "value": "False"
+ },
+ {
+ "type": "radio",
+ "selector": "window-type",
+ "value": "Length Window"
+ },
+ {
+ "type": "input",
+ "selector": "undefined-length-window-size-2",
+ "value": "5"
+ },
+ {
+ "type": "radio",
+ "selector": "event-selection",
+ "value": "Last"
+ }
+ ]
+}
diff --git a/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/expected.csv b/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/expected.csv
new file mode 100644
index 0000000..0d663ca
--- /dev/null
+++ b/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/expected.csv
@@ -0,0 +1,3 @@
+time;randomnumber
+1623871494000;22.0
+1623871499000;26.0
diff --git a/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/input.csv b/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/input.csv
new file mode 100644
index 0000000..0291979
--- /dev/null
+++ b/ui/cypress/fixtures/pipelineElement/filters-jvm/rateLimit1/input.csv
@@ -0,0 +1,12 @@
+timestamp;randomnumber
+1623871490000;62.0
+1623871491000;46.0
+1623871492000;41.0
+1623871493000;41.0
+1623871494000;22.0
+1623871495000;56.0
+1623871496000;95.0
+1623871497000;77.0
+1623871498000;85.0
+1623871499000;26.0
+1623871500000;21.0
diff --git a/ui/cypress/support/utils/StaticPropertyUtils.ts b/ui/cypress/support/utils/StaticPropertyUtils.ts
index 708287c..674c2f2 100644
--- a/ui/cypress/support/utils/StaticPropertyUtils.ts
+++ b/ui/cypress/support/utils/StaticPropertyUtils.ts
@@ -29,7 +29,7 @@ export class StaticPropertyUtils {
} else if (config.type === 'drop-down') {
cy.dataCy(config.selector).click().get('mat-option').contains(config.value).click();
} else if (config.type === 'radio') {
- cy.dataCy(config.selector + config.value).click();
+ cy.dataCy(config.selector.replace(' ', '_').toLowerCase() + '-' + config.value.replace(' ', '_').toLowerCase()).click();
} else if (config.type === 'click') {
cy.dataCy(config.selector).click({ force: true });
} else if (config.type === 'code-input') {
diff --git a/ui/src/app/core-ui/static-properties/static-alternatives/static-alternatives.component.html b/ui/src/app/core-ui/static-properties/static-alternatives/static-alternatives.component.html
index 52769e8..4ce775a 100644
--- a/ui/src/app/core-ui/static-properties/static-alternatives/static-alternatives.component.html
+++ b/ui/src/app/core-ui/static-properties/static-alternatives/static-alternatives.component.html
@@ -26,7 +26,7 @@
[value]="alternative"
[checked]="alternative.selected"
[matTooltip]="alternative.description"
- [attr.data-cy]="alternative.label">
+ [attr.data-cy]="staticProperty.internalName.replace(' ', '_').toLowerCase() + '-' + alternative.label.replace(' ', '_').toLowerCase()">
<label style="font-weight: normal;">
{{alternative.label}}
</label>
@@ -63,4 +63,4 @@
</div>
-</div>
\ No newline at end of file
+</div>
diff --git a/ui/src/app/core-ui/static-properties/static-one-of-input/static-one-of-input.component.html b/ui/src/app/core-ui/static-properties/static-one-of-input/static-one-of-input.component.html
index aed2912..fc31acc 100644
--- a/ui/src/app/core-ui/static-properties/static-one-of-input/static-one-of-input.component.html
+++ b/ui/src/app/core-ui/static-properties/static-one-of-input/static-one-of-input.component.html
@@ -26,7 +26,7 @@
[value]="option"
(click)="select(option.elementId)"
[checked]="option.selected"
- [attr.data-cy]="staticProperty.internalName + option.name">
+ [attr.data-cy]="staticProperty.internalName.replace(' ', '_').toLowerCase() + '-' + option.name.replace(' ', '_').toLowerCase()">
<label style="font-weight: normal">
{{option.name}}
</label>
[incubator-streampipes] 01/02: [hotfix] Fix timestamp format for data lake csv export
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
commit 19430b78bdf6aa4508d28b7dda84cf8562f75e17
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Tue Dec 7 09:49:00 2021 +0100
[hotfix] Fix timestamp format for data lake csv export
---
.../dataexplorer/DataLakeManagementV4.java | 56 ++++++++++++----------
1 file changed, 31 insertions(+), 25 deletions(-)
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
index 3ae4241..f9853fb 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/DataLakeManagementV4.java
@@ -40,6 +40,11 @@ import java.io.IOException;
import java.io.OutputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.time.*;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoField;
+import java.time.temporal.ChronoUnit;
+import java.time.temporal.TemporalAccessor;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@@ -48,6 +53,10 @@ import static org.apache.streampipes.dataexplorer.v4.SupportedDataLakeQueryParam
public class DataLakeManagementV4 {
+ private static final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(
+ "yyyy-MM-dd'T'HH:mm:ss[.SSS]'Z'")
+ .withZone(ZoneId.of("UTC"));
+
public List<DataLakeMeasure> getAllMeasurements() {
return DataExplorerUtils.getInfos();
}
@@ -64,8 +73,6 @@ public class DataLakeManagementV4 {
if (!params.has(QP_LIMIT)) {
params.update(QP_LIMIT, 500000);
}
- SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
- formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
SpQueryResult dataResult;
//JSON
@@ -100,11 +107,7 @@ public class DataLakeManagementV4 {
}
isFirstElementInRow = false;
if (i1 == 0) {
- try {
- element = formatter.parse(element.toString()).getTime();
- } catch (ParseException e) {
- element = element.toString();
- }
+ element = parseTime(element.toString());
}
//produce json e.g. "name": "Pipes" or "load": 42
outputStream.write(toBytes("\"" + dataResult.getHeaders().get(i1) + "\": "
@@ -157,11 +160,7 @@ public class DataLakeManagementV4 {
}
isFirstInRow = false;
if (i1 == 0) {
- try {
- element = formatter.parse(element.toString()).getTime();
- } catch (ParseException e) {
- element = element.toString();
- }
+ element = parseTime(element.toString());
}
if (element == null) {
outputStream.write(toBytes(""));
@@ -195,10 +194,7 @@ public class DataLakeManagementV4 {
if (measure.getMeasureName().equals(measurementID)) {
QueryResult queryResult = new DeleteDataQuery(new DataLakeMeasure(measurementID, null)).executeQuery();
- if (queryResult.hasError() || queryResult.getResults().get(0).getError() != null) {
- return false;
- }
- return true;
+ return !queryResult.hasError() && queryResult.getResults().get(0).getError() == null;
}
}
return false;
@@ -232,11 +228,11 @@ public class DataLakeManagementV4 {
Integer batchSize = config.getBatchSize();
Integer flushDuration = config.getFlushDuration();
- /**
- * TODO:
- * - Implementation of parameter update for batchSize and flushDuration
- * - Updating multiple retention policies
- */
+ //
+ // TODO:
+ // - Implementation of parameter update for batchSize and flushDuration
+ // - Updating multiple retention policies
+ //
String operation = "CREATE";
if (existingRetentionPolicies.size() > 1) {
@@ -247,10 +243,10 @@ public class DataLakeManagementV4 {
}
public List<DataLakeRetentionPolicy> getAllExistingRetentionPolicies() {
- /**
- * TODO:
- * - Implementation of parameter return for batchSize and flushDuration
- */
+ //
+ // TODO:
+ // - Implementation of parameter return for batchSize and flushDuration
+ //
return new ShowRetentionPolicyQuery(RetentionPolicyQueryParams.from("", "0s")).executeQuery();
}
@@ -278,4 +274,14 @@ public class DataLakeManagementV4 {
private byte[] toBytes(String value) {
return value.getBytes();
}
+
+ private static Long parseTime(String v) {
+ TemporalAccessor temporalAccessor = formatter.parseBest(v,
+ ZonedDateTime::from,
+ LocalDateTime::from,
+ LocalDate::from);
+
+ Instant instant = Instant.from(temporalAccessor);
+ return Instant.EPOCH.until(instant, ChronoUnit.MILLIS);
+ }
}