You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2022/11/08 15:01:38 UTC
[incubator-streampipes] branch STREAMPIPES-621 updated: [STREAMPIPES-621] Add option to load incomplete events into data explorer
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch STREAMPIPES-621
in repository https://gitbox.apache.org/repos/asf/incubator-streampipes.git
The following commit(s) were added to refs/heads/STREAMPIPES-621 by this push:
new 945610a50 [STREAMPIPES-621] Add option to load incomplete events into data explorer
945610a50 is described below
commit 945610a5030c10cde4ba111b684b743203f50fc8
Author: Philipp Zehnder <ze...@fzi.de>
AuthorDate: Tue Nov 8 16:01:25 2022 +0100
[STREAMPIPES-621] Add option to load incomplete events into data explorer
---
.../apache/streampipes/ps/DataLakeResourceV4.java | 26 ++++++++++-----
ui/cypress/README.md | 2 +-
ui/cypress/fixtures/datalake/missingData.json | 5 +++
.../support/utils/DataDownloadDialogUtils.ts | 2 +-
.../support/utils/ProcessingElementTestUtils.ts | 2 +-
ui/cypress/support/utils/connect/ConnectUtils.ts | 2 +-
.../support/utils/{ => datalake}/DataLakeUtils.ts | 38 ++++++++++-----------
.../utils/datalake/DataLakeWidgetTableUtils.ts | 28 ++++++++++++++++
.../dataDownloadDialogTest.smoke.spec.ts | 2 +-
.../tests/datalake/configuration.smoke.spec.ts | 2 +-
ui/cypress/tests/datalake/deleteWidget.ts | 2 +-
.../missingDataInDataLake.spec.ts} | 39 ++++++++++------------
.../datalake/widgetDataConfiguration.smoke.spec.ts | 25 ++++++--------
ui/cypress/tests/datalake/widgets/table.ts | 2 +-
.../tests/datalake/widgets/timeSeriesSave.ts | 4 +--
.../restartStreamPipes/restartStreamPipes2.ts | 2 +-
.../lib/model/datalake/DatalakeQueryParameters.ts | 3 ++
.../model/datalake/data-lake-query-config.model.ts | 3 ++
.../src/lib/query/DatalakeQueryParameterBuilder.ts | 8 ++++-
.../lib/query/data-view-query-generator.service.ts | 8 +++++
.../services/data-export.service.ts | 7 ++--
...ta-explorer-widget-data-settings.component.html | 11 +++++-
...data-explorer-widget-data-settings.component.ts | 1 -
.../data-explorer-dashboard-panel.component.ts | 1 +
.../components/widgets/base/base-widget-config.ts | 1 -
25 files changed, 145 insertions(+), 81 deletions(-)
diff --git a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
index 6a20ad7b9..838cd1aa2 100644
--- a/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
+++ b/streampipes-platform-services/src/main/java/org/apache/streampipes/ps/DataLakeResourceV4.java
@@ -149,6 +149,7 @@ public class DataLakeResourceV4 extends AbstractRestResource {
, @Parameter(in = ParameterIn.QUERY, description = "only return the number of results") @QueryParam(QP_COUNT_ONLY) String countOnly
, @Parameter(in = ParameterIn.QUERY, description = "auto-aggregate the number of results to avoid browser overload") @QueryParam(QP_AUTO_AGGREGATE) boolean autoAggregate
, @Parameter(in = ParameterIn.QUERY, description = "filter conditions (a comma-separated list of filter conditions such as [field,operator,condition])") @QueryParam(QP_FILTER) String filter
+ , @Parameter(in = ParameterIn.QUERY, description = "missingValueBehaviour (ignore or empty)") @QueryParam(QP_MISSING_VALUE_BEHAVIOUR) String missingValueBehaviour
, @Parameter(in = ParameterIn.QUERY, description = "the maximum amount of resulting events, when too high the query status is set to TOO_MUCH_DATA") @QueryParam(QP_MAXIMUM_AMOUNT_OF_EVENTS) Integer maximumAmountOfResults
, @Context UriInfo uriInfo) {
@@ -159,7 +160,8 @@ public class DataLakeResourceV4 extends AbstractRestResource {
} else {
ProvidedQueryParams sanitizedParams = populate(measurementID, queryParams);
try {
- SpQueryResult result = this.dataLakeManagement.getData(sanitizedParams, true);
+ SpQueryResult result =
+ this.dataLakeManagement.getData(sanitizedParams, isIgnoreMissingValues(missingValueBehaviour));
return ok(result);
} catch (RuntimeException e) {
return badRequest(StreamPipesErrorMessage.from(e));
@@ -215,18 +217,11 @@ public class DataLakeResourceV4 extends AbstractRestResource {
format = "csv";
}
- boolean ignoreMissingValues;
- if ("ignore".equals(missingValueBehaviour)) {
- ignoreMissingValues = true;
- } else {
- ignoreMissingValues = false;
- }
-
String outputFormat = format;
StreamingOutput streamingOutput = output -> dataLakeManagement.getDataAsStream(
sanitizedParams,
outputFormat,
- ignoreMissingValues,
+ isIgnoreMissingValues(missingValueBehaviour),
output);
return Response.ok(streamingOutput, MediaType.APPLICATION_OCTET_STREAM).
@@ -235,6 +230,7 @@ public class DataLakeResourceV4 extends AbstractRestResource {
}
}
+
@GET
@Path("/configuration")
@Produces(MediaType.APPLICATION_JSON)
@@ -265,4 +261,16 @@ public class DataLakeResourceV4 extends AbstractRestResource {
return new ProvidedQueryParams(measurementId, queryParamMap);
}
+
+ // Checks if the parameter for missing value behaviour is set
+ private boolean isIgnoreMissingValues(String missingValueBehaviour) {
+ boolean ignoreMissingValues;
+ if ("ignore".equals(missingValueBehaviour)) {
+ ignoreMissingValues = true;
+ } else {
+ ignoreMissingValues = false;
+ }
+ return ignoreMissingValues;
+ }
+
}
diff --git a/ui/cypress/README.md b/ui/cypress/README.md
index d777cb9c1..23d1e3f88 100644
--- a/ui/cypress/README.md
+++ b/ui/cypress/README.md
@@ -37,7 +37,7 @@ This folder contains a WIP framework for automated E2E tests of StreamPipes.
**User**: admin@streampipes.apache.org **Password**: admin
->**Note:** The base URL can be configured in **cypress.json**
+>**Note:** To configure the base URL set the environment variable CYPRESS_BASE_URL (e.g. CYPRESS_BASE_URL=http://localhost:8082)
## Design guidlines
* Before each test the whole system is cleaned to have a fresh environment
diff --git a/ui/cypress/fixtures/datalake/missingData.json b/ui/cypress/fixtures/datalake/missingData.json
new file mode 100644
index 000000000..f652d8d59
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/missingData.json
@@ -0,0 +1,5 @@
+[{"timestamp": 1667904471000, "v1": 4.1, "v2": "abc", "v3": true, "v4": 1},
+{"timestamp": 1667904472000, "v1": 4.2, "v2": "abc", "v3": false, "v4": 2},
+{"timestamp": 1667904473000, "v1": 4.3},
+{"timestamp": 1667904474000, "v1": 4.4, "v2": "abc", "v3": true, "v4": 4},
+{"timestamp": 1667904475000, "v1": 4.5, "v4": 5}]
diff --git a/ui/cypress/support/utils/DataDownloadDialogUtils.ts b/ui/cypress/support/utils/DataDownloadDialogUtils.ts
index 8b017ce13..f0336078e 100644
--- a/ui/cypress/support/utils/DataDownloadDialogUtils.ts
+++ b/ui/cypress/support/utils/DataDownloadDialogUtils.ts
@@ -17,7 +17,7 @@
*/
import { ExportConfig } from '../../../src/app/core-ui/data-download-dialog/model/export-config.model';
-import { DataLakeUtils } from './DataLakeUtils';
+import { DataLakeUtils } from './datalake/DataLakeUtils';
import { FileNameService } from '../../../src/app/core-ui/data-download-dialog/services/file-name.service';
import { CsvFormatExportConfig } from '../../../src/app/core-ui/data-download-dialog/model/format-export-config.model';
diff --git a/ui/cypress/support/utils/ProcessingElementTestUtils.ts b/ui/cypress/support/utils/ProcessingElementTestUtils.ts
index 59519aa6b..efbcdf8f8 100644
--- a/ui/cypress/support/utils/ProcessingElementTestUtils.ts
+++ b/ui/cypress/support/utils/ProcessingElementTestUtils.ts
@@ -19,7 +19,7 @@
import { FileManagementUtils } from './FileManagementUtils';
import { ConnectUtils } from './connect/ConnectUtils';
import { PipelineUtils } from './PipelineUtils';
-import { DataLakeUtils } from './DataLakeUtils';
+import { DataLakeUtils } from './datalake/DataLakeUtils';
import { GenericAdapterBuilder } from '../builder/GenericAdapterBuilder';
import { PipelineBuilder } from '../builder/PipelineBuilder';
import { PipelineElementBuilder } from '../builder/PipelineElementBuilder';
diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts
index a6f9a8ad9..727ff9b1b 100644
--- a/ui/cypress/support/utils/connect/ConnectUtils.ts
+++ b/ui/cypress/support/utils/connect/ConnectUtils.ts
@@ -24,7 +24,7 @@ import { SpecificAdapterBuilder } from '../../builder/SpecificAdapterBuilder';
import { AdapterInput } from '../../model/AdapterInput';
import { ConnectEventSchemaUtils } from '../ConnectEventSchemaUtils';
import { GenericAdapterBuilder } from '../../builder/GenericAdapterBuilder';
-import { DataLakeUtils } from '../DataLakeUtils';
+import { DataLakeUtils } from '../datalake/DataLakeUtils';
export class ConnectUtils {
diff --git a/ui/cypress/support/utils/DataLakeUtils.ts b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
similarity index 85%
rename from ui/cypress/support/utils/DataLakeUtils.ts
rename to ui/cypress/support/utils/datalake/DataLakeUtils.ts
index 013777d00..27fa727dc 100644
--- a/ui/cypress/support/utils/DataLakeUtils.ts
+++ b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
@@ -1,28 +1,28 @@
/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*
*/
-import { GenericAdapterBuilder } from '../builder/GenericAdapterBuilder';
-import { DataLakeFilterConfig } from '../model/DataLakeFilterConfig';
-import { DataExplorerWidget } from '../model/DataExplorerWidget';
-import { DataSetUtils } from './DataSetUtils';
-import { PrepareTestDataUtils } from './PrepareTestDataUtils';
-import { FileManagementUtils } from './FileManagementUtils';
-import { ConnectUtils } from './connect/ConnectUtils';
+import { GenericAdapterBuilder } from '../../builder/GenericAdapterBuilder';
+import { DataLakeFilterConfig } from '../../model/DataLakeFilterConfig';
+import { DataExplorerWidget } from '../../model/DataExplorerWidget';
+import { DataSetUtils } from '../DataSetUtils';
+import { PrepareTestDataUtils } from '../PrepareTestDataUtils';
+import { FileManagementUtils } from '../FileManagementUtils';
+import { ConnectUtils } from '../connect/ConnectUtils';
export class DataLakeUtils {
diff --git a/ui/cypress/support/utils/datalake/DataLakeWidgetTableUtils.ts b/ui/cypress/support/utils/datalake/DataLakeWidgetTableUtils.ts
new file mode 100644
index 000000000..23d492f18
--- /dev/null
+++ b/ui/cypress/support/utils/datalake/DataLakeWidgetTableUtils.ts
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+export class DataLakeWidgetTableUtils {
+
+ /**
+ * Checks how many rows are visible within the table widget in the data explorer
+ * @param amount of expected rows
+ */
+ public static checkRows(amount: number) {
+ cy.dataCy('data-explorer-table-row-timestamp', { timeout: 10000 }).should('have.length', amount);
+ }
+}
diff --git a/ui/cypress/tests/dataDownloadDialog/dataDownloadDialogTest.smoke.spec.ts b/ui/cypress/tests/dataDownloadDialog/dataDownloadDialogTest.smoke.spec.ts
index 53b75ff31..926a7d41f 100644
--- a/ui/cypress/tests/dataDownloadDialog/dataDownloadDialogTest.smoke.spec.ts
+++ b/ui/cypress/tests/dataDownloadDialog/dataDownloadDialogTest.smoke.spec.ts
@@ -18,7 +18,7 @@
import { ExportConfig } from '../../../src/app/core-ui/data-download-dialog/model/export-config.model';
import { DataDownloadDialogUtils } from '../../support/utils/DataDownloadDialogUtils';
-import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
import { PrepareTestDataUtils } from '../../support/utils/PrepareTestDataUtils';
diff --git a/ui/cypress/tests/datalake/configuration.smoke.spec.ts b/ui/cypress/tests/datalake/configuration.smoke.spec.ts
index 58170acae..53becc1ed 100644
--- a/ui/cypress/tests/datalake/configuration.smoke.spec.ts
+++ b/ui/cypress/tests/datalake/configuration.smoke.spec.ts
@@ -17,7 +17,7 @@
*/
import { PipelineUtils } from '../../support/utils/PipelineUtils';
-import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
describe('Test Truncate data in datalake', () => {
diff --git a/ui/cypress/tests/datalake/deleteWidget.ts b/ui/cypress/tests/datalake/deleteWidget.ts
index 2f1d43c29..76b75950e 100644
--- a/ui/cypress/tests/datalake/deleteWidget.ts
+++ b/ui/cypress/tests/datalake/deleteWidget.ts
@@ -15,7 +15,7 @@
* limitations under the License.
*
*/
-import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
describe('Test Table View in Data Explorer', () => {
diff --git a/ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts b/ui/cypress/tests/datalake/missingDataInDataLake.spec.ts
similarity index 50%
copy from ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts
copy to ui/cypress/tests/datalake/missingDataInDataLake.spec.ts
index 253870efd..3f22b3e9d 100644
--- a/ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts
+++ b/ui/cypress/tests/datalake/missingDataInDataLake.spec.ts
@@ -16,33 +16,28 @@
*
*/
-import { DashboardUtils } from '../../../support/utils/DashboardUtils';
-import { DataLakeUtils } from '../../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
+import { PrepareTestDataUtils } from '../../support/utils/PrepareTestDataUtils';
+import { DataLakeWidgetTableUtils } from '../../support/utils/datalake/DataLakeWidgetTableUtils';
-describe('Validate StreamPipes after restart', () => {
- beforeEach('Setup Test', () => {
- cy.login();
- });
- it('Perform Test', () => {
- // Truncate data in db
- DataLakeUtils.goToDatalakeConfiguration();
- cy.dataCy('datalake-truncate-btn')
- .should('be.visible')
- .click();
- cy.dataCy('confirm-truncate-data-btn', { timeout: 10000 })
- .should('be.visible')
- .click();
+describe('Test missing properties in data lake', () => {
+
+ const dataViewName = 'TestView';
+ before('Setup Test', () => {
+ cy.initStreamPipesTest();
+ PrepareTestDataUtils.loadDataIntoDataLake('datalake/missingData.json', 'json_array');
+ });
- // open dashboard
- DashboardUtils.goToDashboard();
- DashboardUtils.showDashboard('testDashboard');
+ it('Test table with missing properties', () => {
+ DataLakeUtils.addDataViewAndTableWidget(dataViewName, 'Persist');
- cy.wait(6000);
+ DataLakeWidgetTableUtils.checkRows(3);
- // validate that data is coming
- DashboardUtils.validateRawWidgetEvents(3);
+ DataLakeUtils.selectDataConfig();
+ cy.dataCy('data-explorer-ignore-missing-values-checkbox').children().click();
+
+ DataLakeWidgetTableUtils.checkRows(5);
});
});
-
diff --git a/ui/cypress/tests/datalake/widgetDataConfiguration.smoke.spec.ts b/ui/cypress/tests/datalake/widgetDataConfiguration.smoke.spec.ts
index a97e5d34e..2fcd7a8e9 100644
--- a/ui/cypress/tests/datalake/widgetDataConfiguration.smoke.spec.ts
+++ b/ui/cypress/tests/datalake/widgetDataConfiguration.smoke.spec.ts
@@ -17,7 +17,8 @@
*/
import { DataLakeFilterConfig } from '../../support/model/DataLakeFilterConfig';
-import { DataLakeUtils } from '../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
+import { DataLakeWidgetTableUtils } from '../../support/utils/datalake/DataLakeWidgetTableUtils';
describe('Test Table View in Data Explorer', () => {
@@ -38,7 +39,7 @@ describe('Test Table View in Data Explorer', () => {
DataLakeUtils.addDataViewAndTableWidget('TestView', 'Persist');
// Validate that X lines are available
- checkTableRows(10);
+ DataLakeWidgetTableUtils.checkRows(10);
// Go back to data configuration
DataLakeUtils.selectDataConfig();
@@ -49,26 +50,26 @@ describe('Test Table View in Data Explorer', () => {
// Test number
let filterConfig = new DataLakeFilterConfig('randomnumber', '22', '=');
DataLakeUtils.dataConfigAddFilter(filterConfig);
- checkTableRows(2);
+ DataLakeWidgetTableUtils.checkRows(2);
DataLakeUtils.dataConfigRemoveFilter();
- checkTableRows(10);
+ DataLakeWidgetTableUtils.checkRows(10);
// Test number greater then
filterConfig = new DataLakeFilterConfig('randomnumber', '50', '>');
DataLakeUtils.dataConfigAddFilter(filterConfig);
- checkTableRows(5);
+ DataLakeWidgetTableUtils.checkRows(5);
DataLakeUtils.dataConfigRemoveFilter();
// Test number smaller then
filterConfig = new DataLakeFilterConfig('randomnumber', '50', '<');
DataLakeUtils.dataConfigAddFilter(filterConfig);
- checkTableRows(5);
+ DataLakeWidgetTableUtils.checkRows(5);
DataLakeUtils.dataConfigRemoveFilter();
// Test boolean
filterConfig = new DataLakeFilterConfig('randombool', 'true', '=');
DataLakeUtils.dataConfigAddFilter(filterConfig);
- checkTableRows(6);
+ DataLakeWidgetTableUtils.checkRows(6);
DataLakeUtils.dataConfigRemoveFilter();
// Test string & if filter is persisted correctly
@@ -76,10 +77,10 @@ describe('Test Table View in Data Explorer', () => {
DataLakeUtils.checkIfFilterIsSet(0);
DataLakeUtils.dataConfigAddFilter(filterConfig);
DataLakeUtils.checkIfFilterIsSet(1);
- checkTableRows(4);
+ DataLakeWidgetTableUtils.checkRows(4);
DataLakeUtils.saveAndReEditWidget('TestView');
DataLakeUtils.checkIfFilterIsSet(1);
- checkTableRows(4);
+ DataLakeWidgetTableUtils.checkRows(4);
DataLakeUtils.dataConfigRemoveFilter();
/**
@@ -90,15 +91,11 @@ describe('Test Table View in Data Explorer', () => {
cy.wait(1000);
cy.dataCy('data-explorer-table-row-randomtext', { timeout: 10000 }).first({ timeout: 10000 }).contains('a', { timeout: 10000 });
cy.dataCy('data-explorer-table-row-randomtext', { timeout: 10000 }).last({ timeout: 10000 }).contains('c', { timeout: 10000 });
- checkTableRows(10);
+ DataLakeWidgetTableUtils.checkRows(10);
DataLakeUtils.saveAndReEditWidget('TestView');
cy.dataCy('data-explorer-group-by-randomtext').find('input').should('be.checked');
DataLakeUtils.clickGroupBy('randomtext');
});
- const checkTableRows = (numberOfRows: number) => {
- cy.dataCy('data-explorer-table-row-timestamp', { timeout: 10000 }).should('have.length', numberOfRows);
- };
-
});
diff --git a/ui/cypress/tests/datalake/widgets/table.ts b/ui/cypress/tests/datalake/widgets/table.ts
index 26e21a543..3728569a2 100644
--- a/ui/cypress/tests/datalake/widgets/table.ts
+++ b/ui/cypress/tests/datalake/widgets/table.ts
@@ -16,7 +16,7 @@
*
*/
-import { DataLakeUtils } from '../../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../../support/utils/datalake/DataLakeUtils';
describe('Test Table View in Data Explorer', () => {
diff --git a/ui/cypress/tests/datalake/widgets/timeSeriesSave.ts b/ui/cypress/tests/datalake/widgets/timeSeriesSave.ts
index 1a5fdf606..c4b6243a4 100644
--- a/ui/cypress/tests/datalake/widgets/timeSeriesSave.ts
+++ b/ui/cypress/tests/datalake/widgets/timeSeriesSave.ts
@@ -16,7 +16,7 @@
*
*/
-import { DataLakeUtils } from '../../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../../support/utils/datalake/DataLakeUtils';
const testView1 = 'TestView1';
const testView2 = 'TestView2';
@@ -31,7 +31,7 @@ describe('Test if widget configuration is updated correctly', () => {
// Create first test data view with one time series widget
DataLakeUtils.addDataViewAndTimeSeriesWidget(testView1, dataSet);
DataLakeUtils.saveDataExplorerWidgetConfiguration();
- DataLakeUtils.clickStartTab();
+ // DataLakeUtils.clickStartTab();
// Create second test data view with one time series widget
DataLakeUtils.addDataViewAndTimeSeriesWidget(testView2, dataSet);
diff --git a/ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts b/ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts
index 253870efd..826cf708b 100644
--- a/ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts
+++ b/ui/cypress/tests/experimental/restartStreamPipes/restartStreamPipes2.ts
@@ -17,7 +17,7 @@
*/
import { DashboardUtils } from '../../../support/utils/DashboardUtils';
-import { DataLakeUtils } from '../../../support/utils/DataLakeUtils';
+import { DataLakeUtils } from '../../../support/utils/datalake/DataLakeUtils';
describe('Validate StreamPipes after restart', () => {
beforeEach('Setup Test', () => {
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/datalake/DatalakeQueryParameters.ts b/ui/projects/streampipes/platform-services/src/lib/model/datalake/DatalakeQueryParameters.ts
index d69493548..422aec2a5 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/datalake/DatalakeQueryParameters.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/datalake/DatalakeQueryParameters.ts
@@ -16,6 +16,8 @@
*
*/
+import { MissingValueBehaviour } from './data-lake-query-config.model';
+
export class DatalakeQueryParameters {
public columns: string;
public startDate: number;
@@ -30,6 +32,7 @@ export class DatalakeQueryParameters {
public countOnly: boolean;
public autoAggregate: boolean;
public filter: string;
+ public missingValueBehaviour: MissingValueBehaviour;
public maximumAmountOfEvents: number;
// should be only used for multi-query requests
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/datalake/data-lake-query-config.model.ts b/ui/projects/streampipes/platform-services/src/lib/model/datalake/data-lake-query-config.model.ts
index f7b0d8390..a7b881e83 100644
--- a/ui/projects/streampipes/platform-services/src/lib/model/datalake/data-lake-query-config.model.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/model/datalake/data-lake-query-config.model.ts
@@ -72,4 +72,7 @@ export interface SourceConfig {
export interface DataExplorerDataConfig {
sourceConfigs: SourceConfig[];
ignoreTooMuchDataWarning: boolean;
+ ignoreMissingValues: boolean;
}
+
+export type MissingValueBehaviour = 'ignore' | 'empty';
diff --git a/ui/projects/streampipes/platform-services/src/lib/query/DatalakeQueryParameterBuilder.ts b/ui/projects/streampipes/platform-services/src/lib/query/DatalakeQueryParameterBuilder.ts
index 14dc4a4a9..f14dfa04a 100644
--- a/ui/projects/streampipes/platform-services/src/lib/query/DatalakeQueryParameterBuilder.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/query/DatalakeQueryParameterBuilder.ts
@@ -16,7 +16,7 @@
*
*/
-import { FieldConfig, SelectedFilter } from '../model/datalake/data-lake-query-config.model';
+import { FieldConfig, MissingValueBehaviour, SelectedFilter } from '../model/datalake/data-lake-query-config.model';
import { DatalakeQueryParameters } from '../model/datalake/DatalakeQueryParameters';
export class DatalakeQueryParameterBuilder {
@@ -149,6 +149,12 @@ export class DatalakeQueryParameterBuilder {
return this;
}
+ public withMissingValueBehaviour(missingValueBehaviour: MissingValueBehaviour): DatalakeQueryParameterBuilder {
+ this.queryParams.missingValueBehaviour = missingValueBehaviour;
+
+ return this;
+ }
+
public build(): DatalakeQueryParameters {
return this.queryParams;
}
diff --git a/ui/projects/streampipes/platform-services/src/lib/query/data-view-query-generator.service.ts b/ui/projects/streampipes/platform-services/src/lib/query/data-view-query-generator.service.ts
index 97b64fc5e..44122ee55 100644
--- a/ui/projects/streampipes/platform-services/src/lib/query/data-view-query-generator.service.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/query/data-view-query-generator.service.ts
@@ -44,6 +44,7 @@ export class DataViewQueryGeneratorService {
const dataLakeConfiguration = this.generateQuery(startTime,
endTime,
sourceConfig,
+ dataConfig.ignoreMissingValues,
maximumResultingEvents);
return this.dataLakeRestService
@@ -54,6 +55,7 @@ export class DataViewQueryGeneratorService {
generateQuery(startTime: number,
endTime: number,
sourceConfig: SourceConfig,
+ ignoreEventsWithMissingValues: boolean,
maximumResultingEvents: number = -1): DatalakeQueryParameters {
const queryBuilder = DatalakeQueryParameterBuilder.create(startTime, endTime);
const queryConfig = sourceConfig.queryConfig;
@@ -92,6 +94,12 @@ export class DataViewQueryGeneratorService {
}
}
+ if (ignoreEventsWithMissingValues) {
+ queryBuilder.withMissingValueBehaviour('ignore');
+ } else {
+ queryBuilder.withMissingValueBehaviour('empty');
+ }
+
const dataLakeQueryParameter = queryBuilder.build();
if (maximumResultingEvents !== -1) {
diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
index 61a5d52a6..beda107be 100644
--- a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
+++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
@@ -47,7 +47,8 @@ export class DataExportService {
exportConfig.formatExportConfig.exportFormat,
exportConfig.formatExportConfig['delimiter'],
exportConfig.dataExportConfig.missingValueBehaviour,
- this.generateQueryRequest(exportConfig, dataDownloadDialogModel));
+ this.generateQueryRequest(exportConfig, dataDownloadDialogModel)
+ );
} else {
// case for 'all' and 'customInverval'
let startTime, endTime = undefined;
@@ -95,7 +96,9 @@ export class DataExportService {
.generateQuery(
exportConfig.dataExportConfig.dateRange.startDate.getTime(),
exportConfig.dataExportConfig.dateRange.startDate.getTime(),
- dataDownloadDialogModel.dataExplorerDataConfig.sourceConfigs[selectedQueryIndex]);
+ dataDownloadDialogModel.dataExplorerDataConfig.sourceConfigs[selectedQueryIndex],
+ false
+ );
}
/**
diff --git a/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.html b/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.html
index c27679835..bf7c20f75 100644
--- a/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.html
+++ b/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.html
@@ -149,6 +149,15 @@
</button>
</div>
<div class="p-10">
- <mat-checkbox [(ngModel)]="dataConfig.ignoreTooMuchDataWarning">Deactivate browser overload warning</mat-checkbox>
+ <mat-checkbox
+ [(ngModel)]="dataConfig.ignoreTooMuchDataWarning">
+ Deactivate browser overload warning
+ </mat-checkbox>
+ <mat-checkbox
+ [(ngModel)]="dataConfig.ignoreMissingValues"
+ (change)="triggerDataRefresh()"
+ data-cy="data-explorer-ignore-missing-values-checkbox">
+ Ignore Events with missing values
+ </mat-checkbox>
</div>
</div>
diff --git a/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.ts b/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.ts
index 7768df97a..94f95d2c1 100644
--- a/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.ts
+++ b/ui/src/app/data-explorer/components/designer-panel/data-settings/data-explorer-widget-data-settings.component.ts
@@ -25,7 +25,6 @@ import {
DataViewDataExplorerService,
SourceConfig
} from '@streampipes/platform-services';
-import { MatSelectChange } from '@angular/material/select';
import { Tuple2 } from '../../../../core-model/base/Tuple2';
import { zip } from 'rxjs';
import { WidgetConfigurationService } from '../../../services/widget-configuration.service';
diff --git a/ui/src/app/data-explorer/components/panel/data-explorer-dashboard-panel.component.ts b/ui/src/app/data-explorer/components/panel/data-explorer-dashboard-panel.component.ts
index 37f0a21fd..250610044 100644
--- a/ui/src/app/data-explorer/components/panel/data-explorer-dashboard-panel.component.ts
+++ b/ui/src/app/data-explorer/components/panel/data-explorer-dashboard-panel.component.ts
@@ -279,6 +279,7 @@ export class DataExplorerDashboardPanelComponent implements OnInit, OnDestroy {
this.currentlyConfiguredWidget.baseAppearanceConfig.widgetTitle =
'New Widget';
this.currentlyConfiguredWidget.dataConfig = {};
+ this.currentlyConfiguredWidget.dataConfig.ignoreMissingValues = true;
this.currentlyConfiguredWidget.baseAppearanceConfig.backgroundColor =
'#FFFFFF';
this.currentlyConfiguredWidget.baseAppearanceConfig.textColor = '#3e3e3e';
diff --git a/ui/src/app/data-explorer/components/widgets/base/base-widget-config.ts b/ui/src/app/data-explorer/components/widgets/base/base-widget-config.ts
index 6003ee584..02e86c3e9 100644
--- a/ui/src/app/data-explorer/components/widgets/base/base-widget-config.ts
+++ b/ui/src/app/data-explorer/components/widgets/base/base-widget-config.ts
@@ -31,7 +31,6 @@ import { DataExplorerFieldProviderService } from '../../../services/data-explore
import { WidgetType } from '../../../registry/data-explorer-widgets';
@Directive()
-// eslint-disable-next-line @angular-eslint/directive-class-suffix
export abstract class BaseWidgetConfig<T extends DataExplorerWidgetModel, V extends DataExplorerVisConfig> implements OnChanges {
@Input() currentlyConfiguredWidget: T;