You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ri...@apache.org on 2025/01/31 12:42:27 UTC
(streampipes) 01/02: feat(#3448): Add excel export
This is an automated email from the ASF dual-hosted git repository.
riemer pushed a commit to branch 3448-add-excel-export-feature
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit d7b27ed033f280b67be400c50a02dc6645f50c5f
Author: Dominik Riemer <do...@gmail.com>
AuthorDate: Thu Jan 30 22:38:22 2025 +0100
feat(#3448): Add excel export
---
streampipes-data-explorer-export/pom.xml | 34 ++++--
.../export/ConfiguredCsvOutputWriter.java | 13 ++-
.../export/ConfiguredExcelOutputWriter.java | 122 +++++++++++++++++++++
.../export/ConfiguredJsonOutputWriter.java | 6 +-
.../export/ConfiguredOutputWriter.java | 37 ++++++-
.../dataexplorer/export/OutputFormat.java | 14 ++-
.../export/TestConfiguredCsvOutputWriter.java | 2 +-
.../export/TestConfiguredJsonOutputWriter.java | 2 +-
.../dataexplorer/StreamedQueryResultProvider.java | 5 +-
.../datalake/param/SupportedRestQueryParams.java | 10 +-
...{PipelineElementFile.java => FileResource.java} | 4 +-
.../rest/impl/datalake/DataLakeResource.java | 17 ++-
.../src/lib/apis/datalake-rest.service.ts | 19 ++--
.../basic-inner-panel.component.html | 10 +-
.../select-format/select-format.component.html | 78 ++++++++++++-
.../select-format/select-format.component.ts | 27 ++++-
.../data-download-dialog.component.ts | 3 +-
.../model/format-export-config.model.ts | 18 ++-
.../services/data-export.service.ts | 13 +--
.../services/file-name.service.spec.ts | 6 +-
.../services/file-name.service.ts | 2 +-
21 files changed, 376 insertions(+), 66 deletions(-)
diff --git a/streampipes-data-explorer-export/pom.xml b/streampipes-data-explorer-export/pom.xml
index e108241836..0f7d6eac5d 100644
--- a/streampipes-data-explorer-export/pom.xml
+++ b/streampipes-data-explorer-export/pom.xml
@@ -29,6 +29,10 @@
This module contains all components and functionalities related to exporting data from the data explorer storage.
</description>
+ <properties>
+ <apache-poi.version>5.4.0</apache-poi.version>
+ </properties>
+
<dependencies>
<!-- StreamPipes dependencies -->
<dependency>
@@ -36,6 +40,27 @@
<artifactId>streampipes-model</artifactId>
<version>0.98.0-SNAPSHOT</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-storage-management</artifactId>
+ <version>0.98.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.streampipes</groupId>
+ <artifactId>streampipes-pipeline-management</artifactId>
+ <version>0.98.0-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.poi</groupId>
+ <artifactId>poi</artifactId>
+ <version>${apache-poi.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.poi</groupId>
+ <artifactId>poi-ooxml</artifactId>
+ <version>${apache-poi.version}</version>
+ </dependency>
<!-- Test dependencies -->
<dependency>
@@ -44,11 +69,4 @@
<scope>test</scope>
</dependency>
</dependencies>
-
- <properties>
- <maven.compiler.source>17</maven.compiler.source>
- <maven.compiler.target>17</maven.compiler.target>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
-</project>
\ No newline at end of file
+</project>
diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
index 0f78498656..cbc7eaba95 100644
--- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
+++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredCsvOutputWriter.java
@@ -19,6 +19,7 @@
package org.apache.streampipes.dataexplorer.export;
import org.apache.streampipes.dataexplorer.export.item.CsvItemGenerator;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams;
@@ -35,10 +36,18 @@ public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter {
private CsvItemGenerator itemGenerator;
private String delimiter = COMMA;
+ private DataLakeMeasure schema;
+ private String headerColumnNameStrategy;
@Override
- public void configure(ProvidedRestQueryParams params,
+ public void configure(DataLakeMeasure schema,
+ ProvidedRestQueryParams params,
boolean ignoreMissingValues) {
+ this.schema = schema;
+ this.headerColumnNameStrategy = params
+ .getProvidedParams()
+ .getOrDefault(SupportedRestQueryParams.QP_HEADER_COLUMN_NAME, "key");
+
if (params.has(SupportedRestQueryParams.QP_CSV_DELIMITER)) {
delimiter = params.getAsString(SupportedRestQueryParams.QP_CSV_DELIMITER).equals("comma") ? COMMA : SEMICOLON;
}
@@ -69,7 +78,7 @@ public class ConfiguredCsvOutputWriter extends ConfiguredOutputWriter {
private String makeHeaderLine(List<String> columns) {
StringJoiner joiner = new StringJoiner(this.delimiter);
- columns.forEach(joiner::add);
+ columns.forEach(c -> joiner.add(getHeaderName(schema, c, headerColumnNameStrategy)));
return joiner + LINE_SEPARATOR;
}
}
diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredExcelOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredExcelOutputWriter.java
new file mode 100644
index 0000000000..fbe5a86351
--- /dev/null
+++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredExcelOutputWriter.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.dataexplorer.export;
+
+import org.apache.streampipes.manager.file.FileManager;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
+import org.apache.streampipes.model.datalake.param.SupportedRestQueryParams;
+import org.apache.streampipes.model.file.FileMetadata;
+import org.apache.streampipes.storage.api.CRUDStorage;
+
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.xssf.streaming.SXSSFWorkbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Objects;
+
+public class ConfiguredExcelOutputWriter extends ConfiguredOutputWriter {
+
+ private final CRUDStorage<FileMetadata> storage;
+
+ private SXSSFWorkbook wb;
+ private Sheet ws;
+ private boolean useTemplate = false;
+ private int startRow = 0;
+ private String templateId;
+ private DataLakeMeasure schema;
+ private String headerColumnNameStrategy;
+
+
+ public ConfiguredExcelOutputWriter(CRUDStorage<FileMetadata> fileMetadataStorage) {
+ this.storage = fileMetadataStorage;
+ }
+
+ @Override
+ public void configure(DataLakeMeasure schema,
+ ProvidedRestQueryParams params,
+ boolean ignoreMissingValues) {
+ var qp = params.getProvidedParams();
+ this.schema = schema;
+ this.headerColumnNameStrategy = qp.getOrDefault(SupportedRestQueryParams.QP_HEADER_COLUMN_NAME, "key");
+ if (qp.containsKey(SupportedRestQueryParams.QP_XLSX_USE_TEMPLATE)) {
+ this.useTemplate = true;
+ this.startRow = Integer.parseInt(qp.getOrDefault(SupportedRestQueryParams.QP_XLSX_START_ROW, "0"));
+ this.templateId = qp.getOrDefault(SupportedRestQueryParams.QP_XLSX_TEMPLATE_ID, null);
+ }
+ }
+
+ @Override
+ public void beforeFirstItem(OutputStream outputStream) {
+ if (useTemplate && Objects.nonNull(templateId)) {
+ var fileMetadata = storage.getElementById(templateId);
+ if (fileMetadata != null) {
+ var path = new FileManager().getFile(fileMetadata.getFilename()).getAbsoluteFile().toPath();
+ try (InputStream is = Files.newInputStream(path)) {
+ XSSFWorkbook templateWorkbook = new XSSFWorkbook(is);
+ wb = new SXSSFWorkbook(templateWorkbook);
+ ws = wb.getSheetAt(0);
+ } catch (IOException e) {
+ createNewWorkbook();
+ }
+ }
+ } else {
+ createNewWorkbook();
+ }
+ }
+
+ private void createNewWorkbook() {
+ wb = new SXSSFWorkbook();
+ ws = wb.createSheet();
+ }
+
+ @Override
+ public void afterLastItem(OutputStream outputStream) throws IOException {
+ wb.write(outputStream);
+ wb.close();
+ }
+
+ @Override
+ public void writeItem(OutputStream outputStream,
+ List<Object> row,
+ List<String> columnNames, boolean firstObject) throws IOException {
+ var excelRow = ws.createRow(startRow);
+ int columnIndex = 0;
+ if (!firstObject) {
+ for (Object column : row) {
+ var cell = excelRow.createCell(columnIndex);
+ cell.setCellValue(String.valueOf(column));
+ columnIndex++;
+ }
+ } else {
+ for (String column : columnNames) {
+ var cell = excelRow.createCell(columnIndex);
+ var headerName = getHeaderName(schema, String.valueOf(column), headerColumnNameStrategy);
+ cell.setCellValue(headerName);
+ columnIndex++;
+ }
+ }
+ startRow++;
+ }
+}
diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java
index d7d0b93c8c..5bfb631e02 100644
--- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java
+++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredJsonOutputWriter.java
@@ -19,8 +19,9 @@
package org.apache.streampipes.dataexplorer.export;
import org.apache.streampipes.dataexplorer.export.item.ItemGenerator;
-import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
import org.apache.streampipes.dataexplorer.export.item.JsonItemGenerator;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -40,7 +41,8 @@ public class ConfiguredJsonOutputWriter extends ConfiguredOutputWriter {
}
@Override
- public void configure(ProvidedRestQueryParams params,
+ public void configure(DataLakeMeasure schema,
+ ProvidedRestQueryParams params,
boolean ignoreMissingValues) {
// do nothing
}
diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java
index c7ff305843..6eac3b6145 100644
--- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java
+++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/ConfiguredOutputWriter.java
@@ -18,24 +18,55 @@
package org.apache.streampipes.dataexplorer.export;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
import org.apache.streampipes.model.datalake.param.ProvidedRestQueryParams;
+import org.apache.streampipes.model.schema.EventProperty;
import java.io.IOException;
import java.io.OutputStream;
import java.util.List;
+import java.util.Objects;
public abstract class ConfiguredOutputWriter {
- public static ConfiguredOutputWriter getConfiguredWriter(OutputFormat format,
+ public static ConfiguredOutputWriter getConfiguredWriter(DataLakeMeasure schema,
+ OutputFormat format,
ProvidedRestQueryParams params,
boolean ignoreMissingValues) {
var writer = format.getWriter();
- writer.configure(params, ignoreMissingValues);
+ writer.configure(schema, params, ignoreMissingValues);
return writer;
}
- public abstract void configure(ProvidedRestQueryParams params,
+ protected String getHeaderName(DataLakeMeasure schema,
+ String runtimeName,
+ String headerColumnNameStrategy) {
+ if (Objects.nonNull(schema) && headerColumnNameStrategy.equals("label")) {
+ return schema
+ .getEventSchema()
+ .getEventProperties()
+ .stream()
+ .filter(ep -> ep.getRuntimeName().equals(runtimeName))
+ .findFirst()
+ .map(ep -> extractLabel(ep, runtimeName))
+ .orElse(runtimeName);
+ } else {
+ return runtimeName;
+ }
+ }
+
+ private String extractLabel(EventProperty ep,
+ String runtimeName) {
+ if (Objects.nonNull(ep.getLabel())) {
+ return ep.getLabel();
+ } else {
+ return runtimeName;
+ }
+ }
+
+ public abstract void configure(DataLakeMeasure schema,
+ ProvidedRestQueryParams params,
boolean ignoreMissingValues);
public abstract void beforeFirstItem(OutputStream outputStream) throws IOException;
diff --git a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java
index 2342f5879f..d1237afc23 100644
--- a/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java
+++ b/streampipes-data-explorer-export/src/main/java/org/apache/streampipes/dataexplorer/export/OutputFormat.java
@@ -18,11 +18,15 @@
package org.apache.streampipes.dataexplorer.export;
+import org.apache.streampipes.storage.management.StorageDispatcher;
+
+import java.util.Arrays;
import java.util.function.Supplier;
public enum OutputFormat {
JSON(ConfiguredJsonOutputWriter::new),
- CSV(ConfiguredCsvOutputWriter::new);
+ CSV(ConfiguredCsvOutputWriter::new),
+ XLSX(() -> new ConfiguredExcelOutputWriter(StorageDispatcher.INSTANCE.getNoSqlStore().getFileMetadataStorage()));
private final Supplier<ConfiguredOutputWriter> writerSupplier;
@@ -33,4 +37,12 @@ public enum OutputFormat {
public ConfiguredOutputWriter getWriter() {
return writerSupplier.get();
}
+
+ public static OutputFormat fromString(String desiredFormat) {
+ return Arrays.stream(
+ OutputFormat.values())
+ .filter(format -> format.name().equalsIgnoreCase(desiredFormat))
+ .findFirst()
+ .orElseThrow(() -> new IllegalArgumentException(String.format("Could not find format %s", desiredFormat)));
+ }
}
diff --git a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
index 0fc7cec228..15a6b66e28 100644
--- a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
+++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredCsvOutputWriter.java
@@ -36,7 +36,7 @@ public class TestConfiguredCsvOutputWriter extends TestConfiguredOutputWriter {
@Test
public void testCsvOutputWriter() throws IOException {
var writer = new ConfiguredCsvOutputWriter();
- writer.configure(new ProvidedRestQueryParams(null, new HashMap<>()), true);
+ writer.configure(null, new ProvidedRestQueryParams(null, new HashMap<>()), true);
try (var outputStream = new ByteArrayOutputStream()) {
writer.beforeFirstItem(outputStream);
diff --git a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java
index b67b65ea28..3d5dc795d1 100644
--- a/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java
+++ b/streampipes-data-explorer-export/src/test/java/org/apache/streampipes/dataexplorer/export/TestConfiguredJsonOutputWriter.java
@@ -37,7 +37,7 @@ public class TestConfiguredJsonOutputWriter extends TestConfiguredOutputWriter {
@Test
public void testJsonOutputWriter() throws IOException {
var writer = new ConfiguredJsonOutputWriter();
- writer.configure(new ProvidedRestQueryParams(null, new HashMap<>()), true);
+ writer.configure(null, new ProvidedRestQueryParams(null, new HashMap<>()), true);
try (var outputStream = new ByteArrayOutputStream()) {
writer.beforeFirstItem(outputStream);
diff --git a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java
index 18e288ec98..936fc093d4 100644
--- a/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java
+++ b/streampipes-data-explorer/src/main/java/org/apache/streampipes/dataexplorer/StreamedQueryResultProvider.java
@@ -51,15 +51,16 @@ public class StreamedQueryResultProvider extends QueryResultProvider {
public void getDataAsStream(OutputStream outputStream) throws IOException {
var usesLimit = queryParams.has(SupportedRestQueryParams.QP_LIMIT);
+ var measurement = findByMeasurementName(queryParams.getMeasurementId()).get();
+
var configuredWriter = ConfiguredOutputWriter
- .getConfiguredWriter(format, queryParams, ignoreMissingData);
+ .getConfiguredWriter(measurement, format, queryParams, ignoreMissingData);
if (!queryParams.has(SupportedRestQueryParams.QP_LIMIT)) {
queryParams.update(SupportedRestQueryParams.QP_LIMIT, MAX_RESULTS_PER_QUERY);
}
var limit = queryParams.getAsInt(SupportedRestQueryParams.QP_LIMIT);
- var measurement = findByMeasurementName(queryParams.getMeasurementId()).get();
SpQueryResult dataResult;
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java
index 881be1c659..4ec3a83894 100644
--- a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/param/SupportedRestQueryParams.java
@@ -40,6 +40,10 @@ public class SupportedRestQueryParams {
public static final String QP_AUTO_AGGREGATE = "autoAggregate";
public static final String QP_FILTER = "filter";
public static final String QP_MAXIMUM_AMOUNT_OF_EVENTS = "maximumAmountOfEvents";
+ public static final String QP_XLSX_USE_TEMPLATE = "useTemplate";
+ public static final String QP_XLSX_TEMPLATE_ID = "templateId";
+ public static final String QP_XLSX_START_ROW = "startRow";
+ public static final String QP_HEADER_COLUMN_NAME = "headerColumnName";
public static final List<String> SUPPORTED_PARAMS = Arrays.asList(
QP_COLUMNS,
@@ -58,7 +62,11 @@ public class SupportedRestQueryParams {
QP_AUTO_AGGREGATE,
QP_MISSING_VALUE_BEHAVIOUR,
QP_FILTER,
- QP_MAXIMUM_AMOUNT_OF_EVENTS
+ QP_MAXIMUM_AMOUNT_OF_EVENTS,
+ QP_XLSX_START_ROW,
+ QP_XLSX_TEMPLATE_ID,
+ QP_XLSX_USE_TEMPLATE,
+ QP_HEADER_COLUMN_NAME
);
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java
similarity index 98%
rename from streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
rename to streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java
index c5ff418e82..14b43b4eed 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/PipelineElementFile.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/FileResource.java
@@ -52,11 +52,11 @@ import static org.springframework.http.HttpStatus.NOT_FOUND;
@RestController
@RequestMapping("/api/v2/files")
-public class PipelineElementFile extends AbstractAuthGuardedRestResource {
+public class FileResource extends AbstractAuthGuardedRestResource {
private final FileManager fileManager;
- public PipelineElementFile() {
+ public FileResource() {
this.fileManager = new FileManager();
}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
index acc464d960..c757a61688 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeResource.java
@@ -66,6 +66,7 @@ import static org.apache.streampipes.model.datalake.param.SupportedRestQueryPara
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_FILTER;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_FORMAT;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_GROUP_BY;
+import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_HEADER_COLUMN_NAME;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_LIMIT;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_MAXIMUM_AMOUNT_OF_EVENTS;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_MISSING_VALUE_BEHAVIOUR;
@@ -74,6 +75,9 @@ import static org.apache.streampipes.model.datalake.param.SupportedRestQueryPara
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_PAGE;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_START_DATE;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_TIME_INTERVAL;
+import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_XLSX_START_ROW;
+import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_XLSX_TEMPLATE_ID;
+import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.QP_XLSX_USE_TEMPLATE;
import static org.apache.streampipes.model.datalake.param.SupportedRestQueryParams.SUPPORTED_PARAMS;
@RestController
@@ -315,6 +319,14 @@ public class DataLakeResource extends AbstractRestResource {
description = "filter conditions (a comma-separated list of filter conditions"
+ "such as [field,operator,condition])")
@RequestParam(value = QP_FILTER, required = false) String filter,
+ @Parameter(in = ParameterIn.QUERY, description = "Excel export with template")
+ @RequestParam(value = QP_XLSX_USE_TEMPLATE, required = false) boolean useTemplate
+ , @Parameter(in = ParameterIn.QUERY, description = "ID of the excel template file to use")
+ @RequestParam(value = QP_XLSX_TEMPLATE_ID, required = false) String templateId
+ , @Parameter(in = ParameterIn.QUERY, description = "The first row in the excel file where data should be written")
+ @RequestParam(value = QP_XLSX_START_ROW, required = false) Integer startRow,
+ @Parameter(in = ParameterIn.QUERY, description = "Use either label or key as the column header")
+ @RequestParam(value = QP_HEADER_COLUMN_NAME, required = false) String headerColumnName,
@RequestParam Map<String, String> queryParams) {
@@ -326,7 +338,7 @@ public class DataLakeResource extends AbstractRestResource {
format = "csv";
}
- OutputFormat outputFormat = format.equals("csv") ? OutputFormat.CSV : OutputFormat.JSON;
+ var outputFormat = OutputFormat.fromString(format);
StreamingResponseBody streamingOutput = output -> dataExplorerQueryManagement.getDataAsStream(
sanitizedParams,
outputFormat,
@@ -335,7 +347,8 @@ public class DataLakeResource extends AbstractRestResource {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
- headers.setContentDispositionFormData("attachment", "datalake." + outputFormat);
+ headers.setContentDispositionFormData("attachment",
+ "datalake." + outputFormat.toString().toLowerCase());
return ResponseEntity.ok()
.headers(headers)
diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
index 76f552ac3e..4b54c3e802 100644
--- a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
@@ -120,18 +120,16 @@ export class DatalakeRestService {
downloadRawData(
index: string,
- format: string,
- delimiter: string,
+ formatConfig: Record<string, any>,
missingValueBehaviour: string,
startTime?: number,
endTime?: number,
) {
const queryParams =
startTime && endTime
- ? { format, delimiter, startDate: startTime, endDate: endTime }
+ ? { ...formatConfig, startDate: startTime, endDate: endTime }
: {
- format,
- delimiter,
+ ...formatConfig,
missingValueBehaviour,
};
return this.buildDownloadRequest(index, queryParams);
@@ -139,23 +137,20 @@ export class DatalakeRestService {
downloadQueriedData(
index: string,
- format: string,
- delimiter: string,
+ formatConfig: Record<string, any>,
missingValueBehaviour: string,
queryParams: DatalakeQueryParameters,
) {
- (queryParams as any).format = format;
- (queryParams as any).delimiter = delimiter;
- (queryParams as any).missingValueBehaviour = missingValueBehaviour;
+ const qp = { ...formatConfig, ...queryParams, missingValueBehaviour };
- return this.buildDownloadRequest(index, queryParams);
+ return this.buildDownloadRequest(index, qp);
}
buildDownloadRequest(index: string, queryParams: any) {
const url = this.dataLakeUrl + '/measurements/' + index + '/download';
const request = new HttpRequest('GET', url, {
reportProgress: true,
- responseType: 'text',
+ responseType: 'blob',
params: this.toHttpParams(queryParams),
});
diff --git a/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html b/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html
index 73038461b7..7cae7a6df9 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html
+++ b/ui/projects/streampipes/shared-ui/src/lib/components/basic-inner-panel/basic-inner-panel.component.html
@@ -20,12 +20,11 @@
fxLayout="column"
fxFlex="100"
class="panel-outer"
- [ngStyle]="{ margin: outerMargin }"
+ [ngStyle]="{ margin: outerMargin, height: '100%' }"
>
<div
class="general-panel-header"
fxLayout="row"
- fxFlex="100"
fxLayoutAlign="start center"
*ngIf="!hideToolbar"
>
@@ -35,11 +34,14 @@
{{ panelTitle }}
</div>
</div>
- <ng-content select="[header]" fxFlex class="pr-5"></ng-content>
+ <ng-content select="[header]" class="pr-5"></ng-content>
</div>
</div>
- <div class="general-panel" [ngStyle]="{ padding: innerPadding }">
+ <div
+ class="general-panel"
+ [ngStyle]="{ padding: innerPadding, height: '100%' }"
+ >
<ng-content fxFlex="100"></ng-content>
</div>
</div>
diff --git a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html
index 77a4f7445b..423ac7035c 100644
--- a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html
+++ b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.html
@@ -21,7 +21,7 @@
<h5>Download Format</h5>
<mat-radio-group
class="sp-radio-group"
- [(ngModel)]="formatExportConfig.exportFormat"
+ [(ngModel)]="formatExportConfig.format"
>
<mat-radio-button
value="json"
@@ -37,9 +37,16 @@
>
CSV
</mat-radio-button>
+ <mat-radio-button
+ value="xlsx"
+ class="sp-radio-button"
+ data-cy="download-configuration-xlsx"
+ >
+ Excel
+ </mat-radio-button>
</mat-radio-group>
</div>
-<div *ngIf="formatExportConfig.exportFormat === 'csv'" class="mt-10">
+<div *ngIf="formatExportConfig.format === 'csv'" class="mt-10">
<h5>Delimiter</h5>
<mat-radio-group
[(ngModel)]="formatExportConfig.delimiter"
@@ -49,13 +56,72 @@
value="comma"
class="sp-radio-button"
data-cy="download-configuration-delimiter-comma"
- > ,</mat-radio-button
- >
+ > ,
+ </mat-radio-button>
<mat-radio-button
value="semicolon"
class="sp-radio-button"
data-cy="download-configuration-delimiter-semicolon"
- > ;</mat-radio-button
- >
+ > ;
+ </mat-radio-button>
+ </mat-radio-group>
+</div>
+<div
+ *ngIf="formatExportConfig.format === 'xlsx' && hasReadFilePrivilege"
+ fxLayout="column"
+>
+ <mat-checkbox
+ [(ngModel)]="formatExportConfig.useTemplate"
+ [disabled]="excelTemplates.length === 0"
+ >
+ Use uploaded file template
+ </mat-checkbox>
+ @if (formatExportConfig.useTemplate && excelTemplates.length > 0) {
+ <mat-form-field class="mt-10" color="accent">
+ <mat-select
+ [(ngModel)]="formatExportConfig.templateId"
+ placeholder="Choose template"
+ >
+ <mat-option
+ *ngFor="let template of excelTemplates"
+ [value]="template.fileId"
+ >
+ {{ template.filename }}
+ </mat-option>
+ </mat-select>
+ </mat-form-field>
+ <mat-form-field color="accent">
+ <mat-label>First row index to append data</mat-label>
+ <input
+ matInput
+ [(ngModel)]="formatExportConfig.startRow"
+ type="number"
+ />
+ </mat-form-field>
+ }
+</div>
+<div
+ *ngIf="
+ formatExportConfig.format === 'xlsx' ||
+ formatExportConfig.format === 'csv'
+ "
+>
+ <h5>Header column name</h5>
+ <mat-radio-group
+ [(ngModel)]="formatExportConfig.headerColumnName"
+ class="sp-radio-group"
+ >
+ <mat-radio-button
+ value="key"
+ class="sp-radio-button"
+ data-cy="download-configuration-column-name-key"
+ >Use field key (runtime name) as header column
+ </mat-radio-button>
+ <mat-radio-button
+ value="label"
+ class="sp-radio-button"
+ data-cy="download-configuration-column-name-label"
+ >Use field label as header column if available
+ </mat-radio-button>
</mat-radio-group>
</div>
diff --git a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts
index d69b5e267c..23db806489 100644
--- a/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts
+++ b/ui/src/app/core-ui/data-download-dialog/components/select-format/select-format.component.ts
@@ -16,8 +16,11 @@
*
*/
-import { Component, Input } from '@angular/core';
+import { Component, inject, Input, OnInit } from '@angular/core';
import { FormatExportConfig } from '../../model/format-export-config.model';
+import { FileMetadata, FilesService } from '@streampipes/platform-services';
+import { UserPrivilege } from '../../../../_enums/user-privilege.enum';
+import { CurrentUserService } from '@streampipes/shared-ui';
@Component({
selector: 'sp-select-format',
@@ -27,8 +30,28 @@ import { FormatExportConfig } from '../../model/format-export-config.model';
'../../data-download-dialog.component.scss',
],
})
-export class SelectFormatComponent {
+export class SelectFormatComponent implements OnInit {
@Input() formatExportConfig: FormatExportConfig;
+ hasReadFilePrivilege = false;
+ excelTemplates: FileMetadata[] = [];
+
+ private fileService = inject(FilesService);
+ private currentUserService = inject(CurrentUserService);
+
constructor() {}
+
+ ngOnInit() {
+ this.hasReadFilePrivilege = this.currentUserService.hasRole(
+ UserPrivilege.PRIVILEGE_READ_FILES,
+ );
+ if (this.hasReadFilePrivilege) {
+ this.fileService
+ .getFileMetadata(['xlsx'])
+ .subscribe(excelTemplates => {
+ this.excelTemplates = excelTemplates;
+ console.log(this.excelTemplates);
+ });
+ }
+ }
}
diff --git a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
index 4a0b6a199b..59e7ea58cd 100644
--- a/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
+++ b/ui/src/app/core-ui/data-download-dialog/data-download-dialog.component.ts
@@ -55,8 +55,9 @@ export class DataDownloadDialogComponent implements OnInit {
measurement: measurementName,
},
formatExportConfig: {
- exportFormat: 'csv',
+ format: 'csv',
delimiter: 'comma',
+ headerColumnName: 'key',
},
};
}
diff --git a/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts b/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts
index 6562bfb909..fab91f13c0 100644
--- a/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts
+++ b/ui/src/app/core-ui/data-download-dialog/model/format-export-config.model.ts
@@ -34,13 +34,25 @@
*
*/
-export type FormatExportConfig = JsonFormatExportConfig | CsvFormatExportConfig;
+export type FormatExportConfig =
+ | JsonFormatExportConfig
+ | CsvFormatExportConfig
+ | ExcelFormatConfig;
export interface JsonFormatExportConfig {
- exportFormat: 'json';
+ format: 'json';
}
export interface CsvFormatExportConfig {
- exportFormat: 'csv';
+ format: 'csv';
delimiter: 'comma' | 'semicolon';
+ headerColumnName: 'key' | 'label';
+}
+
+export interface ExcelFormatConfig {
+ format: 'xlsx';
+ templateId: string;
+ startRow: number;
+ useTemplate: boolean;
+ headerColumnName: 'key' | 'label';
}
diff --git a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
index 4725487714..a014ed97f9 100644
--- a/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
+++ b/ui/src/app/core-ui/data-download-dialog/services/data-export.service.ts
@@ -52,8 +52,7 @@ export class DataExportService {
) {
downloadRequest = this.dataLakeRestService.downloadQueriedData(
exportConfig.dataExportConfig.measurement,
- exportConfig.formatExportConfig.exportFormat,
- exportConfig.formatExportConfig['delimiter'],
+ exportConfig.formatExportConfig,
exportConfig.dataExportConfig.missingValueBehaviour,
this.generateQueryRequest(
exportConfig,
@@ -75,8 +74,7 @@ export class DataExportService {
}
downloadRequest = this.dataLakeRestService.downloadRawData(
exportConfig.dataExportConfig.measurement,
- exportConfig.formatExportConfig.exportFormat,
- exportConfig.formatExportConfig['delimiter'],
+ exportConfig.formatExportConfig,
exportConfig.dataExportConfig.missingValueBehaviour,
startTime,
endTime,
@@ -139,12 +137,9 @@ export class DataExportService {
exportConfig,
new Date(),
);
+ const blob = new Blob([data]);
- const url = window.URL.createObjectURL(
- new Blob([String(data)], {
- type: `data:text/${exportConfig.formatExportConfig.exportFormat};charset=utf-8`,
- }),
- );
+ const url = window.URL.createObjectURL(blob);
a.href = url;
a.download = name;
a.click();
diff --git a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts
index 74430dc99e..deef6c48ed 100644
--- a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts
+++ b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.spec.ts
@@ -40,7 +40,7 @@ describe('FileNameService', () => {
measurement: 'measurement',
},
formatExportConfig: {
- exportFormat: 'csv',
+ format: 'csv',
delimiter: 'comma',
},
};
@@ -55,7 +55,7 @@ describe('FileNameService', () => {
});
it('Name for all data json', () => {
- defaultExportConfig.formatExportConfig.exportFormat = 'json';
+ defaultExportConfig.formatExportConfig.format = 'json';
const result = service.generateName(
defaultExportConfig,
defaultExportDate,
@@ -78,7 +78,7 @@ describe('FileNameService', () => {
});
it('Name for custom visible json', () => {
- defaultExportConfig.formatExportConfig.exportFormat = 'json';
+ defaultExportConfig.formatExportConfig.format = 'json';
defaultExportConfig.dataExportConfig.dataRangeConfiguration = 'visible';
defaultExportConfig.dataExportConfig.dateRange = defaultDateRange;
diff --git a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts
index 878bd50355..66d8fe5b60 100644
--- a/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts
+++ b/ui/src/app/core-ui/data-download-dialog/services/file-name.service.ts
@@ -30,7 +30,7 @@ export class FileNameService {
const dataRangeOption =
exportConfig.dataExportConfig.dataRangeConfiguration;
let dateRange = '';
- const fileExtension = `.${exportConfig.formatExportConfig.exportFormat}`;
+ const fileExtension = `.${exportConfig.formatExportConfig.format}`;
if (
exportConfig.dataExportConfig.dateRange !== undefined &&