You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2026/03/17 08:18:25 UTC
(streampipes) branch dev updated: feat(#4250): add csv dataset import with preview for data sets (#4254)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/streampipes.git
The following commit(s) were added to refs/heads/dev by this push:
new 0d679f838b feat(#4250): add csv dataset import with preview for data sets (#4254)
0d679f838b is described below
commit 0d679f838b333c88c537b8eb3a480e4b6a85e144
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Tue Mar 17 09:18:19 2026 +0100
feat(#4250): add csv dataset import with preview for data sets (#4254)
---
.../model/datalake/importer/CsvImportColumn.java | 104 +++
.../datalake/importer/CsvImportConfiguration.java | 59 ++
.../datalake/importer/CsvImportPreviewRequest.java | 79 +++
.../datalake/importer/CsvImportPreviewResult.java | 100 +++
.../model/datalake/importer/CsvImportRequest.java | 89 +++
.../model/datalake/importer/CsvImportResult.java | 71 ++
.../datalake/importer/CsvImportSchemaIssue.java | 74 ++
.../importer/CsvImportSchemaIssueType.java | 26 +
.../importer/CsvImportSchemaValidationRequest.java | 53 ++
.../importer/CsvImportSchemaValidationResult.java | 53 ++
.../model/datalake/importer/CsvImportTarget.java | 41 ++
.../datalake/importer/CsvImportTargetMode.java | 24 +
.../importer/CsvImportValidationMessage.java | 49 ++
.../impl/datalake/AbstractDataLakeResource.java | 4 +
.../rest/impl/datalake/DataLakeDataWriter.java | 52 +-
.../importer/CsvDataLakeImportService.java | 325 +++++++++
.../impl/datalake/importer/CsvImportParser.java | 619 ++++++++++++++++
.../datalake/importer/CsvImportUploadStorage.java | 130 ++++
.../importer/CsvImportValidationException.java | 37 +
.../importer/CsvImportValidationService.java | 339 +++++++++
.../datalake/importer/DataLakeImportResource.java | 145 ++++
.../rest/impl/datalake/DataLakeDataWriterTest.java | 56 ++
.../importer/CsvDataLakeImportServiceTest.java | 326 +++++++++
.../datalake/importer/CsvImportParserTest.java | 120 ++++
.../importer/CsvImportValidationServiceTest.java | 169 +++++
...hine-data-simulator-import-later-timestamps.csv | 8 +
...-data-simulator-import-mismatched-timestamp.csv | 8 +
...achine-data-simulator-import-missing-values.csv | 8 +
...hine-data-simulator-import-string-timestamp.csv | 8 +
.../datalake/machine-data-simulator-import.csv | 8 +
ui/cypress/support/utils/PrepareTestDataUtils.ts | 64 +-
ui/cypress/support/utils/chart/ChartUtils.ts | 85 +--
.../support/utils/chart/ChartWidgetTableUtils.ts | 15 +-
.../support/utils/dataset/DataLakeSeedUtils.ts | 275 +++++++
ui/cypress/support/utils/dataset/DatasetBtns.ts | 97 +++
ui/cypress/support/utils/dataset/DatasetUtils.ts | 108 +++
ui/cypress/tests/chart/configuration.smoke.spec.ts | 2 -
.../chart/filterNumericalStringProperties.spec.ts | 32 +-
ui/cypress/tests/dataset/csvImport.spec.ts | 113 +++
.../userManagement/testUserRoleDashboard.spec.ts | 12 +-
.../userManagement/testUserRoleDataset.spec.ts | 17 +-
.../src/lib/apis/datalake-rest.service.ts | 51 ++
.../src/lib/model/datalake/csv-import.model.ts | 114 +++
.../platform-services/src/public-api.ts | 1 +
.../data-download-dialog.component.html | 2 +-
.../standard-dialog/standard-dialog.component.scss | 3 +-
.../datalake-configuration.component.html | 9 +
.../datalake-configuration.component.ts | 24 +
.../csv-import-dialog.component.html | 324 +++++++++
.../csv-import-dialog.component.scss | 95 +++
.../csv-import-dialog.component.ts | 786 +++++++++++++++++++++
.../csv-import-preview-table.component.html | 186 +++++
.../csv-import-preview-table.component.scss | 77 ++
.../csv-import-preview-table.component.ts | 102 +++
.../csv-import-upload-state.component.html | 56 ++
.../csv-import-upload-state.component.scss} | 53 +-
.../csv-import-upload-state.component.ts | 47 ++
.../dialog/csv-import-dialog/csv-import.model.ts | 43 ++
58 files changed, 5820 insertions(+), 157 deletions(-)
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportColumn.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportColumn.java
new file mode 100644
index 0000000000..95f91269b4
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportColumn.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public class CsvImportColumn {
+
+ private String csvColumn;
+ private String runtimeName;
+ private String runtimeType;
+ private String propertyScope;
+ private String semanticType;
+ private String label;
+ private String description;
+ private String inferredType;
+ private boolean timestampCandidate;
+
+ public String getCsvColumn() {
+ return csvColumn;
+ }
+
+ public void setCsvColumn(String csvColumn) {
+ this.csvColumn = csvColumn;
+ }
+
+ public String getRuntimeName() {
+ return runtimeName;
+ }
+
+ public void setRuntimeName(String runtimeName) {
+ this.runtimeName = runtimeName;
+ }
+
+ public String getRuntimeType() {
+ return runtimeType;
+ }
+
+ public void setRuntimeType(String runtimeType) {
+ this.runtimeType = runtimeType;
+ }
+
+ public String getPropertyScope() {
+ return propertyScope;
+ }
+
+ public void setPropertyScope(String propertyScope) {
+ this.propertyScope = propertyScope;
+ }
+
+ public String getSemanticType() {
+ return semanticType;
+ }
+
+ public void setSemanticType(String semanticType) {
+ this.semanticType = semanticType;
+ }
+
+ public String getLabel() {
+ return label;
+ }
+
+ public void setLabel(String label) {
+ this.label = label;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getInferredType() {
+ return inferredType;
+ }
+
+ public void setInferredType(String inferredType) {
+ this.inferredType = inferredType;
+ }
+
+ public boolean isTimestampCandidate() {
+ return timestampCandidate;
+ }
+
+ public void setTimestampCandidate(boolean timestampCandidate) {
+ this.timestampCandidate = timestampCandidate;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportConfiguration.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportConfiguration.java
new file mode 100644
index 0000000000..8fac2972a9
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportConfiguration.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public class CsvImportConfiguration {
+
+ private String delimiter;
+ private String decimalSeparator;
+ private boolean hasHeader;
+ private String timestampFormat;
+
+ public String getDelimiter() {
+ return delimiter;
+ }
+
+ public void setDelimiter(String delimiter) {
+ this.delimiter = delimiter;
+ }
+
+ public String getDecimalSeparator() {
+ return decimalSeparator;
+ }
+
+ public void setDecimalSeparator(String decimalSeparator) {
+ this.decimalSeparator = decimalSeparator;
+ }
+
+ public boolean isHasHeader() {
+ return hasHeader;
+ }
+
+ public void setHasHeader(boolean hasHeader) {
+ this.hasHeader = hasHeader;
+ }
+
+ public String getTimestampFormat() {
+ return timestampFormat;
+ }
+
+ public void setTimestampFormat(String timestampFormat) {
+ this.timestampFormat = timestampFormat;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportPreviewRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportPreviewRequest.java
new file mode 100644
index 0000000000..7af3a0f44b
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportPreviewRequest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+import java.util.List;
+
+public class CsvImportPreviewRequest {
+
+ private String uploadId;
+ private String fileName;
+ private CsvImportConfiguration csvConfig;
+ private List<String> headers;
+ private List<List<String>> rows;
+ private CsvImportTarget target;
+
+ public String getUploadId() {
+ return uploadId;
+ }
+
+ public void setUploadId(String uploadId) {
+ this.uploadId = uploadId;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+
+ public CsvImportConfiguration getCsvConfig() {
+ return csvConfig;
+ }
+
+ public void setCsvConfig(CsvImportConfiguration csvConfig) {
+ this.csvConfig = csvConfig;
+ }
+
+ public List<String> getHeaders() {
+ return headers;
+ }
+
+ public void setHeaders(List<String> headers) {
+ this.headers = headers;
+ }
+
+ public List<List<String>> getRows() {
+ return rows;
+ }
+
+ public void setRows(List<List<String>> rows) {
+ this.rows = rows;
+ }
+
+ public CsvImportTarget getTarget() {
+ return target;
+ }
+
+ public void setTarget(CsvImportTarget target) {
+ this.target = target;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportPreviewResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportPreviewResult.java
new file mode 100644
index 0000000000..4bb9b84cf8
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportPreviewResult.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+import org.apache.streampipes.model.schema.EventSchema;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvImportPreviewResult {
+
+ private String uploadId;
+ private List<String> headers = new ArrayList<>();
+ private List<List<String>> previewRows = new ArrayList<>();
+ private List<CsvImportColumn> columns = new ArrayList<>();
+ private EventSchema guessedEventSchema;
+ private List<String> timestampCandidates = new ArrayList<>();
+ private boolean valid;
+ private List<CsvImportValidationMessage> validationMessages = new ArrayList<>();
+
+ public String getUploadId() {
+ return uploadId;
+ }
+
+ public void setUploadId(String uploadId) {
+ this.uploadId = uploadId;
+ }
+
+ public List<String> getHeaders() {
+ return headers;
+ }
+
+ public void setHeaders(List<String> headers) {
+ this.headers = headers;
+ }
+
+ public List<List<String>> getPreviewRows() {
+ return previewRows;
+ }
+
+ public void setPreviewRows(List<List<String>> previewRows) {
+ this.previewRows = previewRows;
+ }
+
+ public List<CsvImportColumn> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<CsvImportColumn> columns) {
+ this.columns = columns;
+ }
+
+ public EventSchema getGuessedEventSchema() {
+ return guessedEventSchema;
+ }
+
+ public void setGuessedEventSchema(EventSchema guessedEventSchema) {
+ this.guessedEventSchema = guessedEventSchema;
+ }
+
+ public List<String> getTimestampCandidates() {
+ return timestampCandidates;
+ }
+
+ public void setTimestampCandidates(List<String> timestampCandidates) {
+ this.timestampCandidates = timestampCandidates;
+ }
+
+ public boolean isValid() {
+ return valid;
+ }
+
+ public void setValid(boolean valid) {
+ this.valid = valid;
+ }
+
+ public List<CsvImportValidationMessage> getValidationMessages() {
+ return validationMessages;
+ }
+
+ public void setValidationMessages(List<CsvImportValidationMessage> validationMessages) {
+ this.validationMessages = validationMessages;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportRequest.java
new file mode 100644
index 0000000000..c882562ae7
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportRequest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvImportRequest {
+
+ private String uploadId;
+ private CsvImportConfiguration csvConfig;
+ private List<String> headers = new ArrayList<>();
+ private List<List<String>> rows = new ArrayList<>();
+ private CsvImportTarget target;
+ private String timestampColumn;
+ private List<CsvImportColumn> columns = new ArrayList<>();
+
+ public String getUploadId() {
+ return uploadId;
+ }
+
+ public void setUploadId(String uploadId) {
+ this.uploadId = uploadId;
+ }
+
+ public CsvImportConfiguration getCsvConfig() {
+ return csvConfig;
+ }
+
+ public void setCsvConfig(CsvImportConfiguration csvConfig) {
+ this.csvConfig = csvConfig;
+ }
+
+ public List<String> getHeaders() {
+ return headers;
+ }
+
+ public void setHeaders(List<String> headers) {
+ this.headers = headers;
+ }
+
+ public List<List<String>> getRows() {
+ return rows;
+ }
+
+ public void setRows(List<List<String>> rows) {
+ this.rows = rows;
+ }
+
+ public CsvImportTarget getTarget() {
+ return target;
+ }
+
+ public void setTarget(CsvImportTarget target) {
+ this.target = target;
+ }
+
+ public String getTimestampColumn() {
+ return timestampColumn;
+ }
+
+ public void setTimestampColumn(String timestampColumn) {
+ this.timestampColumn = timestampColumn;
+ }
+
+ public List<CsvImportColumn> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<CsvImportColumn> columns) {
+ this.columns = columns;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportResult.java
new file mode 100644
index 0000000000..c393160010
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportResult.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvImportResult {
+
+ private String measurementName;
+ private String measurementId;
+ private boolean createdNewMeasurement;
+ private int importedRowCount;
+ private List<CsvImportValidationMessage> validationMessages = new ArrayList<>();
+
+ public String getMeasurementName() {
+ return measurementName;
+ }
+
+ public void setMeasurementName(String measurementName) {
+ this.measurementName = measurementName;
+ }
+
+ public String getMeasurementId() {
+ return measurementId;
+ }
+
+ public void setMeasurementId(String measurementId) {
+ this.measurementId = measurementId;
+ }
+
+ public boolean isCreatedNewMeasurement() {
+ return createdNewMeasurement;
+ }
+
+ public void setCreatedNewMeasurement(boolean createdNewMeasurement) {
+ this.createdNewMeasurement = createdNewMeasurement;
+ }
+
+ public int getImportedRowCount() {
+ return importedRowCount;
+ }
+
+ public void setImportedRowCount(int importedRowCount) {
+ this.importedRowCount = importedRowCount;
+ }
+
+ public List<CsvImportValidationMessage> getValidationMessages() {
+ return validationMessages;
+ }
+
+ public void setValidationMessages(List<CsvImportValidationMessage> validationMessages) {
+ this.validationMessages = validationMessages;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaIssue.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaIssue.java
new file mode 100644
index 0000000000..3b67198757
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaIssue.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public class CsvImportSchemaIssue {
+
+ private CsvImportSchemaIssueType type;
+ private String columnName;
+ private String expected;
+ private String actual;
+
+ public CsvImportSchemaIssue() {
+ }
+
+ public CsvImportSchemaIssue(
+ CsvImportSchemaIssueType type,
+ String columnName,
+ String expected,
+ String actual
+ ) {
+ this.type = type;
+ this.columnName = columnName;
+ this.expected = expected;
+ this.actual = actual;
+ }
+
+ public CsvImportSchemaIssueType getType() {
+ return type;
+ }
+
+ public void setType(CsvImportSchemaIssueType type) {
+ this.type = type;
+ }
+
+ public String getColumnName() {
+ return columnName;
+ }
+
+ public void setColumnName(String columnName) {
+ this.columnName = columnName;
+ }
+
+ public String getExpected() {
+ return expected;
+ }
+
+ public void setExpected(String expected) {
+ this.expected = expected;
+ }
+
+ public String getActual() {
+ return actual;
+ }
+
+ public void setActual(String actual) {
+ this.actual = actual;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaIssueType.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaIssueType.java
new file mode 100644
index 0000000000..13ac996bb8
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaIssueType.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public enum CsvImportSchemaIssueType {
+ TIMESTAMP_COLUMN_MISMATCH,
+ COLUMN_NAME_MISMATCH,
+ COLUMN_TYPE_MISMATCH,
+ COLUMN_SCOPE_MISMATCH
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaValidationRequest.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaValidationRequest.java
new file mode 100644
index 0000000000..3c25e2952a
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaValidationRequest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvImportSchemaValidationRequest {
+
+ private CsvImportTarget target;
+ private String timestampColumn;
+ private List<CsvImportColumn> columns = new ArrayList<>();
+
+ public CsvImportTarget getTarget() {
+ return target;
+ }
+
+ public void setTarget(CsvImportTarget target) {
+ this.target = target;
+ }
+
+ public String getTimestampColumn() {
+ return timestampColumn;
+ }
+
+ public void setTimestampColumn(String timestampColumn) {
+ this.timestampColumn = timestampColumn;
+ }
+
+ public List<CsvImportColumn> getColumns() {
+ return columns;
+ }
+
+ public void setColumns(List<CsvImportColumn> columns) {
+ this.columns = columns;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaValidationResult.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaValidationResult.java
new file mode 100644
index 0000000000..5b7e23ef93
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportSchemaValidationResult.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class CsvImportSchemaValidationResult {
+
+ private boolean valid;
+ private List<CsvImportValidationMessage> validationMessages = new ArrayList<>();
+ private List<CsvImportSchemaIssue> issues = new ArrayList<>();
+
+ public boolean isValid() {
+ return valid;
+ }
+
+ public void setValid(boolean valid) {
+ this.valid = valid;
+ }
+
+ public List<CsvImportValidationMessage> getValidationMessages() {
+ return validationMessages;
+ }
+
+ public void setValidationMessages(List<CsvImportValidationMessage> validationMessages) {
+ this.validationMessages = validationMessages;
+ }
+
+ public List<CsvImportSchemaIssue> getIssues() {
+ return issues;
+ }
+
+ public void setIssues(List<CsvImportSchemaIssue> issues) {
+ this.issues = issues;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportTarget.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportTarget.java
new file mode 100644
index 0000000000..61d018884e
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportTarget.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public class CsvImportTarget {
+
+ private CsvImportTargetMode mode;
+ private String measurementName;
+
+ public CsvImportTargetMode getMode() {
+ return mode;
+ }
+
+ public void setMode(CsvImportTargetMode mode) {
+ this.mode = mode;
+ }
+
+ public String getMeasurementName() {
+ return measurementName;
+ }
+
+ public void setMeasurementName(String measurementName) {
+ this.measurementName = measurementName;
+ }
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportTargetMode.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportTargetMode.java
new file mode 100644
index 0000000000..86a16562c3
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportTargetMode.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public enum CsvImportTargetMode {
+ NEW,
+ EXISTING
+}
diff --git a/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportValidationMessage.java b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportValidationMessage.java
new file mode 100644
index 0000000000..cd5abd99f3
--- /dev/null
+++ b/streampipes-model/src/main/java/org/apache/streampipes/model/datalake/importer/CsvImportValidationMessage.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.model.datalake.importer;
+
+public class CsvImportValidationMessage {
+
+ private String field;
+ private String message;
+
+ public CsvImportValidationMessage() {
+ }
+
+ public CsvImportValidationMessage(String field, String message) {
+ this.field = field;
+ this.message = message;
+ }
+
+ public String getField() {
+ return field;
+ }
+
+ public void setField(String field) {
+ this.field = field;
+ }
+
+ public String getMessage() {
+ return message;
+ }
+
+ public void setMessage(String message) {
+ this.message = message;
+ }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/AbstractDataLakeResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/AbstractDataLakeResource.java
index aac1a722e7..36df49e52e 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/AbstractDataLakeResource.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/AbstractDataLakeResource.java
@@ -56,6 +56,10 @@ public class AbstractDataLakeResource extends AbstractAuthGuardedRestResource {
return isAdminOrHasAnyAuthority(DefaultPrivilege.Constants.PRIVILEGE_WRITE_DATASET_VALUE);
}
+ protected IDataExplorerSchemaManagement getDataLakeMeasureManagement() {
+ return dataLakeMeasureManagement;
+ }
+
/**
* required by Spring expression, do not delete if IDE shows this as unused
* @param measurementName the name of the data lake measure
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriter.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriter.java
index 5bbdaa8bd6..c841e88a34 100644
--- a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriter.java
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriter.java
@@ -31,17 +31,24 @@ import org.apache.streampipes.storage.management.StorageDispatcher;
import java.util.ArrayList;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.stream.Collectors;
-import java.util.stream.IntStream;
public class DataLakeDataWriter {
private final boolean ignoreSchemaMismatch;
+ private final boolean allowMissingFields;
public DataLakeDataWriter(boolean ignoreSchemaMismatch) {
+ this(ignoreSchemaMismatch, false);
+ }
+
+ public DataLakeDataWriter(boolean ignoreSchemaMismatch, boolean allowMissingFields) {
this.ignoreSchemaMismatch = ignoreSchemaMismatch;
+ this.allowMissingFields = allowMissingFields;
}
public void writeData(String measureName, SpQueryResult queryResult) {
@@ -49,10 +56,22 @@ public class DataLakeDataWriter {
if (measure == null) {
throw new SpRuntimeException("Measure \"" + measureName + "\" not found");
}
+ writeData(measure, queryResult);
+ }
+
+ public void writeData(DataLakeMeasure measure, SpQueryResult queryResult) {
var dataSeries = getDataSeries(queryResult);
getTimeSeriesStoreAndPersistQueryResult(dataSeries, measure);
}
+ public void writeData(DataLakeMeasure measure, List<String> headers, List<List<Object>> rows) {
+ var dataSeries = new DataSeries();
+ dataSeries.setHeaders(headers);
+ dataSeries.setRows(rows);
+ dataSeries.setTotal(rows.size());
+ getTimeSeriesStoreAndPersistQueryResult(dataSeries, measure);
+ }
+
private void getTimeSeriesStoreAndPersistQueryResult(DataSeries dataSeries,
DataLakeMeasure measure){
var timeSeriesStore = getTimeSeriesStore(measure);
@@ -95,13 +114,24 @@ public class DataLakeDataWriter {
.collect(Collectors.toSet());
var runtimeNameSet = new HashSet<>(runtimeNames);
- if (!runtimeNameSet.equals(strippedEventKeys)){
+ if (!matchesRuntimeNames(runtimeNameSet, strippedEventKeys, allowMissingFields)) {
throw new SpRuntimeException("The fields of the event do not match. Use \"ignoreSchemaMismatch\" to "
+ "ignore this error. Fields of the event: " + strippedEventKeys);
}
}
}
+ static boolean matchesRuntimeNames(
+ Set<String> expectedRuntimeNames,
+ Set<String> actualRuntimeNames,
+ boolean allowMissingFields
+ ) {
+ if (allowMissingFields) {
+ return expectedRuntimeNames.containsAll(actualRuntimeNames);
+ }
+ return expectedRuntimeNames.equals(actualRuntimeNames);
+ }
+
private List<String> getRuntimeNames(DataLakeMeasure measure) {
var runtimeNames = new ArrayList<String>();
runtimeNames.add(measure.getTimestampFieldName());
@@ -120,10 +150,18 @@ public class DataLakeDataWriter {
}
private Event rowToEvent(List<Object> row, List<String> headers){
- Map<String, Object> eventMap = IntStream.range(0, headers.size())
- .boxed()
- .collect(Collectors.toMap(headers::get, row::get));
- return EventFactory.fromMap(eventMap);
+ return EventFactory.fromMap(toEventMap(row, headers));
+ }
+
+ static Map<String, Object> toEventMap(List<Object> row, List<String> headers) {
+ var eventMap = new LinkedHashMap<String, Object>();
+ for (int i = 0; i < headers.size(); i++) {
+ var value = row.get(i);
+ if (value != null) {
+ eventMap.put(headers.get(i), value);
+ }
+ }
+ return eventMap;
}
private void renameTimestampField(Event event, String timestampField){
@@ -133,5 +171,3 @@ public class DataLakeDataWriter {
}
}
-
-
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvDataLakeImportService.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvDataLakeImportService.java
new file mode 100644
index 0000000000..5ba84ddd2a
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvDataLakeImportService.java
@@ -0,0 +1,325 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.importer.CsvImportColumn;
+import org.apache.streampipes.model.datalake.importer.CsvImportPreviewRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportPreviewResult;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportResult;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaValidationRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaValidationResult;
+import org.apache.streampipes.model.datalake.importer.CsvImportTargetMode;
+import org.apache.streampipes.model.datalake.importer.CsvImportValidationMessage;
+import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.rest.impl.datalake.DataLakeDataWriter;
+
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class CsvDataLakeImportService {
+
+ private static final int MAX_PREVIEW_ROWS = 50;
+ private static final int MAX_ANALYSIS_ROWS = 200;
+ private static final String STREAM_PREFIX = "s0::";
+
+ private final DataLakeDataWriter dataWriter;
+ private final CsvImportUploadStorage uploadStorage;
+ private final CsvImportParser parser;
+ private final CsvImportValidationService validationService;
+ private final IDataExplorerSchemaManagement schemaManagement;
+
+ public CsvDataLakeImportService(IDataExplorerSchemaManagement schemaManagement) {
+ this(
+ schemaManagement,
+ new DataLakeDataWriter(false, true),
+ new CsvImportUploadStorage(),
+ new CsvImportParser()
+ );
+ }
+
+ CsvDataLakeImportService(
+ IDataExplorerSchemaManagement schemaManagement,
+ DataLakeDataWriter dataWriter
+ ) {
+ this(schemaManagement, dataWriter, new CsvImportUploadStorage(), new CsvImportParser());
+ }
+
+ CsvDataLakeImportService(
+ IDataExplorerSchemaManagement schemaManagement,
+ DataLakeDataWriter dataWriter,
+ CsvImportUploadStorage uploadStorage
+ ) {
+ this(schemaManagement, dataWriter, uploadStorage, new CsvImportParser());
+ }
+
+ CsvDataLakeImportService(
+ IDataExplorerSchemaManagement schemaManagement,
+ DataLakeDataWriter dataWriter,
+ CsvImportUploadStorage uploadStorage,
+ CsvImportParser parser
+ ) {
+ this.schemaManagement = schemaManagement;
+ this.dataWriter = dataWriter;
+ this.uploadStorage = uploadStorage;
+ this.parser = parser;
+ this.validationService = new CsvImportValidationService(schemaManagement);
+ }
+
+ public CsvImportPreviewResult preview(CsvImportPreviewRequest request) {
+ var validationMessages = validationService.validatePreviewRequest(request);
+ var headers = parser.sanitizeHeaders(request.getHeaders());
+ var rows = Optional.ofNullable(request.getRows()).orElseGet(Collections::emptyList);
+ return buildPreviewResult(request, headers, rows, validationMessages, null);
+ }
+
+ public CsvImportPreviewResult preview(CsvImportPreviewRequest request, String principalSid) {
+ if (!hasUploadId(request)) {
+ return preview(request);
+ }
+
+ var validationMessages = validationService.validatePreviewConfiguration(request);
+ if (!validationMessages.isEmpty()) {
+ return buildInvalidPreviewResult(validationMessages, request.getUploadId());
+ }
+
+ try {
+ var upload = resolveUpload(request.getUploadId(), principalSid);
+ var csvSample = parser.readCsvSample(upload.path(), request.getCsvConfig(), MAX_ANALYSIS_ROWS);
+ return buildPreviewResult(request, csvSample.headers(), csvSample.rows(), validationMessages, upload.uploadId());
+ } catch (CsvImportValidationException e) {
+ return buildInvalidPreviewResult(e.getValidationMessages(), request.getUploadId());
+ } catch (IOException | UncheckedIOException e) {
+ return buildInvalidPreviewResult(
+ List.of(message("file", "The CSV file could not be parsed with the current settings.")),
+ request.getUploadId()
+ );
+ }
+ }
+
+ public CsvImportPreviewResult preview(MultipartFile file, CsvImportPreviewRequest request, String principalSid)
+ throws IOException {
+ var validationMessages = validationService.validatePreviewConfiguration(request);
+ if (!validationMessages.isEmpty()) {
+ return buildInvalidPreviewResult(validationMessages, null);
+ }
+
+ var upload = uploadStorage.store(file, principalSid);
+ try {
+ var csvSample = parser.readCsvSample(upload.path(), request.getCsvConfig(), MAX_ANALYSIS_ROWS);
+ return buildPreviewResult(request, csvSample.headers(), csvSample.rows(), validationMessages, upload.uploadId());
+ } catch (IOException | UncheckedIOException e) {
+ uploadStorage.remove(upload.uploadId());
+ throw e;
+ }
+ }
+
+ public CsvImportSchemaValidationResult validateSchema(CsvImportSchemaValidationRequest request) {
+ var validationMessages = validationService.validateSchemaRequest(request);
+ var issues = new ArrayList<org.apache.streampipes.model.datalake.importer.CsvImportSchemaIssue>();
+ if (validationMessages.isEmpty()) {
+ var eventSchema = parser.buildConfiguredEventSchema(
+ parser.sanitizeImportColumns(request.getColumns()),
+ request.getTimestampColumn()
+ );
+ issues.addAll(validationService.validateSchemaTarget(
+ request.getTarget(),
+ eventSchema,
+ request.getTimestampColumn()
+ ));
+ }
+
+ var result = new CsvImportSchemaValidationResult();
+ result.setValidationMessages(validationMessages);
+ result.setIssues(issues);
+ result.setValid(validationMessages.isEmpty() && issues.isEmpty());
+ return result;
+ }
+
+ public CsvImportResult importData(CsvImportRequest request, String principalSid) {
+ if (hasUploadId(request)) {
+ return importUploadedData(request, principalSid);
+ }
+
+ var validationMessages = validationService.validateInlineImportRequest(request);
+ if (!validationMessages.isEmpty()) {
+ throw new CsvImportValidationException(validationMessages);
+ }
+
+ var eventSchema = parser.buildEventSchema(
+ parser.sanitizeImportColumns(request.getColumns()),
+ request.getRows(),
+ request.getCsvConfig(),
+ request.getTimestampColumn()
+ );
+
+ validationMessages.addAll(
+ validationService.validateImportTarget(request.getTarget(), eventSchema, request.getTimestampColumn())
+ );
+ if (!validationMessages.isEmpty()) {
+ throw new CsvImportValidationException(validationMessages);
+ }
+
+ var measure = resolveTargetMeasurement(request, principalSid, eventSchema);
+ var rows = parser.toImportRows(request);
+ dataWriter.writeData(
+ measure.measure(),
+ request.getColumns().stream().map(CsvImportColumn::getRuntimeName).collect(Collectors.toList()),
+ rows
+ );
+
+ return buildImportResult(measure, rows.size());
+ }
+
+ private CsvImportResult importUploadedData(CsvImportRequest request, String principalSid) {
+ var validationMessages = validationService.validateStoredImportRequest(request);
+ if (!validationMessages.isEmpty()) {
+ throw new CsvImportValidationException(validationMessages);
+ }
+
+ var upload = resolveUpload(request.getUploadId(), principalSid);
+ var sanitizedColumns = parser.sanitizeImportColumns(request.getColumns());
+ var eventSchema = parser.buildConfiguredEventSchema(sanitizedColumns, request.getTimestampColumn());
+
+ validationMessages.addAll(
+ validationService.validateImportTarget(request.getTarget(), eventSchema, request.getTimestampColumn())
+ );
+ if (!validationMessages.isEmpty()) {
+ throw new CsvImportValidationException(validationMessages);
+ }
+
+ var measure = resolveTargetMeasurement(request, principalSid, eventSchema);
+
+ try {
+ var importedRowCount = parser.importCsvFile(upload.path(), request, measure.measure(), dataWriter);
+ uploadStorage.remove(upload.uploadId());
+ return buildImportResult(measure, importedRowCount);
+ } catch (IOException | UncheckedIOException e) {
+ throw new CsvImportValidationException(List.of(
+ message("file", "The CSV file could not be parsed with the current settings.")
+ ));
+ }
+ }
+
+ private CsvImportPreviewResult buildPreviewResult(
+ CsvImportPreviewRequest request,
+ List<String> headers,
+ List<List<String>> rows,
+ List<CsvImportValidationMessage> validationMessages,
+ String uploadId
+ ) {
+ var messages = new ArrayList<>(validationMessages);
+ var columns = parser.inferColumns(headers, rows, request.getCsvConfig());
+ var eventSchema = parser.buildEventSchema(columns, rows, request.getCsvConfig(), null);
+ messages.addAll(validationService.validatePreviewTarget(request.getTarget()));
+
+ var result = new CsvImportPreviewResult();
+ result.setUploadId(uploadId);
+ result.setHeaders(headers);
+ result.setPreviewRows(rows.stream().limit(MAX_PREVIEW_ROWS).collect(Collectors.toList()));
+ result.setColumns(columns);
+ result.setGuessedEventSchema(eventSchema);
+ result.setTimestampCandidates(columns.stream()
+ .filter(CsvImportColumn::isTimestampCandidate)
+ .map(CsvImportColumn::getRuntimeName)
+ .collect(Collectors.toList()));
+ result.setValidationMessages(messages);
+ result.setValid(messages.isEmpty());
+ return result;
+ }
+
+ private CsvImportPreviewResult buildInvalidPreviewResult(
+ List<CsvImportValidationMessage> validationMessages,
+ String uploadId
+ ) {
+ var result = new CsvImportPreviewResult();
+ result.setUploadId(uploadId);
+ result.setValidationMessages(validationMessages);
+ result.setValid(false);
+ return result;
+ }
+
+ private CsvImportUploadStorage.StoredUpload resolveUpload(String uploadId, String principalSid) {
+ var upload = uploadStorage.get(uploadId).orElseThrow(() -> new CsvImportValidationException(List.of(
+ message("uploadId", "The uploaded CSV file was not found. Please upload the file again.")
+ )));
+
+ if (!Objects.equals(upload.ownerSid(), principalSid)) {
+ throw new CsvImportValidationException(List.of(
+ message("uploadId", "The uploaded CSV file is no longer available for this user.")
+ ));
+ }
+
+ return upload;
+ }
+
+ private boolean hasUploadId(CsvImportPreviewRequest request) {
+ return request != null && request.getUploadId() != null && !request.getUploadId().isBlank();
+ }
+
+ private boolean hasUploadId(CsvImportRequest request) {
+ return request != null && request.getUploadId() != null && !request.getUploadId().isBlank();
+ }
+
+ private CsvImportValidationMessage message(String field, String message) {
+ return new CsvImportValidationMessage(field, message);
+ }
+
+ private StoredMeasure resolveTargetMeasurement(
+ CsvImportRequest request,
+ String principalSid,
+ EventSchema eventSchema
+ ) {
+ if (request.getTarget().getMode() == CsvImportTargetMode.NEW) {
+ var measure = new DataLakeMeasure();
+ measure.setMeasureName(request.getTarget().getMeasurementName().trim());
+ measure.setTimestampField(STREAM_PREFIX + request.getTimestampColumn());
+ measure.setEventSchema(eventSchema);
+ return new StoredMeasure(schemaManagement.createOrUpdateMeasurement(measure, principalSid), true);
+ }
+
+ return new StoredMeasure(
+ validationService.requireExistingMeasurement(request.getTarget().getMeasurementName()),
+ false
+ );
+ }
+
+ private CsvImportResult buildImportResult(StoredMeasure measure, int importedRowCount) {
+ var result = new CsvImportResult();
+ result.setMeasurementId(measure.measure().getElementId());
+ result.setMeasurementName(measure.measure().getMeasureName());
+ result.setCreatedNewMeasurement(measure.createdNewMeasurement());
+ result.setImportedRowCount(importedRowCount);
+ result.setValidationMessages(List.of());
+ return result;
+ }
+
+ private record StoredMeasure(DataLakeMeasure measure, boolean createdNewMeasurement) {
+ }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportParser.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportParser.java
new file mode 100644
index 0000000000..ee43045a4a
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportParser.java
@@ -0,0 +1,619 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.connect.management.util.EventSchemaUtils;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.importer.CsvImportColumn;
+import org.apache.streampipes.model.datalake.importer.CsvImportConfiguration;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportValidationMessage;
+import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
+import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.model.schema.PropertyScope;
+import org.apache.streampipes.rest.impl.datalake.DataLakeDataWriter;
+import org.apache.streampipes.vocabulary.SO;
+import org.apache.streampipes.vocabulary.XSD;
+
+import java.io.IOException;
+import java.io.PushbackReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+class CsvImportParser {
+
+ private static final int IMPORT_BATCH_SIZE = 5000;
+
+ List<String> sanitizeHeaders(List<String> headers) {
+ return headers.stream()
+ .map(header -> header == null ? "" : header.trim())
+ .collect(Collectors.toList());
+ }
+
+ List<CsvImportColumn> sanitizeImportColumns(List<CsvImportColumn> columns) {
+ return columns.stream().map(column -> {
+ column.setRuntimeName(column.getRuntimeName().trim());
+ return column;
+ }).collect(Collectors.toList());
+ }
+
+ EventSchema buildEventSchema(
+ List<CsvImportColumn> columns,
+ List<List<String>> rows,
+ CsvImportConfiguration config,
+ String selectedTimestampColumn
+ ) {
+ var sampleEvent = new LinkedHashMap<String, Object>();
+ for (int i = 0; i < columns.size(); i++) {
+ sampleEvent.put(columns.get(i).getRuntimeName(), sampleValue(rows, i, columns.get(i), config, selectedTimestampColumn));
+ }
+
+ var eventSchema = EventSchemaUtils.guessEventSchema(sampleEvent);
+ var properties = eventSchema.getEventProperties()
+ .stream()
+ .collect(Collectors.toMap(EventProperty::getRuntimeName, property -> property, (a, b) -> a, LinkedHashMap::new));
+
+ for (var column : columns) {
+ var property = (EventPropertyPrimitive) properties.get(column.getRuntimeName());
+ if (property == null) {
+ continue;
+ }
+ property.setRuntimeType(toRuntimeTypeUri(finalRuntimeType(column, selectedTimestampColumn)));
+ property.setPropertyScope(finalPropertyScope(column, selectedTimestampColumn));
+ property.setSemanticType(finalSemanticType(column, selectedTimestampColumn));
+ property.setLabel(Optional.ofNullable(column.getLabel()).orElse(""));
+ property.setDescription(Optional.ofNullable(column.getDescription()).orElse(""));
+ }
+
+ eventSchema.setEventProperties(new ArrayList<>(properties.values()));
+ return eventSchema;
+ }
+
+ EventSchema buildConfiguredEventSchema(
+ List<CsvImportColumn> columns,
+ String selectedTimestampColumn
+ ) {
+ var properties = new ArrayList<EventProperty>();
+ for (var column : columns) {
+ var property = new EventPropertyPrimitive();
+ property.setRuntimeName(column.getRuntimeName());
+ property.setRuntimeType(toRuntimeTypeUri(finalRuntimeType(column, selectedTimestampColumn)));
+ property.setPropertyScope(finalPropertyScope(column, selectedTimestampColumn));
+ property.setSemanticType(finalSemanticType(column, selectedTimestampColumn));
+ property.setLabel(Optional.ofNullable(column.getLabel()).orElse(""));
+ property.setDescription(Optional.ofNullable(column.getDescription()).orElse(""));
+ properties.add(property);
+ }
+ return new EventSchema(properties);
+ }
+
+ List<CsvImportColumn> inferColumns(
+ List<String> headers,
+ List<List<String>> rows,
+ CsvImportConfiguration config
+ ) {
+ var columns = new ArrayList<CsvImportColumn>();
+ for (int i = 0; i < headers.size(); i++) {
+ var runtimeName = headers.get(i);
+ var inferredType = inferType(rows, i, config);
+ var timestampCandidate = isTimestampCandidate(rows, i, config);
+ var column = new CsvImportColumn();
+ column.setCsvColumn(headers.get(i));
+ column.setRuntimeName(runtimeName);
+ column.setRuntimeType(inferredType);
+ column.setInferredType(inferredType);
+ column.setPropertyScope(timestampCandidate ? PropertyScope.HEADER_PROPERTY.name() : suggestedScope(inferredType));
+ column.setSemanticType(timestampCandidate ? SO.DATE_TIME : null);
+ column.setTimestampCandidate(timestampCandidate);
+ columns.add(column);
+ }
+ return columns;
+ }
+
+ List<List<Object>> toImportRows(CsvImportRequest request) {
+ var rows = new ArrayList<List<Object>>();
+ for (int rowIndex = 0; rowIndex < request.getRows().size(); rowIndex++) {
+ rows.add(convertRow(request.getRows().get(rowIndex), request, rowIndex + 1));
+ }
+ return rows;
+ }
+
+ int importCsvFile(
+ Path path,
+ CsvImportRequest request,
+ DataLakeMeasure measure,
+ DataLakeDataWriter dataWriter
+ ) throws IOException {
+ var runtimeHeaders = request.getColumns().stream()
+ .map(CsvImportColumn::getRuntimeName)
+ .collect(Collectors.toList());
+ var batch = new ArrayList<List<Object>>();
+ var importedRows = new int[]{0};
+
+ parseCsvFile(path, request.getCsvConfig(), new CsvRowConsumer() {
+ private List<String> parsedHeaders;
+
+ @Override
+ public void onHeaders(List<String> headers) {
+ parsedHeaders = headers;
+ validateUploadedHeaders(headers, request.getColumns());
+ }
+
+ @Override
+ public void onRow(int rowNumber, List<String> row) {
+ if (row.size() != parsedHeaders.size()) {
+ throw new CsvImportValidationException(List.of(
+ message("rows", "Row " + rowNumber + " does not match the header size.")
+ ));
+ }
+ batch.add(convertRow(row, request, rowNumber));
+ if (batch.size() >= IMPORT_BATCH_SIZE) {
+ dataWriter.writeData(measure, runtimeHeaders, new ArrayList<>(batch));
+ importedRows[0] += IMPORT_BATCH_SIZE;
+ batch.clear();
+ }
+ }
+ });
+
+ if (!batch.isEmpty()) {
+ var batchSize = batch.size();
+ dataWriter.writeData(measure, runtimeHeaders, new ArrayList<>(batch));
+ importedRows[0] += batchSize;
+ }
+
+ return importedRows[0];
+ }
+
+ CsvFileSample readCsvSample(Path path, CsvImportConfiguration config, int maxRows) throws IOException {
+ var headers = new ArrayList<String>();
+ var rows = new ArrayList<List<String>>();
+
+ parseCsvFile(path, config, new CsvRowConsumer() {
+ @Override
+ public void onHeaders(List<String> parsedHeaders) {
+ headers.addAll(parsedHeaders);
+ }
+
+ @Override
+ public void onRow(int rowNumber, List<String> row) {
+ if (row.size() != headers.size()) {
+ throw new CsvImportValidationException(List.of(
+ message("rows", "Row " + rowNumber + " does not match the header size.")
+ ));
+ }
+ if (rows.size() < maxRows) {
+ rows.add(row);
+ }
+ }
+ });
+
+ if (headers.isEmpty()) {
+ throw new CsvImportValidationException(List.of(message("headers", "At least one header must be provided.")));
+ }
+ if (rows.isEmpty()) {
+ throw new CsvImportValidationException(List.of(message("rows", "At least one row must be provided.")));
+ }
+
+ return new CsvFileSample(headers, rows);
+ }
+
+ private List<Object> convertRow(List<String> row, CsvImportRequest request, int rowNumber) {
+ var converted = new ArrayList<Object>();
+ for (int i = 0; i < row.size(); i++) {
+ converted.add(convertValue(
+ row.get(i),
+ request.getColumns().get(i),
+ request.getCsvConfig(),
+ request.getTimestampColumn(),
+ rowNumber
+ ));
+ }
+ return converted;
+ }
+
+ private void validateUploadedHeaders(List<String> headers, List<CsvImportColumn> columns) {
+ if (headers.size() != columns.size()) {
+ throw new CsvImportValidationException(List.of(
+ message("headers", "The uploaded CSV file no longer matches the previewed column count.")
+ ));
+ }
+
+ for (int i = 0; i < headers.size(); i++) {
+ if (!Objects.equals(headers.get(i), columns.get(i).getCsvColumn())) {
+ throw new CsvImportValidationException(List.of(
+ message("headers", "The uploaded CSV file no longer matches the previewed headers.")
+ ));
+ }
+ }
+ }
+
+ private Object sampleValue(
+ List<List<String>> rows,
+ int columnIndex,
+ CsvImportColumn column,
+ CsvImportConfiguration config,
+ String selectedTimestampColumn
+ ) {
+ for (var row : rows) {
+ var converted = convertValue(row.get(columnIndex), column, config, selectedTimestampColumn, null);
+ if (converted != null) {
+ return converted;
+ }
+ }
+ return null;
+ }
+
+ private String inferType(List<List<String>> rows, int columnIndex, CsvImportConfiguration config) {
+ var allBoolean = true;
+ var allLong = true;
+ var allNumber = true;
+
+ for (var row : rows) {
+ var value = row.get(columnIndex);
+ if (value == null || value.isBlank()) {
+ continue;
+ }
+ if (!isBoolean(value)) {
+ allBoolean = false;
+ }
+ if (!isLong(value, config)) {
+ allLong = false;
+ }
+ if (!isNumber(value, config)) {
+ allNumber = false;
+ }
+ }
+
+ var timestampCandidate = isTimestampCandidate(rows, columnIndex, config);
+ if (timestampCandidate || allLong) {
+ return "LONG";
+ } else if (allBoolean) {
+ return "BOOLEAN";
+ } else if (allNumber) {
+ return "FLOAT";
+ }
+ return "STRING";
+ }
+
+ private boolean isTimestampCandidate(List<List<String>> rows, int columnIndex, CsvImportConfiguration config) {
+ var hasValue = false;
+ for (var row : rows) {
+ var value = row.get(columnIndex);
+ if (value == null || value.isBlank()) {
+ continue;
+ }
+ hasValue = true;
+ if (!canParseTimestamp(value, config)) {
+ return false;
+ }
+ }
+ return hasValue;
+ }
+
+ private boolean canParseTimestamp(String value, CsvImportConfiguration config) {
+ if (isLong(value, config)) {
+ return true;
+ }
+ var timestampFormat = config == null ? null : config.getTimestampFormat();
+ if (timestampFormat == null || timestampFormat.isBlank()) {
+ return false;
+ }
+ try {
+ DateTimeFormatter.ofPattern(timestampFormat, Locale.ENGLISH).parse(value.trim());
+ return true;
+ } catch (DateTimeParseException e) {
+ return false;
+ }
+ }
+
+ private Object convertValue(
+ String rawValue,
+ CsvImportColumn column,
+ CsvImportConfiguration config,
+ String timestampColumn,
+ Integer rowNumber
+ ) {
+ if (rawValue == null || rawValue.isBlank()) {
+ if (rowNumber != null && Objects.equals(column.getRuntimeName(), timestampColumn)) {
+ throw new CsvImportValidationException(List.of(
+ message(
+ "rows",
+ "Row " + rowNumber + " is missing a value for timestamp column \"" + column.getCsvColumn() + "\"."
+ )
+ ));
+ }
+ return null;
+ }
+
+ var trimmed = rawValue.trim();
+ try {
+ if (Objects.equals(column.getRuntimeName(), timestampColumn)) {
+ return parseTimestamp(trimmed, config);
+ }
+
+ return switch (finalRuntimeType(column, timestampColumn)) {
+ case "BOOLEAN" -> Boolean.parseBoolean(trimmed.toLowerCase(Locale.ENGLISH));
+ case "LONG" -> Long.parseLong(normalizeNumber(trimmed, config));
+ case "FLOAT" -> Double.parseDouble(normalizeNumber(trimmed, config));
+ default -> trimmed;
+ };
+ } catch (RuntimeException e) {
+ if (rowNumber == null) {
+ throw e;
+ }
+
+ throw new CsvImportValidationException(List.of(
+ message(
+ "rows",
+ "Row " + rowNumber + " contains an invalid value for column \"" + column.getCsvColumn() + "\"."
+ )
+ ));
+ }
+ }
+
+ private long parseTimestamp(String value, CsvImportConfiguration config) {
+ if (isLong(value, config)) {
+ return Long.parseLong(normalizeNumber(value, config));
+ }
+
+ var formatter = DateTimeFormatter.ofPattern(config.getTimestampFormat(), Locale.ENGLISH);
+ try {
+ return LocalDateTime.parse(value, formatter).toInstant(ZoneOffset.UTC).toEpochMilli();
+ } catch (DateTimeParseException e) {
+ return Instant.from(formatter.parse(value)).toEpochMilli();
+ }
+ }
+
+ private boolean isBoolean(String value) {
+ return "true".equalsIgnoreCase(value.trim()) || "false".equalsIgnoreCase(value.trim());
+ }
+
+ private boolean isNumber(String value, CsvImportConfiguration config) {
+ try {
+ Double.parseDouble(normalizeNumber(value, config));
+ return true;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+
+ private boolean isLong(String value, CsvImportConfiguration config) {
+ try {
+ var normalized = normalizeNumber(value, config);
+ if (normalized.contains(".")) {
+ return false;
+ }
+ Long.parseLong(normalized);
+ return true;
+ } catch (NumberFormatException e) {
+ return false;
+ }
+ }
+
+ private String normalizeNumber(String value, CsvImportConfiguration config) {
+ if (config != null && ",".equals(config.getDecimalSeparator())) {
+ return value.replace(".", "").replace(',', '.');
+ }
+ return value.replace(",", "");
+ }
+
+ private String suggestedScope(String inferredType) {
+ return switch (inferredType) {
+ case "LONG", "FLOAT" -> PropertyScope.MEASUREMENT_PROPERTY.name();
+ case "BOOLEAN", "STRING" -> PropertyScope.DIMENSION_PROPERTY.name();
+ default -> PropertyScope.MEASUREMENT_PROPERTY.name();
+ };
+ }
+
+ private String finalRuntimeType(CsvImportColumn column, String timestampColumn) {
+ if (Objects.equals(column.getRuntimeName(), timestampColumn)) {
+ return "LONG";
+ }
+ return Optional.ofNullable(column.getRuntimeType())
+ .orElseGet(column::getInferredType);
+ }
+
+ private String finalPropertyScope(CsvImportColumn column, String timestampColumn) {
+ if (Objects.equals(column.getRuntimeName(), timestampColumn)) {
+ return PropertyScope.HEADER_PROPERTY.name();
+ }
+ return Optional.ofNullable(column.getPropertyScope()).orElse(PropertyScope.MEASUREMENT_PROPERTY.name());
+ }
+
+ private String finalSemanticType(CsvImportColumn column, String timestampColumn) {
+ if (Objects.equals(column.getRuntimeName(), timestampColumn)) {
+ return SO.DATE_TIME;
+ }
+ return column.getSemanticType();
+ }
+
+ private String toRuntimeTypeUri(String runtimeType) {
+ return switch (runtimeType) {
+ case "BOOLEAN" -> XSD.BOOLEAN.toString();
+ case "LONG" -> XSD.LONG.toString();
+ case "FLOAT" -> XSD.FLOAT.toString();
+ default -> XSD.STRING.toString();
+ };
+ }
+
+ private CsvImportValidationMessage message(String field, String message) {
+ return new CsvImportValidationMessage(field, message);
+ }
+
+ private void parseCsvFile(Path path, CsvImportConfiguration config, CsvRowConsumer consumer) throws IOException {
+ try (var reader = new PushbackReader(Files.newBufferedReader(path, StandardCharsets.UTF_8), 1)) {
+ var delimiter = normalizeDelimiter(config == null ? null : config.getDelimiter());
+ var hasHeader = config == null || config.isHasHeader();
+ List<String> headers = null;
+ int rowNumber = 0;
+ List<String> row;
+
+ while ((row = readNextRow(reader, delimiter)) != null) {
+ if (isBlankRow(row)) {
+ continue;
+ }
+
+ if (headers == null) {
+ if (hasHeader) {
+ headers = normalizeHeaders(row);
+ consumer.onHeaders(headers);
+ } else {
+ headers = generateHeaders(row.size());
+ consumer.onHeaders(headers);
+ rowNumber += 1;
+ consumer.onRow(rowNumber, row);
+ }
+ } else {
+ rowNumber += 1;
+ consumer.onRow(rowNumber, row);
+ }
+ }
+ }
+ }
+
+ private List<String> readNextRow(PushbackReader reader, char delimiter) throws IOException {
+ var currentRow = new ArrayList<String>();
+ var currentValue = new StringBuilder();
+ var inQuotes = false;
+ var readAny = false;
+
+ while (true) {
+ var nextInt = reader.read();
+ if (nextInt == -1) {
+ if (!readAny && currentValue.length() == 0 && currentRow.isEmpty()) {
+ return null;
+ }
+ currentRow.add(currentValue.toString());
+ return currentRow;
+ }
+
+ readAny = true;
+ var currentChar = (char) nextInt;
+ if (currentChar == '"') {
+ if (inQuotes) {
+ var escapedCandidate = reader.read();
+ if (escapedCandidate == '"') {
+ currentValue.append('"');
+ } else {
+ inQuotes = false;
+ if (escapedCandidate != -1) {
+ reader.unread(escapedCandidate);
+ }
+ }
+ } else {
+ inQuotes = true;
+ }
+ } else if (!inQuotes && currentChar == delimiter) {
+ currentRow.add(currentValue.toString());
+ currentValue = new StringBuilder();
+ } else if (!inQuotes && (currentChar == '\n' || currentChar == '\r')) {
+ if (currentChar == '\r') {
+ var maybeLineFeed = reader.read();
+ if (maybeLineFeed != '\n' && maybeLineFeed != -1) {
+ reader.unread(maybeLineFeed);
+ }
+ }
+ currentRow.add(currentValue.toString());
+ return currentRow;
+ } else {
+ currentValue.append(currentChar);
+ }
+ }
+ }
+
+ private char normalizeDelimiter(String delimiter) {
+ if (delimiter == null || delimiter.isEmpty()) {
+ return ',';
+ }
+ if ("\\t".equals(delimiter)) {
+ return '\t';
+ }
+ return delimiter.charAt(0);
+ }
+
+ private List<String> normalizeHeaders(List<String> headers) {
+ var normalized = new ArrayList<String>();
+ for (int i = 0; i < headers.size(); i++) {
+ var value = headers.get(i);
+ if (i == 0) {
+ value = stripBom(value);
+ }
+ var trimmed = value == null ? "" : value.trim();
+ normalized.add(trimmed.isEmpty() ? "column_" + (i + 1) : trimmed);
+ }
+ return normalized;
+ }
+
+ private String stripBom(String value) {
+ return value == null ? null : value.replace("\uFEFF", "");
+ }
+
+ private List<String> generateHeaders(int size) {
+ var headers = new ArrayList<String>();
+ for (int i = 0; i < size; i++) {
+ headers.add("column_" + (i + 1));
+ }
+ return headers;
+ }
+
+ private boolean isBlankRow(List<String> row) {
+ return row.stream().allMatch(cell -> cell == null || cell.trim().isEmpty());
+ }
+
+ @FunctionalInterface
+ private interface CsvRowConsumer {
+ default void onHeaders(List<String> headers) {
+ }
+
+ void onRow(int rowNumber, List<String> row);
+ }
+
+ static final class CsvFileSample {
+
+ private final List<String> headers;
+ private final List<List<String>> rows;
+
+ CsvFileSample(List<String> headers, List<List<String>> rows) {
+ this.headers = headers;
+ this.rows = rows;
+ }
+
+ List<String> headers() {
+ return headers;
+ }
+
+ List<List<String>> rows() {
+ return rows;
+ }
+ }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportUploadStorage.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportUploadStorage.java
new file mode 100644
index 0000000000..a62dc1495c
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportUploadStorage.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Optional;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+class CsvImportUploadStorage {
+
+ private static final Duration DEFAULT_TTL = Duration.ofHours(12);
+
+ private final ConcurrentMap<String, StoredUpload> uploads;
+ private final Duration ttl;
+
+ CsvImportUploadStorage() {
+ this(DEFAULT_TTL);
+ }
+
+ CsvImportUploadStorage(Duration ttl) {
+ this.uploads = new ConcurrentHashMap<>();
+ this.ttl = ttl;
+ }
+
+ StoredUpload store(MultipartFile file, String ownerSid) throws IOException {
+ cleanupExpired();
+
+ var uploadId = UUID.randomUUID().toString();
+ var tempFile = Files.createTempFile("streampipes-csv-import-", ".csv");
+ try (var inputStream = file.getInputStream()) {
+ Files.copy(inputStream, tempFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ var upload = new StoredUpload(
+ uploadId,
+ tempFile,
+ ownerSid,
+ Instant.now()
+ );
+ uploads.put(uploadId, upload);
+ return upload;
+ }
+
+ Optional<StoredUpload> get(String uploadId) {
+ cleanupExpired();
+ return Optional.ofNullable(uploads.get(uploadId));
+ }
+
+ void remove(String uploadId) {
+ var removed = uploads.remove(uploadId);
+ if (removed != null) {
+ deleteQuietly(removed.path());
+ }
+ }
+
+ private void cleanupExpired() {
+ var expiresBefore = Instant.now().minus(ttl);
+ uploads.entrySet().removeIf(entry -> {
+ var expired = entry.getValue().createdAt().isBefore(expiresBefore);
+ if (expired) {
+ deleteQuietly(entry.getValue().path());
+ }
+ return expired;
+ });
+ }
+
+ private void deleteQuietly(Path path) {
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException ignored) {
+ // Best-effort cleanup of temporary uploads.
+ }
+ }
+
+ static final class StoredUpload {
+
+ private final String uploadId;
+ private final Path path;
+ private final String ownerSid;
+ private final Instant createdAt;
+
+ StoredUpload(String uploadId, Path path, String ownerSid, Instant createdAt) {
+ this.uploadId = uploadId;
+ this.path = path;
+ this.ownerSid = ownerSid;
+ this.createdAt = createdAt;
+ }
+
+ String uploadId() {
+ return uploadId;
+ }
+
+ Path path() {
+ return path;
+ }
+
+ String ownerSid() {
+ return ownerSid;
+ }
+
+ Instant createdAt() {
+ return createdAt;
+ }
+ }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationException.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationException.java
new file mode 100644
index 0000000000..f68f603c99
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.model.datalake.importer.CsvImportValidationMessage;
+
+import java.util.List;
+
+public class CsvImportValidationException extends RuntimeException {
+
+ private final List<CsvImportValidationMessage> validationMessages;
+
+ public CsvImportValidationException(List<CsvImportValidationMessage> validationMessages) {
+ super(validationMessages.isEmpty() ? "CSV import validation failed" : validationMessages.get(0).getMessage());
+ this.validationMessages = validationMessages;
+ }
+
+ public List<CsvImportValidationMessage> getValidationMessages() {
+ return validationMessages;
+ }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationService.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationService.java
new file mode 100644
index 0000000000..8a8586fc0e
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationService.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.importer.CsvImportConfiguration;
+import org.apache.streampipes.model.datalake.importer.CsvImportPreviewRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaIssue;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaIssueType;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaValidationRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportTarget;
+import org.apache.streampipes.model.datalake.importer.CsvImportTargetMode;
+import org.apache.streampipes.model.datalake.importer.CsvImportValidationMessage;
+import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
+import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.vocabulary.SO;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+class CsvImportValidationService {
+
+ private final IDataExplorerSchemaManagement schemaManagement;
+
+ CsvImportValidationService(IDataExplorerSchemaManagement schemaManagement) {
+ this.schemaManagement = schemaManagement;
+ }
+
+ List<CsvImportValidationMessage> validatePreviewConfiguration(CsvImportPreviewRequest request) {
+ var messages = new ArrayList<CsvImportValidationMessage>();
+ if (request == null) {
+ messages.add(message("request", "Import request must be provided."));
+ return messages;
+ }
+ validateCsvConfig(request.getCsvConfig(), messages);
+ return messages;
+ }
+
+ List<CsvImportValidationMessage> validatePreviewRequest(CsvImportPreviewRequest request) {
+ var messages = validatePreviewConfiguration(request);
+ if (request == null) {
+ return messages;
+ }
+ if (request.getHeaders() == null || request.getHeaders().isEmpty()) {
+ messages.add(message("headers", "At least one header must be provided."));
+ }
+ if (request.getRows() == null || request.getRows().isEmpty()) {
+ messages.add(message("rows", "At least one row must be provided."));
+ }
+ validateRowsMatchHeaders(request.getHeaders(), request.getRows(), messages);
+ return messages;
+ }
+
+ List<CsvImportValidationMessage> validateInlineImportRequest(CsvImportRequest request) {
+ var messages = validateStoredImportRequest(request);
+ if (request == null) {
+ return messages;
+ }
+ if (request.getHeaders() == null || request.getHeaders().isEmpty()) {
+ messages.add(message("headers", "At least one header must be provided."));
+ }
+ if (request.getRows() == null || request.getRows().isEmpty()) {
+ messages.add(message("rows", "At least one row must be provided."));
+ }
+ validateRowsMatchHeaders(request.getHeaders(), request.getRows(), messages);
+ return messages;
+ }
+
+ List<CsvImportValidationMessage> validateStoredImportRequest(CsvImportRequest request) {
+ var messages = new ArrayList<CsvImportValidationMessage>();
+ if (request == null) {
+ messages.add(message("request", "Import request must be provided."));
+ return messages;
+ }
+ validateCsvConfig(request.getCsvConfig(), messages);
+
+ if (request.getTarget() == null || request.getTarget().getMode() == null) {
+ messages.add(message("target", "An import target must be selected."));
+ }
+ if (request.getTimestampColumn() == null || request.getTimestampColumn().isBlank()) {
+ messages.add(message("timestampColumn", "A timestamp column must be selected."));
+ }
+ if (request.getColumns() == null || request.getColumns().isEmpty()) {
+ messages.add(message("columns", "Column configuration must be provided."));
+ }
+ if (!hasUploadId(request)
+ && (request.getRows() == null || request.getRows().isEmpty())
+ && (request.getHeaders() == null || request.getHeaders().isEmpty())) {
+ messages.add(message("uploadId", "Either an uploadId or inline CSV rows must be provided."));
+ }
+ return messages;
+ }
+
+ List<CsvImportValidationMessage> validateSchemaRequest(CsvImportSchemaValidationRequest request) {
+ var messages = new ArrayList<CsvImportValidationMessage>();
+ if (request == null) {
+ messages.add(message("request", "Schema validation request must be provided."));
+ return messages;
+ }
+ if (request.getTarget() == null || request.getTarget().getMode() == null) {
+ messages.add(message("target", "An import target must be selected."));
+ }
+ if (request.getTimestampColumn() == null || request.getTimestampColumn().isBlank()) {
+ messages.add(message("timestampColumn", "A timestamp column must be selected."));
+ }
+ if (request.getColumns() == null || request.getColumns().isEmpty()) {
+ messages.add(message("columns", "Column configuration must be provided."));
+ }
+ return messages;
+ }
+
+ List<CsvImportValidationMessage> validatePreviewTarget(CsvImportTarget target) {
+ var messages = new ArrayList<CsvImportValidationMessage>();
+ if (target == null || target.getMode() == null) {
+ return messages;
+ }
+
+ var measurementName = target.getMeasurementName();
+ if (measurementName == null || measurementName.isBlank()) {
+ messages.add(message("target.measurementName", "A measurement name must be provided."));
+ return messages;
+ }
+
+ if (target.getMode() == CsvImportTargetMode.NEW) {
+ if (schemaManagement.getExistingMeasureByName(measurementName.trim()).isPresent()) {
+ messages.add(message("target.measurementName", "A measurement with this name already exists."));
+ }
+ } else if (schemaManagement.getExistingMeasureByName(measurementName.trim()).isEmpty()) {
+ messages.add(message("target.measurementName", "The selected measurement does not exist."));
+ }
+
+ return messages;
+ }
+
+ List<CsvImportSchemaIssue> validateSchemaTarget(
+ CsvImportTarget target,
+ EventSchema importSchema,
+ String timestampColumn
+ ) {
+ var issues = new ArrayList<CsvImportSchemaIssue>();
+ if (target == null || target.getMode() == null) {
+ return issues;
+ }
+
+ var measurementName = target.getMeasurementName();
+ if (measurementName == null || measurementName.isBlank() || target.getMode() == CsvImportTargetMode.NEW) {
+ return issues;
+ }
+
+ var existingOpt = schemaManagement.getExistingMeasureByName(measurementName.trim());
+ if (existingOpt.isPresent() && importSchema != null) {
+ var effectiveTimestampColumn = timestampColumn;
+ if (effectiveTimestampColumn == null) {
+ effectiveTimestampColumn = importSchema.getEventProperties().stream()
+ .filter(property -> SO.DATE_TIME.equals(property.getSemanticType()))
+ .map(EventProperty::getRuntimeName)
+ .findFirst()
+ .orElse(null);
+ }
+ if (effectiveTimestampColumn != null) {
+ issues.addAll(compareSchemas(existingOpt.get(), importSchema, effectiveTimestampColumn));
+ }
+ }
+ return issues;
+ }
+
+ List<CsvImportValidationMessage> validateImportTarget(
+ CsvImportTarget target,
+ EventSchema importSchema,
+ String timestampColumn
+ ) {
+ var messages = new ArrayList<CsvImportValidationMessage>();
+ if (target == null || target.getMode() == null) {
+ return messages;
+ }
+
+ var measurementName = target.getMeasurementName();
+ if (measurementName == null || measurementName.isBlank()) {
+ messages.add(message("target.measurementName", "A measurement name must be provided."));
+ return messages;
+ }
+
+ if (target.getMode() == CsvImportTargetMode.NEW) {
+ if (schemaManagement.getExistingMeasureByName(measurementName.trim()).isPresent()) {
+ messages.add(message("target.measurementName", "A measurement with this name already exists."));
+ }
+ return messages;
+ }
+
+ var existingOpt = schemaManagement.getExistingMeasureByName(measurementName.trim());
+ if (existingOpt.isEmpty()) {
+ messages.add(message("target.measurementName", "The selected measurement does not exist."));
+ } else if (importSchema != null && !compareSchemas(existingOpt.get(), importSchema, timestampColumn).isEmpty()) {
+ messages.add(message("columns", "Imported columns must exactly match the existing measurement schema."));
+ }
+ return messages;
+ }
+
+ DataLakeMeasure requireExistingMeasurement(String measurementName) {
+ return schemaManagement.getExistingMeasureByName(measurementName)
+ .orElseThrow(() -> new CsvImportValidationException(List.of(
+ message("target.measurementName", "The selected measurement does not exist.")
+ )));
+ }
+
+ private List<CsvImportSchemaIssue> compareSchemas(
+ DataLakeMeasure existingMeasure,
+ EventSchema importSchema,
+ String timestampColumn
+ ) {
+ var issues = new ArrayList<CsvImportSchemaIssue>();
+
+ if (!Objects.equals(existingMeasure.getTimestampFieldName(), timestampColumn)) {
+ issues.add(issue(
+ CsvImportSchemaIssueType.TIMESTAMP_COLUMN_MISMATCH,
+ timestampColumn,
+ existingMeasure.getTimestampFieldName(),
+ timestampColumn
+ ));
+ }
+
+ var existingProperties = existingMeasure.getEventSchema()
+ .getEventProperties()
+ .stream()
+ .collect(Collectors.toMap(EventProperty::getRuntimeName, property -> property));
+ var importedProperties = importSchema.getEventProperties()
+ .stream()
+ .collect(Collectors.toMap(EventProperty::getRuntimeName, property -> property));
+
+ importedProperties.keySet().stream()
+ .filter(property -> !existingProperties.containsKey(property))
+ .sorted()
+ .forEach(property -> issues.add(issue(
+ CsvImportSchemaIssueType.COLUMN_NAME_MISMATCH,
+ property,
+ null,
+ property
+ )));
+
+ for (var entry : existingProperties.entrySet()) {
+ var imported = importedProperties.get(entry.getKey());
+ if (imported == null) {
+ continue;
+ }
+
+ if (!Objects.equals(getRuntimeType(entry.getValue()), getRuntimeType(imported))) {
+ issues.add(issue(
+ CsvImportSchemaIssueType.COLUMN_TYPE_MISMATCH,
+ entry.getKey(),
+ getRuntimeType(entry.getValue()),
+ getRuntimeType(imported)
+ ));
+ }
+
+ if (!Objects.equals(entry.getValue().getPropertyScope(), imported.getPropertyScope())) {
+ issues.add(issue(
+ CsvImportSchemaIssueType.COLUMN_SCOPE_MISMATCH,
+ entry.getKey(),
+ entry.getValue().getPropertyScope(),
+ imported.getPropertyScope()
+ ));
+ }
+ }
+
+ return issues;
+ }
+
+ private void validateRowsMatchHeaders(
+ List<String> headers,
+ List<List<String>> rows,
+ List<CsvImportValidationMessage> messages
+ ) {
+ if (headers == null || rows == null) {
+ return;
+ }
+ for (int i = 0; i < rows.size(); i++) {
+ if (rows.get(i).size() != headers.size()) {
+ messages.add(message("rows", "Row " + (i + 1) + " does not match the header size."));
+ return;
+ }
+ }
+ }
+
+ private void validateCsvConfig(CsvImportConfiguration csvConfig, List<CsvImportValidationMessage> messages) {
+ if (csvConfig == null) {
+ messages.add(message("csvConfig", "CSV configuration must be provided."));
+ return;
+ }
+ if (csvConfig.getDecimalSeparator() == null
+ || (!",".equals(csvConfig.getDecimalSeparator()) && !".".equals(csvConfig.getDecimalSeparator()))) {
+ messages.add(message("csvConfig.decimalSeparator", "Decimal separator must be '.' or ','."));
+ }
+ }
+
+ private boolean hasUploadId(CsvImportRequest request) {
+ return request != null && request.getUploadId() != null && !request.getUploadId().isBlank();
+ }
+
+ private CsvImportValidationMessage message(String field, String message) {
+ return new CsvImportValidationMessage(field, message);
+ }
+
+ private CsvImportSchemaIssue issue(
+ CsvImportSchemaIssueType type,
+ String columnName,
+ String expected,
+ String actual
+ ) {
+ return new CsvImportSchemaIssue(type, columnName, expected, actual);
+ }
+
+ private String getRuntimeType(EventProperty property) {
+ if (property instanceof EventPropertyPrimitive primitive) {
+ return primitive.getRuntimeType();
+ }
+ return null;
+ }
+}
diff --git a/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/DataLakeImportResource.java b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/DataLakeImportResource.java
new file mode 100644
index 0000000000..7fde0ca48a
--- /dev/null
+++ b/streampipes-rest/src/main/java/org/apache/streampipes/rest/impl/datalake/importer/DataLakeImportResource.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.model.datalake.importer.CsvImportPreviewRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportPreviewResult;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportResult;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaValidationRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaValidationResult;
+import org.apache.streampipes.model.datalake.importer.CsvImportTargetMode;
+import org.apache.streampipes.rest.impl.datalake.AbstractDataLakeResource;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestPart;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.IOException;
+
+@RestController
+@RequestMapping("/api/v4/datalake/import")
+public class DataLakeImportResource extends AbstractDataLakeResource {
+
+ private final CsvDataLakeImportService importService;
+
+ public DataLakeImportResource() {
+ super();
+ this.importService = new CsvDataLakeImportService(getDataLakeMeasureManagement());
+ }
+
+ /**
+ * Generates a CSV preview from inline JSON content or reuses a previously uploaded file via {@code uploadId}.
+ * <p>
+ * This endpoint supports the second step of the single-upload workflow: the UI uploads the file once through the
+ * multipart preview endpoint, receives an {@code uploadId}, and can then call this JSON endpoint again when preview
+ * settings change without re-uploading the CSV.
+ */
+ @PostMapping(
+ path = "/preview",
+ consumes = MediaType.APPLICATION_JSON_VALUE,
+ produces = MediaType.APPLICATION_JSON_VALUE
+ )
+ @PreAuthorize("this.hasWriteAuthority()")
+ public ResponseEntity<CsvImportPreviewResult> preview(@RequestBody CsvImportPreviewRequest request) {
+ if (!hasWritePermission(request.getTarget())) {
+ return ResponseEntity.status(HttpStatus.FORBIDDEN).build();
+ }
+ return ok(importService.preview(request, getAuthenticatedUserSid()));
+ }
+
+ /**
+ * Uploads a CSV file, stores it temporarily on the server, and returns a preview together with an {@code uploadId}.
+ * <p>
+ * The returned {@code uploadId} is later used by the final import call so large files only need to be transferred
+ * once.
+ */
+ @PostMapping(
+ path = "/preview",
+ consumes = MediaType.MULTIPART_FORM_DATA_VALUE,
+ produces = MediaType.APPLICATION_JSON_VALUE
+ )
+ @PreAuthorize("this.hasWriteAuthority()")
+ public ResponseEntity<CsvImportPreviewResult> preview(
+ @RequestPart("file") MultipartFile file,
+ @RequestPart("request") CsvImportPreviewRequest request
+ ) throws IOException {
+ if (!hasWritePermission(request.getTarget())) {
+ return ResponseEntity.status(HttpStatus.FORBIDDEN).build();
+ }
+ return ok(importService.preview(file, request, getAuthenticatedUserSid()));
+ }
+
+ /**
+ * Validates the configured CSV column mapping against an existing target measurement schema.
+ */
+ @PostMapping(
+ path = "/validate-schema",
+ consumes = MediaType.APPLICATION_JSON_VALUE,
+ produces = MediaType.APPLICATION_JSON_VALUE
+ )
+ @PreAuthorize("this.hasWriteAuthority()")
+ public ResponseEntity<CsvImportSchemaValidationResult> validateSchema(
+ @RequestBody CsvImportSchemaValidationRequest request
+ ) {
+ if (!hasWritePermission(request.getTarget())) {
+ return ResponseEntity.status(HttpStatus.FORBIDDEN).build();
+ }
+ return ok(importService.validateSchema(request));
+ }
+
+ /**
+ * Imports CSV data either from inline JSON rows or from a previously uploaded file referenced by {@code uploadId}.
+ * <p>
+ * In the single-upload workflow, clients first call the multipart preview endpoint, keep the returned
+ * {@code uploadId}, and then call this endpoint with that identifier and the final import configuration.
+ */
+ @PostMapping(
+ consumes = MediaType.APPLICATION_JSON_VALUE,
+ produces = MediaType.APPLICATION_JSON_VALUE
+ )
+ @PreAuthorize("this.hasWriteAuthority()")
+ public ResponseEntity<CsvImportResult> importData(@RequestBody CsvImportRequest request) {
+ if (!hasWritePermission(request.getTarget())) {
+ return ResponseEntity.status(HttpStatus.FORBIDDEN).build();
+ }
+
+ try {
+ return ok(importService.importData(request, getAuthenticatedUserSid()));
+ } catch (CsvImportValidationException e) {
+ var result = new CsvImportResult();
+ result.setValidationMessages(e.getValidationMessages());
+ return ResponseEntity.badRequest().body(result);
+ }
+ }
+
+ private boolean hasWritePermission(org.apache.streampipes.model.datalake.importer.CsvImportTarget target) {
+ return target == null
+ || target.getMode() != CsvImportTargetMode.EXISTING
+ || getDataLakeMeasureManagement().getExistingMeasureByName(target.getMeasurementName()).isEmpty()
+ || this.checkPermissionByName(target.getMeasurementName(), "WRITE");
+ }
+}
diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriterTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriterTest.java
new file mode 100644
index 0000000000..ebeee70cfb
--- /dev/null
+++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/DataLakeDataWriterTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DataLakeDataWriterTest {
+
+ @Test
+ void shouldSkipNullValuesWhenBuildingEventMap() {
+ var headers = List.of("timestamp", "temperature", "status");
+ List<Object> row = new java.util.ArrayList<>();
+ row.add(1710000000000L);
+ row.add(null);
+ row.add("ok");
+
+ var eventMap = DataLakeDataWriter.toEventMap(row, headers);
+
+ assertEquals(2, eventMap.size());
+ assertEquals(1710000000000L, eventMap.get("timestamp"));
+ assertEquals("ok", eventMap.get("status"));
+ assertFalse(eventMap.containsKey("temperature"));
+ }
+
+ @Test
+ void shouldAllowMissingFieldsWhenConfigured() {
+ var expected = new HashSet<>(List.of("timestamp", "temperature", "status"));
+ var actual = new HashSet<>(List.of("timestamp", "status"));
+
+ assertTrue(DataLakeDataWriter.matchesRuntimeNames(expected, actual, true));
+ assertFalse(DataLakeDataWriter.matchesRuntimeNames(expected, actual, false));
+ }
+}
diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvDataLakeImportServiceTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvDataLakeImportServiceTest.java
new file mode 100644
index 0000000000..f0dcec8947
--- /dev/null
+++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvDataLakeImportServiceTest.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.importer.CsvImportColumn;
+import org.apache.streampipes.model.datalake.importer.CsvImportConfiguration;
+import org.apache.streampipes.model.datalake.importer.CsvImportPreviewRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaIssueType;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaValidationRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportTarget;
+import org.apache.streampipes.model.datalake.importer.CsvImportTargetMode;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
+import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.rest.impl.datalake.DataLakeDataWriter;
+import org.apache.streampipes.vocabulary.SO;
+import org.apache.streampipes.vocabulary.XSD;
+
+import org.junit.jupiter.api.Test;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.ByteArrayInputStream;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class CsvDataLakeImportServiceTest {
+
+ @Test
+ void shouldInferTypesAndRejectDuplicateNewMeasurementNames() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var dataWriter = mock(DataLakeDataWriter.class);
+ var service = new CsvDataLakeImportService(schemaManagement, dataWriter);
+
+ var existingMeasure = new DataLakeMeasure();
+ existingMeasure.setMeasureName("existing-measure");
+ when(schemaManagement.getExistingMeasureByName("existing-measure"))
+ .thenReturn(Optional.of(existingMeasure));
+
+ var result = service.preview(makePreviewRequest("existing-measure"));
+
+ assertFalse(result.isValid());
+ assertEquals("LONG", result.getColumns().get(0).getInferredType());
+ assertEquals("FLOAT", result.getColumns().get(1).getInferredType());
+ assertTrue(result.getValidationMessages()
+ .stream()
+ .anyMatch(message -> message.getMessage().contains("already exists")));
+ }
+
+ @Test
+ void shouldReportSchemaMismatchInDedicatedValidationCall() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var dataWriter = mock(DataLakeDataWriter.class);
+ var service = new CsvDataLakeImportService(schemaManagement, dataWriter);
+
+ var existingMeasure = new DataLakeMeasure();
+ existingMeasure.setMeasureName("existing-measure");
+ existingMeasure.setTimestampField("s0::timestamp");
+ existingMeasure.setEventSchema(makeExistingSchema());
+
+ when(schemaManagement.getExistingMeasureByName("existing-measure"))
+ .thenReturn(Optional.of(existingMeasure));
+
+ var request = new CsvImportSchemaValidationRequest();
+ request.setTarget(makeTarget(CsvImportTargetMode.EXISTING, "existing-measure"));
+ request.setTimestampColumn("timestamp");
+ request.setColumns(makeImportRequest(CsvImportTargetMode.EXISTING, "existing-measure").getColumns());
+ request.getColumns().get(1).setRuntimeType("STRING");
+
+ var result = service.validateSchema(request);
+
+ assertFalse(result.isValid());
+ assertTrue(result.getIssues()
+ .stream()
+ .anyMatch(issue -> issue.getType() == CsvImportSchemaIssueType.COLUMN_TYPE_MISMATCH));
+ }
+
+ @Test
+ void shouldCreateNewMeasurementAndReuseSharedWriter() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var dataWriter = mock(DataLakeDataWriter.class);
+ var service = new CsvDataLakeImportService(schemaManagement, dataWriter);
+
+ when(schemaManagement.getExistingMeasureByName("new-measure"))
+ .thenReturn(Optional.empty());
+ when(schemaManagement.createOrUpdateMeasurement(any(DataLakeMeasure.class), any()))
+ .thenAnswer(invocation -> {
+ var measure = invocation.getArgument(0, DataLakeMeasure.class);
+ measure.setElementId("measure-id");
+ return measure;
+ });
+
+ var result = service.importData(makeImportRequest(CsvImportTargetMode.NEW, "new-measure"), "sid");
+
+ assertTrue(result.isCreatedNewMeasurement());
+ assertEquals(2, result.getImportedRowCount());
+ assertEquals("new-measure", result.getMeasurementName());
+ verify(dataWriter).writeData(any(DataLakeMeasure.class), anyList(), anyList());
+ }
+
+ @Test
+ void shouldPreviewOnceAndImportFromStoredUpload() throws Exception {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var dataWriter = mock(DataLakeDataWriter.class);
+ var uploadStorage = new CsvImportUploadStorage();
+ var service = new CsvDataLakeImportService(schemaManagement, dataWriter, uploadStorage);
+
+ when(schemaManagement.getExistingMeasureByName("uploaded-measure"))
+ .thenReturn(Optional.empty());
+ when(schemaManagement.createOrUpdateMeasurement(any(DataLakeMeasure.class), any()))
+ .thenAnswer(invocation -> {
+ var measure = invocation.getArgument(0, DataLakeMeasure.class);
+ measure.setElementId("measure-id");
+ return measure;
+ });
+
+ var previewRequest = new CsvImportPreviewRequest();
+ previewRequest.setCsvConfig(makeCsvConfigWithCommaDelimiter());
+ previewRequest.setTarget(makeTarget(CsvImportTargetMode.NEW, "uploaded-measure"));
+
+ var multipartFile = mock(MultipartFile.class);
+ when(multipartFile.getOriginalFilename()).thenReturn("upload.csv");
+ when(multipartFile.getInputStream()).thenReturn(new ByteArrayInputStream(
+ "timestamp,temperature\n1710000000000,21.3\n1710000060000,22.1\n".getBytes()
+ ));
+
+ var previewResult = service.preview(
+ multipartFile,
+ previewRequest,
+ "sid"
+ );
+
+ assertTrue(previewResult.isValid());
+ assertEquals(2, previewResult.getPreviewRows().size());
+ assertTrue(previewResult.getUploadId() != null && !previewResult.getUploadId().isBlank());
+
+ var importRequest = new CsvImportRequest();
+ importRequest.setUploadId(previewResult.getUploadId());
+ importRequest.setCsvConfig(makeCsvConfigWithCommaDelimiter());
+ importRequest.setTarget(makeTarget(CsvImportTargetMode.NEW, "uploaded-measure"));
+ importRequest.setTimestampColumn("timestamp");
+ importRequest.setColumns(previewResult.getColumns());
+
+ var importResult = service.importData(importRequest, "sid");
+
+ assertTrue(importResult.isCreatedNewMeasurement());
+ assertEquals(2, importResult.getImportedRowCount());
+ verify(dataWriter).writeData(any(DataLakeMeasure.class), anyList(), anyList());
+ }
+
+ @Test
+ void shouldRejectMissingTimestampValuesInUploadedCsv() throws Exception {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var dataWriter = mock(DataLakeDataWriter.class);
+ var uploadStorage = new CsvImportUploadStorage();
+ var service = new CsvDataLakeImportService(schemaManagement, dataWriter, uploadStorage);
+
+ when(schemaManagement.getExistingMeasureByName("uploaded-measure"))
+ .thenReturn(Optional.empty());
+ when(schemaManagement.createOrUpdateMeasurement(any(DataLakeMeasure.class), any()))
+ .thenAnswer(invocation -> {
+ var measure = invocation.getArgument(0, DataLakeMeasure.class);
+ measure.setElementId("measure-id");
+ return measure;
+ });
+
+ var previewRequest = new CsvImportPreviewRequest();
+ previewRequest.setCsvConfig(makeCsvConfigWithCommaDelimiter());
+ previewRequest.setTarget(makeTarget(CsvImportTargetMode.NEW, "uploaded-measure"));
+
+ var multipartFile = mock(MultipartFile.class);
+ when(multipartFile.getOriginalFilename()).thenReturn("upload.csv");
+ when(multipartFile.getInputStream()).thenReturn(new ByteArrayInputStream(
+ "timestamp,temperature\n1710000000000,21.3\n,22.1\n".getBytes()
+ ));
+
+ var previewResult = service.preview(multipartFile, previewRequest, "sid");
+
+ var importRequest = new CsvImportRequest();
+ importRequest.setUploadId(previewResult.getUploadId());
+ importRequest.setCsvConfig(makeCsvConfigWithCommaDelimiter());
+ importRequest.setTarget(makeTarget(CsvImportTargetMode.NEW, "uploaded-measure"));
+ importRequest.setTimestampColumn("timestamp");
+ importRequest.setColumns(previewResult.getColumns());
+
+ var exception = assertThrows(
+ CsvImportValidationException.class,
+ () -> service.importData(importRequest, "sid")
+ );
+
+ assertTrue(exception.getValidationMessages()
+ .stream()
+ .anyMatch(message -> message.getMessage().contains("missing a value for timestamp column")));
+ }
+
+ @Test
+ void shouldRejectSchemaMismatchForExistingMeasurement() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var dataWriter = mock(DataLakeDataWriter.class);
+ var service = new CsvDataLakeImportService(schemaManagement, dataWriter);
+
+ var existingMeasure = new DataLakeMeasure();
+ existingMeasure.setMeasureName("existing-measure");
+ existingMeasure.setTimestampField("s0::timestamp");
+ existingMeasure.setEventSchema(makeExistingSchema());
+
+ when(schemaManagement.getExistingMeasureByName("existing-measure"))
+ .thenReturn(Optional.of(existingMeasure));
+
+ var request = makeImportRequest(CsvImportTargetMode.EXISTING, "existing-measure");
+ request.getColumns().get(1).setRuntimeType("STRING");
+
+ var exception = assertThrows(
+ CsvImportValidationException.class,
+ () -> service.importData(request, "sid")
+ );
+
+ assertTrue(exception.getValidationMessages()
+ .stream()
+ .anyMatch(message -> message.getMessage().contains("exactly match")));
+ }
+
+ private CsvImportPreviewRequest makePreviewRequest(String measurementName) {
+ var request = new CsvImportPreviewRequest();
+ request.setCsvConfig(makeCsvConfig());
+ request.setHeaders(List.of("timestamp", "temperature"));
+ request.setRows(List.of(
+ List.of("1710000000000", "21.3"),
+ List.of("1710000060000", "22.1")
+ ));
+ request.setTarget(makeTarget(CsvImportTargetMode.NEW, measurementName));
+ return request;
+ }
+
+ private CsvImportRequest makeImportRequest(CsvImportTargetMode mode, String measurementName) {
+ var timestampColumn = new CsvImportColumn();
+ timestampColumn.setCsvColumn("timestamp");
+ timestampColumn.setRuntimeName("timestamp");
+ timestampColumn.setRuntimeType("LONG");
+ timestampColumn.setPropertyScope("HEADER_PROPERTY");
+ timestampColumn.setSemanticType(SO.DATE_TIME);
+
+ var measurementColumn = new CsvImportColumn();
+ measurementColumn.setCsvColumn("temperature");
+ measurementColumn.setRuntimeName("temperature");
+ measurementColumn.setRuntimeType("FLOAT");
+ measurementColumn.setPropertyScope("MEASUREMENT_PROPERTY");
+
+ var request = new CsvImportRequest();
+ request.setCsvConfig(makeCsvConfig());
+ request.setHeaders(List.of("timestamp", "temperature"));
+ request.setRows(List.of(
+ List.of("1710000000000", "21.3"),
+ List.of("1710000060000", "22.1")
+ ));
+ request.setTarget(makeTarget(mode, measurementName));
+ request.setTimestampColumn("timestamp");
+ request.setColumns(List.of(timestampColumn, measurementColumn));
+ return request;
+ }
+
+ private CsvImportTarget makeTarget(CsvImportTargetMode mode, String measurementName) {
+ var target = new CsvImportTarget();
+ target.setMode(mode);
+ target.setMeasurementName(measurementName);
+ return target;
+ }
+
+ private CsvImportConfiguration makeCsvConfig() {
+ var config = new CsvImportConfiguration();
+ config.setDelimiter(";");
+ config.setDecimalSeparator(".");
+ config.setHasHeader(true);
+ return config;
+ }
+
+ private CsvImportConfiguration makeCsvConfigWithCommaDelimiter() {
+ var config = new CsvImportConfiguration();
+ config.setDelimiter(",");
+ config.setDecimalSeparator(".");
+ config.setHasHeader(true);
+ return config;
+ }
+
+ private EventSchema makeExistingSchema() {
+ var timestamp = new EventPropertyPrimitive();
+ timestamp.setRuntimeName("timestamp");
+ timestamp.setRuntimeType(XSD.LONG.toString());
+ timestamp.setPropertyScope("HEADER_PROPERTY");
+ timestamp.setSemanticType(SO.DATE_TIME);
+
+ var temperature = new EventPropertyPrimitive();
+ temperature.setRuntimeName("temperature");
+ temperature.setRuntimeType(XSD.FLOAT.toString());
+ temperature.setPropertyScope("MEASUREMENT_PROPERTY");
+
+ return new EventSchema(List.of(timestamp, temperature));
+ }
+}
diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportParserTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportParserTest.java
new file mode 100644
index 0000000000..5857f4fe5c
--- /dev/null
+++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportParserTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.model.datalake.importer.CsvImportColumn;
+import org.apache.streampipes.model.datalake.importer.CsvImportConfiguration;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class CsvImportParserTest {
+
+ @TempDir
+ Path tempDir;
+
+ private final CsvImportParser parser = new CsvImportParser();
+
+ @Test
+ void shouldInferTimestampAndFloatColumns() {
+ var config = makeConfig(",", ".", true);
+ var headers = List.of("timestamp", "temperature", "active");
+ var rows = List.of(
+ List.of("1710000000000", "21.3", "true"),
+ List.of("1710000060000", "22.1", "false")
+ );
+
+ var columns = parser.inferColumns(headers, rows, config);
+
+ assertEquals("LONG", columns.get(0).getInferredType());
+ assertTrue(columns.get(0).isTimestampCandidate());
+ assertEquals("FLOAT", columns.get(1).getInferredType());
+ assertEquals("BOOLEAN", columns.get(2).getInferredType());
+ assertFalse(columns.get(1).isTimestampCandidate());
+ }
+
+ @Test
+ void shouldReadQuotedCsvRows() throws Exception {
+ var path = tempDir.resolve("quoted.csv");
+ Files.writeString(path, "timestamp,text\n1710000000000,\"a, b\"\n1710000060000,\"escaped \"\"quote\"\"\"\n");
+
+ var sample = parser.readCsvSample(path, makeConfig(",", ".", true), 10);
+
+ assertEquals(List.of("timestamp", "text"), sample.headers());
+ assertEquals("a, b", sample.rows().get(0).get(1));
+ assertEquals("escaped \"quote\"", sample.rows().get(1).get(1));
+ }
+
+ @Test
+ void shouldGenerateHeadersWhenCsvHasNoHeader() throws Exception {
+ var path = tempDir.resolve("no-header.csv");
+ Files.writeString(path, "1710000000000\t21.3\n1710000060000\t22.1\n");
+
+ var sample = parser.readCsvSample(path, makeConfig("\\t", ".", false), 10);
+
+ assertEquals(List.of("column_1", "column_2"), sample.headers());
+ assertEquals("1710000000000", sample.rows().get(0).get(0));
+ assertEquals("21.3", sample.rows().get(0).get(1));
+ }
+
+ @Test
+ void shouldRejectMissingTimestampValuesWhenConvertingRows() {
+ var request = new CsvImportRequest();
+ request.setCsvConfig(makeConfig(",", ".", true));
+ request.setTimestampColumn("timestamp");
+ request.setColumns(List.of(
+ makeColumn("timestamp", "timestamp", "LONG"),
+ makeColumn("temperature", "temperature", "FLOAT")
+ ));
+ request.setRows(List.of(
+ List.of("1710000000000", "21.3"),
+ List.of("", "22.1")
+ ));
+
+ var exception = assertThrows(CsvImportValidationException.class, () -> parser.toImportRows(request));
+
+ assertTrue(exception.getValidationMessages().get(0).getMessage().contains("missing a value for timestamp"));
+ }
+
+ private CsvImportConfiguration makeConfig(String delimiter, String decimalSeparator, boolean hasHeader) {
+ var config = new CsvImportConfiguration();
+ config.setDelimiter(delimiter);
+ config.setDecimalSeparator(decimalSeparator);
+ config.setHasHeader(hasHeader);
+ return config;
+ }
+
+ private CsvImportColumn makeColumn(String csvColumn, String runtimeName, String runtimeType) {
+ var column = new CsvImportColumn();
+ column.setCsvColumn(csvColumn);
+ column.setRuntimeName(runtimeName);
+ column.setRuntimeType(runtimeType);
+ return column;
+ }
+}
diff --git a/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationServiceTest.java b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationServiceTest.java
new file mode 100644
index 0000000000..d36a50a5e0
--- /dev/null
+++ b/streampipes-rest/src/test/java/org/apache/streampipes/rest/impl/datalake/importer/CsvImportValidationServiceTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.rest.impl.datalake.importer;
+
+import org.apache.streampipes.dataexplorer.api.IDataExplorerSchemaManagement;
+import org.apache.streampipes.model.datalake.DataLakeMeasure;
+import org.apache.streampipes.model.datalake.importer.CsvImportColumn;
+import org.apache.streampipes.model.datalake.importer.CsvImportConfiguration;
+import org.apache.streampipes.model.datalake.importer.CsvImportRequest;
+import org.apache.streampipes.model.datalake.importer.CsvImportSchemaIssueType;
+import org.apache.streampipes.model.datalake.importer.CsvImportTarget;
+import org.apache.streampipes.model.datalake.importer.CsvImportTargetMode;
+import org.apache.streampipes.model.schema.EventProperty;
+import org.apache.streampipes.model.schema.EventPropertyPrimitive;
+import org.apache.streampipes.model.schema.EventSchema;
+import org.apache.streampipes.vocabulary.SO;
+import org.apache.streampipes.vocabulary.XSD;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class CsvImportValidationServiceTest {
+
+ @Test
+ void shouldRequireUploadIdOrInlineRowsForStoredImportRequest() {
+ var service = new CsvImportValidationService(mock(IDataExplorerSchemaManagement.class));
+ var request = new CsvImportRequest();
+ request.setCsvConfig(makeConfig());
+ request.setTarget(makeTarget(CsvImportTargetMode.NEW, "new-measure"));
+ request.setTimestampColumn("timestamp");
+ request.setColumns(List.of(makeImportColumn("timestamp", "LONG", "HEADER_PROPERTY", SO.DATE_TIME)));
+
+ var result = service.validateStoredImportRequest(request);
+
+ assertTrue(result.stream().anyMatch(message -> "uploadId".equals(message.getField())));
+ }
+
+ @Test
+ void shouldRejectDuplicateMeasurementDuringPreviewValidation() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var existingMeasure = new DataLakeMeasure();
+ existingMeasure.setMeasureName("existing-measure");
+ when(schemaManagement.getExistingMeasureByName("existing-measure"))
+ .thenReturn(Optional.of(existingMeasure));
+
+ var service = new CsvImportValidationService(schemaManagement);
+ var result = service.validatePreviewTarget(makeTarget(CsvImportTargetMode.NEW, "existing-measure"));
+
+ assertTrue(result.stream().anyMatch(message -> message.getMessage().contains("already exists")));
+ }
+
+ @Test
+ void shouldReportSchemaIssuesForExistingMeasurement() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ var existingMeasure = new DataLakeMeasure();
+ existingMeasure.setMeasureName("existing-measure");
+ existingMeasure.setTimestampField("s0::timestamp");
+ existingMeasure.setEventSchema(new EventSchema(List.of(
+ makeEventProperty("timestamp", XSD.LONG.toString(), "HEADER_PROPERTY", SO.DATE_TIME),
+ makeEventProperty("temperature", XSD.FLOAT.toString(), "MEASUREMENT_PROPERTY", null)
+ )));
+ when(schemaManagement.getExistingMeasureByName("existing-measure"))
+ .thenReturn(Optional.of(existingMeasure));
+
+ var service = new CsvImportValidationService(schemaManagement);
+ var importSchema = new EventSchema(List.of(
+ makeEventProperty("event_time", XSD.LONG.toString(), "HEADER_PROPERTY", SO.DATE_TIME),
+ makeEventProperty("temperature", XSD.STRING.toString(), "DIMENSION_PROPERTY", null)
+ ));
+
+ var issues = service.validateSchemaTarget(
+ makeTarget(CsvImportTargetMode.EXISTING, "existing-measure"),
+ importSchema,
+ "event_time"
+ );
+
+ assertEquals(4, issues.size());
+ assertTrue(issues.stream().anyMatch(issue -> issue.getType() == CsvImportSchemaIssueType.TIMESTAMP_COLUMN_MISMATCH));
+ assertTrue(issues.stream().anyMatch(issue -> issue.getType() == CsvImportSchemaIssueType.COLUMN_NAME_MISMATCH));
+ assertTrue(issues.stream().anyMatch(issue -> issue.getType() == CsvImportSchemaIssueType.COLUMN_TYPE_MISMATCH));
+ assertTrue(issues.stream().anyMatch(issue -> issue.getType() == CsvImportSchemaIssueType.COLUMN_SCOPE_MISMATCH));
+ }
+
+ @Test
+ void shouldThrowWhenExistingMeasurementIsMissing() {
+ var schemaManagement = mock(IDataExplorerSchemaManagement.class);
+ when(schemaManagement.getExistingMeasureByName("missing-measure"))
+ .thenReturn(Optional.empty());
+
+ var service = new CsvImportValidationService(schemaManagement);
+
+ var exception = assertThrows(
+ CsvImportValidationException.class,
+ () -> service.requireExistingMeasurement("missing-measure")
+ );
+
+ assertFalse(exception.getValidationMessages().isEmpty());
+ assertTrue(exception.getValidationMessages().get(0).getMessage().contains("does not exist"));
+ }
+
+ private CsvImportConfiguration makeConfig() {
+ var config = new CsvImportConfiguration();
+ config.setDelimiter(",");
+ config.setDecimalSeparator(".");
+ config.setHasHeader(true);
+ return config;
+ }
+
+ private CsvImportTarget makeTarget(CsvImportTargetMode mode, String measurementName) {
+ var target = new CsvImportTarget();
+ target.setMode(mode);
+ target.setMeasurementName(measurementName);
+ return target;
+ }
+
+ private CsvImportColumn makeImportColumn(
+ String runtimeName,
+ String runtimeType,
+ String propertyScope,
+ String semanticType
+ ) {
+ var column = new CsvImportColumn();
+ column.setCsvColumn(runtimeName);
+ column.setRuntimeName(runtimeName);
+ column.setRuntimeType(runtimeType);
+ column.setPropertyScope(propertyScope);
+ column.setSemanticType(semanticType);
+ return column;
+ }
+
+ private EventProperty makeEventProperty(
+ String runtimeName,
+ String runtimeType,
+ String propertyScope,
+ String semanticType
+ ) {
+ var property = new EventPropertyPrimitive();
+ property.setRuntimeName(runtimeName);
+ property.setRuntimeType(runtimeType);
+ property.setPropertyScope(propertyScope);
+ property.setSemanticType(semanticType);
+ return property;
+ }
+}
diff --git a/ui/cypress/fixtures/datalake/machine-data-simulator-import-later-timestamps.csv b/ui/cypress/fixtures/datalake/machine-data-simulator-import-later-timestamps.csv
new file mode 100644
index 0000000000..ffebf58cdb
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/machine-data-simulator-import-later-timestamps.csv
@@ -0,0 +1,8 @@
+timestamp,density,mass_flow,sensorId,sensor_fault_flags,temperature,volume_flow
+1773332612874,45.09791564941406,5.38472843170166,flowrate01,false,48.357521057128906,7.121278285980225
+1773332613877,45.41450881958008,3.8461437225341797,flowrate01,false,45.560585021972656,2.3554582595825195
+1773332614881,43.88202667236328,8.558311462402344,flowrate01,false,48.41770935058594,10.315744400024414
+1773332615891,41.70826721191406,9.090323448181152,flowrate01,false,43.636566162109375,3.0096521377563477
+1773332616893,45.57435607910156,6.7500104904174805,flowrate01,false,46.06317138671875,5.4591851234436035
+1773332617898,50.77702331542969,0.3111061155796051,flowrate01,false,41.02153396606445,8.603775024414062
+1773332618901,41.054115295410156,3.339233160018921,flowrate01,false,47.03628158569336,9.04764461517334
diff --git a/ui/cypress/fixtures/datalake/machine-data-simulator-import-mismatched-timestamp.csv b/ui/cypress/fixtures/datalake/machine-data-simulator-import-mismatched-timestamp.csv
new file mode 100644
index 0000000000..bbdca87e6c
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/machine-data-simulator-import-mismatched-timestamp.csv
@@ -0,0 +1,8 @@
+event_time,density,mass_flow,sensorId,sensor_fault_flags,temperature,volume_flow
+1773332622874,45.09791564941406,5.38472843170166,flowrate01,false,48.357521057128906,7.121278285980225
+1773332623877,45.41450881958008,3.8461437225341797,flowrate01,false,45.560585021972656,2.3554582595825195
+1773332624881,43.88202667236328,8.558311462402344,flowrate01,false,48.41770935058594,10.315744400024414
+1773332625891,41.70826721191406,9.090323448181152,flowrate01,false,43.636566162109375,3.0096521377563477
+1773332626893,45.57435607910156,6.7500104904174805,flowrate01,false,46.06317138671875,5.4591851234436035
+1773332627898,50.77702331542969,0.3111061155796051,flowrate01,false,41.02153396606445,8.603775024414062
+1773332628901,41.054115295410156,3.339233160018921,flowrate01,false,47.03628158569336,9.04764461517334
diff --git a/ui/cypress/fixtures/datalake/machine-data-simulator-import-missing-values.csv b/ui/cypress/fixtures/datalake/machine-data-simulator-import-missing-values.csv
new file mode 100644
index 0000000000..8c3c0cce83
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/machine-data-simulator-import-missing-values.csv
@@ -0,0 +1,8 @@
+timestamp,dmc,druckoelraumAnlagenkompensation,druckoelraumBauteiltemperatur1,druckoelraumDifferenztemperatur,druckoelraumDrehzahl,druckoelraumDruck
+1772722801725,0.0,0.0,,0.0,0.0,0.0
+1772722802743,0,0.0,0.0,0.0,0.0,0.0
+1772722803756,0.0,,0.0,0.0,0.0,0.0
+1772722804768,0.0,0.0,,0.0,0.0,0.0
+1772722805777,0.0,0.0,,,0.0,0.0
+1772722806784,0.0,0.0,0.0,0.0,,0.0
+1772722807791,0.0,0.0,0.0,0.0,0.0,
diff --git a/ui/cypress/fixtures/datalake/machine-data-simulator-import-string-timestamp.csv b/ui/cypress/fixtures/datalake/machine-data-simulator-import-string-timestamp.csv
new file mode 100644
index 0000000000..f753413777
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/machine-data-simulator-import-string-timestamp.csv
@@ -0,0 +1,8 @@
+timestamp_string,density,mass_flow,sensorId,sensor_fault_flags,temperature,volume_flow
+2026-03-11 12:23:22,45.09791564941406,5.38472843170166,flowrate01,false,48.357521057128906,7.121278285980225
+2026-03-11 12:23:23,45.41450881958008,3.8461437225341797,flowrate01,false,45.560585021972656,2.3554582595825195
+2026-03-11 12:23:24,43.88202667236328,8.558311462402344,flowrate01,false,48.41770935058594,10.315744400024414
+2026-03-11 12:23:25,41.70826721191406,9.090323448181152,flowrate01,false,43.636566162109375,3.0096521377563477
+2026-03-11 12:23:26,45.57435607910156,6.7500104904174805,flowrate01,false,46.06317138671875,5.4591851234436035
+2026-03-11 12:23:27,50.77702331542969,0.3111061155796051,flowrate01,false,41.02153396606445,8.603775024414062
+2026-03-11 12:23:28,41.054115295410156,3.339233160018921,flowrate01,false,47.03628158569336,9.04764461517334
diff --git a/ui/cypress/fixtures/datalake/machine-data-simulator-import.csv b/ui/cypress/fixtures/datalake/machine-data-simulator-import.csv
new file mode 100644
index 0000000000..b78fdbe7e1
--- /dev/null
+++ b/ui/cypress/fixtures/datalake/machine-data-simulator-import.csv
@@ -0,0 +1,8 @@
+timestamp,density,mass_flow,sensorId,sensor_fault_flags,temperature,volume_flow
+1773332602874,45.09791564941406,5.38472843170166,flowrate01,false,48.357521057128906,7.121278285980225
+1773332603877,45.41450881958008,3.8461437225341797,flowrate01,false,45.560585021972656,2.3554582595825195
+1773332604881,43.88202667236328,8.558311462402344,flowrate01,false,48.41770935058594,10.315744400024414
+1773332605891,41.70826721191406,9.090323448181152,flowrate01,false,43.636566162109375,3.0096521377563477
+1773332606893,45.57435607910156,6.7500104904174805,flowrate01,false,46.06317138671875,5.4591851234436035
+1773332607898,50.77702331542969,0.3111061155796051,flowrate01,false,41.02153396606445,8.603775024414062
+1773332608901,41.054115295410156,3.339233160018921,flowrate01,false,47.03628158569336,9.04764461517334
diff --git a/ui/cypress/support/utils/PrepareTestDataUtils.ts b/ui/cypress/support/utils/PrepareTestDataUtils.ts
index be300a7932..da2bceaea2 100644
--- a/ui/cypress/support/utils/PrepareTestDataUtils.ts
+++ b/ui/cypress/support/utils/PrepareTestDataUtils.ts
@@ -16,10 +16,7 @@
*
*/
-import { FileManagementUtils } from './FileManagementUtils';
-import { ConnectUtils } from './connect/ConnectUtils';
-import { AdapterBuilder } from '../builder/AdapterBuilder';
-import { ConnectBtns } from './connect/ConnectBtns';
+import { DataLakeSeedUtils } from './dataset/DataLakeSeedUtils';
export class PrepareTestDataUtils {
public static dataName = 'prepared_data';
@@ -29,52 +26,25 @@ export class PrepareTestDataUtils {
format: 'csv' | 'json_array' = 'csv',
storeInDataLake: boolean = true,
) {
- // Create adapter with dataset
- FileManagementUtils.addFile(dataSet);
-
- const adapter = this.getDataLakeTestAdapter(
- PrepareTestDataUtils.dataName,
- format,
- storeInDataLake,
- );
-
- ConnectUtils.addAdapter(adapter);
-
- ConnectUtils.startAdapter(adapter, true);
- }
-
- private static getDataLakeTestAdapter(
- name: string,
- format: 'csv' | 'json_array',
- storeInDataLake: boolean = true,
- ) {
- const adapterBuilder = AdapterBuilder.create('File_Stream')
- .setName(name)
- .setTimestampProperty('timestamp')
- .addProtocolInput(
- 'radio',
- 'speed',
- 'fastest_\\(ignore_original_time\\)',
- )
- .addProtocolInput('radio', 'replayonce', 'yes');
+ if (!storeInDataLake) {
+ throw new Error(
+ 'Direct datalake test seeding only supports persisted datasets.',
+ );
+ }
if (format === 'csv') {
- adapterBuilder
- .setFormat('csv')
- .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
- .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check');
+ return DataLakeSeedUtils.importCsvFixture({
+ fixture: dataSet,
+ measurementName: PrepareTestDataUtils.dataName,
+ delimiter: ';',
+ timestampColumn: 'timestamp',
+ });
} else {
- adapterBuilder
- .setFormat('json')
- .addFormatInput('radio', 'json_options-array', '');
- }
-
- adapterBuilder.setStartAdapter(true);
-
- if (storeInDataLake) {
- adapterBuilder.setStoreInDataLake();
+ return DataLakeSeedUtils.importJsonArrayFixture({
+ fixture: dataSet,
+ measurementName: PrepareTestDataUtils.dataName,
+ timestampColumn: 'timestamp',
+ });
}
-
- return adapterBuilder.build();
}
}
diff --git a/ui/cypress/support/utils/chart/ChartUtils.ts b/ui/cypress/support/utils/chart/ChartUtils.ts
index 6bf716d333..8498b045be 100644
--- a/ui/cypress/support/utils/chart/ChartUtils.ts
+++ b/ui/cypress/support/utils/chart/ChartUtils.ts
@@ -20,13 +20,11 @@ import { DataLakeFilterConfig } from '../../model/DataLakeFilterConfig';
import { ChartWidget } from '../../model/ChartWidget';
import { DataSetUtils } from '../DataSetUtils';
import { PrepareTestDataUtils } from '../PrepareTestDataUtils';
-import { FileManagementUtils } from '../FileManagementUtils';
-import { ConnectUtils } from '../connect/ConnectUtils';
-import { ConnectBtns } from '../connect/ConnectBtns';
-import { AdapterBuilder } from '../../builder/AdapterBuilder';
import { GeneralUtils } from '../GeneralUtils';
import { ChartBtns } from './ChartBtns';
import { SharedBtns } from '../shared/SharedBtns';
+import { DataLakeSeedUtils } from '../dataset/DataLakeSeedUtils';
+import { ConnectBtns } from '../connect/ConnectBtns';
export class ChartUtils {
public static ADAPTER_NAME = 'datalake_configuration';
@@ -86,52 +84,29 @@ export class ChartUtils {
ChartUtils.loadRandomDataSetIntoDataLake();
}
- public static getDataLakeTestSetAdapter(
- name: string,
- storeInDataLake: boolean = true,
- format: 'csv' | 'json_array',
- ) {
- const adapterBuilder = AdapterBuilder.create('File_Stream')
- .setName(name)
- .setTimestampProperty('timestamp')
- .addDimensionProperty('randomtext')
- .addProtocolInput(
- 'radio',
- 'speed',
- 'fastest_\\(ignore_original_time\\)',
- )
- .setStartAdapter(true);
-
- if (format === 'csv') {
- adapterBuilder
- .setFormat('csv')
- .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
- .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check');
- } else {
- adapterBuilder.setFormat('json_array');
- }
-
- if (storeInDataLake) {
- adapterBuilder.setStoreInDataLake();
- }
- return adapterBuilder.build();
- }
-
public static loadDataIntoDataLake(
dataSet: string,
format: 'csv' | 'json_array' = 'csv',
) {
- // Create adapter with dataset
- FileManagementUtils.addFile(dataSet);
-
- const adapter = this.getDataLakeTestSetAdapter(
- ChartUtils.ADAPTER_NAME,
- true,
- format,
- );
-
- ConnectUtils.addAdapter(adapter);
- ConnectUtils.startAdapter(adapter);
+ if (format === 'csv') {
+ return DataLakeSeedUtils.importCsvFixture({
+ fixture: dataSet,
+ measurementName: ChartUtils.ADAPTER_NAME,
+ delimiter: ';',
+ timestampColumn: 'timestamp',
+ columnOverrides: {
+ randomtext: {
+ propertyScope: 'DIMENSION_PROPERTY',
+ },
+ },
+ });
+ } else {
+ return DataLakeSeedUtils.importJsonArrayFixture({
+ fixture: dataSet,
+ measurementName: ChartUtils.ADAPTER_NAME,
+ timestampColumn: 'timestamp',
+ });
+ }
}
public static addDataViewAndWidget(
@@ -304,6 +279,8 @@ export class ChartUtils {
public static createAndEditDataView() {
// Create new data view
ChartBtns.openNewDataViewBtn().click();
+ cy.location('hash').should('include', '/chart/create');
+ cy.location('hash').should('include', 'editMode=true');
}
public static removeWidget(dataViewName: string) {
@@ -431,11 +408,17 @@ export class ChartUtils {
}
public static selectDataSet(dataSet: string) {
- cy.dataCy('data-explorer-select-data-set')
- .click()
- .get('mat-option')
- .contains(dataSet)
- .click();
+ cy.get('body').then($body => {
+ if (
+ $body.find('[data-cy="data-explorer-select-data-set"]').length
+ ) {
+ cy.dataCy('data-explorer-select-data-set')
+ .click()
+ .get('mat-option')
+ .contains(dataSet)
+ .click();
+ }
+ });
}
public static assertSelectDataSet(dataSet: string) {
diff --git a/ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts b/ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts
index 1f71ce8d8f..96f2540001 100644
--- a/ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts
+++ b/ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts
@@ -17,10 +17,15 @@
*/
export class ChartWidgetTableUtils {
- public static chartTableRowTimestamp() {
- return cy.dataCy('data-explorer-table-row-timestamp', {
- timeout: 10000,
- });
+ public static chartTableRows() {
+ return cy
+ .dataCy('data-explorer-table', {
+ timeout: 10000,
+ })
+ .filter(':visible')
+ .first()
+ .find('tbody tr')
+ .not(':has(.table-empty-row)');
}
/**
@@ -28,6 +33,6 @@ export class ChartWidgetTableUtils {
* @param amount of expected rows
*/
public static checkAmountOfRows(amount: number) {
- this.chartTableRowTimestamp().should('have.length', amount);
+ this.chartTableRows().should('have.length', amount);
}
}
diff --git a/ui/cypress/support/utils/dataset/DataLakeSeedUtils.ts b/ui/cypress/support/utils/dataset/DataLakeSeedUtils.ts
new file mode 100644
index 0000000000..a81fbd6c6f
--- /dev/null
+++ b/ui/cypress/support/utils/dataset/DataLakeSeedUtils.ts
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import * as CSV from 'csv-string';
+
+type CsvRuntimeType = 'STRING' | 'BOOLEAN' | 'LONG' | 'FLOAT';
+type CsvImportTargetMode = 'NEW' | 'EXISTING';
+
+interface CsvImportConfiguration {
+ delimiter: string;
+ decimalSeparator: '.' | ',';
+ hasHeader: boolean;
+}
+
+interface CsvImportColumn {
+ csvColumn: string;
+ runtimeName: string;
+ runtimeType: CsvRuntimeType;
+ propertyScope?: string;
+ semanticType?: string;
+ inferredType?: CsvRuntimeType;
+ timestampCandidate?: boolean;
+}
+
+interface CsvImportTarget {
+ mode: CsvImportTargetMode;
+ measurementName: string;
+}
+
+interface CsvImportPreviewResult {
+ headers: string[];
+ previewRows: string[][];
+ columns: CsvImportColumn[];
+}
+
+interface CsvImportResult {
+ importedRowCount: number;
+ validationMessages: Array<{ field: string; message: string }>;
+}
+
+interface ColumnOverride {
+ runtimeName?: string;
+ runtimeType?: CsvRuntimeType;
+ propertyScope?: string;
+ semanticType?: string;
+}
+
+interface CsvFixtureImportOptions {
+ fixture: string;
+ measurementName: string;
+ delimiter?: string;
+ decimalSeparator?: '.' | ',';
+ timestampColumn?: string;
+ columnOverrides?: Record<string, ColumnOverride>;
+}
+
+interface JsonArrayFixtureImportOptions {
+ fixture: string;
+ measurementName: string;
+ timestampColumn?: string;
+ columnOverrides?: Record<string, ColumnOverride>;
+}
+
+interface ImportRequest {
+ csvConfig: CsvImportConfiguration;
+ headers: string[];
+ rows: string[][];
+ target: CsvImportTarget;
+ timestampColumn: string;
+ columns: CsvImportColumn[];
+}
+
+export class DataLakeSeedUtils {
+ private static readonly TIMESTAMP_SEMANTIC_TYPE =
+ 'http://schema.org/DateTime';
+
+ public static importCsvFixture(
+ options: CsvFixtureImportOptions,
+ ): Cypress.Chainable<CsvImportResult> {
+ const delimiter = options.delimiter ?? ';';
+ const decimalSeparator = options.decimalSeparator ?? '.';
+
+ return cy.fixture(options.fixture, 'utf8').then((content: string) => {
+ const parseCsv = CSV.parse as any;
+ const parsedCsv = parseCsv(content, delimiter);
+ const headers = parsedCsv[0];
+ const rows = parsedCsv.slice(1);
+ const timestampColumn = options.timestampColumn ?? headers[0];
+
+ return this.previewAndImport({
+ headers,
+ rows,
+ csvConfig: {
+ delimiter,
+ decimalSeparator,
+ hasHeader: true,
+ },
+ measurementName: options.measurementName,
+ timestampColumn,
+ columnOverrides: options.columnOverrides,
+ });
+ });
+ }
+
+ public static importJsonArrayFixture(
+ options: JsonArrayFixtureImportOptions,
+ ): Cypress.Chainable<CsvImportResult> {
+ return cy.fixture(options.fixture).then((records: Array<any>) => {
+ const headers = this.extractHeaders(records);
+ const rows = records.map(record =>
+ headers.map(header =>
+ this.serializeCell(record ? record[header] : undefined),
+ ),
+ );
+ const timestampColumn = options.timestampColumn ?? headers[0];
+
+ return this.previewAndImport({
+ headers,
+ rows,
+ csvConfig: {
+ delimiter: ';',
+ decimalSeparator: '.',
+ hasHeader: true,
+ },
+ measurementName: options.measurementName,
+ timestampColumn,
+ columnOverrides: options.columnOverrides,
+ });
+ });
+ }
+
+ private static previewAndImport(options: {
+ headers: string[];
+ rows: string[][];
+ csvConfig: CsvImportConfiguration;
+ measurementName: string;
+ timestampColumn: string;
+ columnOverrides?: Record<string, ColumnOverride>;
+ }): Cypress.Chainable<CsvImportResult> {
+ const token = window.localStorage.getItem('auth-token');
+ const target = {
+ mode: 'NEW' as CsvImportTargetMode,
+ measurementName: options.measurementName,
+ };
+
+ return cy
+ .request<CsvImportPreviewResult>({
+ method: 'POST',
+ url: '/streampipes-backend/api/v4/datalake/import/preview',
+ body: {
+ csvConfig: options.csvConfig,
+ headers: options.headers,
+ rows: options.rows,
+ target,
+ },
+ headers: {
+ Authorization: `Bearer ${token}`,
+ },
+ })
+ .then(previewResponse => {
+ const columns = this.buildColumns(
+ previewResponse.body.columns,
+ options.timestampColumn,
+ options.columnOverrides ?? {},
+ );
+
+ const request: ImportRequest = {
+ csvConfig: options.csvConfig,
+ headers: options.headers,
+ rows: options.rows,
+ target,
+ timestampColumn: options.timestampColumn,
+ columns,
+ };
+
+ return cy
+ .request<CsvImportResult>({
+ method: 'POST',
+ url: '/streampipes-backend/api/v4/datalake/import',
+ body: request,
+ headers: {
+ Authorization: `Bearer ${token}`,
+ },
+ })
+ .then(importResponse => {
+ expect(
+ importResponse.body.validationMessages,
+ 'import validation messages',
+ ).to.have.length(0);
+ expect(
+ importResponse.body.importedRowCount,
+ 'imported row count',
+ ).to.equal(options.rows.length);
+ return importResponse.body;
+ });
+ });
+ }
+
+ private static buildColumns(
+ previewColumns: CsvImportColumn[],
+ timestampColumn: string,
+ columnOverrides: Record<string, ColumnOverride>,
+ ) {
+ return previewColumns.map(previewColumn => {
+ const override = columnOverrides[previewColumn.csvColumn] ?? {};
+ if (previewColumn.csvColumn === timestampColumn) {
+ return {
+ ...previewColumn,
+ runtimeName:
+ override.runtimeName ?? previewColumn.runtimeName,
+ runtimeType: override.runtimeType ?? 'LONG',
+ propertyScope: 'HEADER_PROPERTY',
+ semanticType:
+ override.semanticType ??
+ DataLakeSeedUtils.TIMESTAMP_SEMANTIC_TYPE,
+ };
+ }
+
+ return {
+ ...previewColumn,
+ runtimeName: override.runtimeName ?? previewColumn.runtimeName,
+ runtimeType:
+ override.runtimeType ??
+ this.defaultRuntimeType(previewColumn.inferredType),
+ propertyScope: override.propertyScope ?? 'MEASUREMENT_PROPERTY',
+ semanticType: override.semanticType ?? undefined,
+ };
+ });
+ }
+
+ private static defaultRuntimeType(
+ inferredType: CsvRuntimeType = 'STRING',
+ ): CsvRuntimeType {
+ if (inferredType === 'LONG' || inferredType === 'FLOAT') {
+ return 'FLOAT';
+ }
+
+ return inferredType;
+ }
+
+ private static extractHeaders(records: Array<any>) {
+ const headers: string[] = [];
+ records.forEach(record => {
+ Object.keys(record || {}).forEach(key => {
+ if (headers.indexOf(key) === -1) {
+ headers.push(key);
+ }
+ });
+ });
+ return headers;
+ }
+
+ private static serializeCell(value: any): string {
+ if (value === undefined || value === null) {
+ return '';
+ }
+
+ return String(value);
+ }
+}
diff --git a/ui/cypress/support/utils/dataset/DatasetBtns.ts b/ui/cypress/support/utils/dataset/DatasetBtns.ts
index 36985d2541..670f5f34fb 100644
--- a/ui/cypress/support/utils/dataset/DatasetBtns.ts
+++ b/ui/cypress/support/utils/dataset/DatasetBtns.ts
@@ -20,4 +20,101 @@ export class DatasetBtns {
public static datasetTable() {
return cy.dataCy('datalake-settings', { timeout: 10000 });
}
+
+ public static openCsvImportDialog() {
+ return cy.dataCy('open-csv-import-dialog', { timeout: 10000 });
+ }
+
+ public static csvImportFileInput() {
+ return cy.dataCy('csv-import-file-input', { timeout: 10000 });
+ }
+
+ public static csvImportTargetMode() {
+ return cy.dataCy('csv-import-target-mode', { timeout: 10000 });
+ }
+
+ public static csvImportTargetModeNew() {
+ return cy.dataCy('csv-import-target-mode-new', { timeout: 10000 });
+ }
+
+ public static csvImportTargetModeExisting() {
+ return cy.dataCy('csv-import-target-mode-existing', { timeout: 10000 });
+ }
+
+ public static csvImportNewMeasurement() {
+ return cy.dataCy('csv-import-new-measurement', { timeout: 10000 });
+ }
+
+ public static csvImportExistingMeasurement() {
+ return cy.dataCy('csv-import-existing-measurement', { timeout: 10000 });
+ }
+
+ public static csvImportExistingMeasurementOption(measurementName: string) {
+ return cy.dataCy(
+ `csv-import-existing-measurement-option-${measurementName}`,
+ { timeout: 10000 },
+ );
+ }
+
+ public static csvImportDelimiter() {
+ return cy.dataCy('csv-import-delimiter', { timeout: 10000 });
+ }
+
+ public static csvImportDelimiterComma() {
+ return cy.dataCy('csv-import-delimiter-comma', { timeout: 10000 });
+ }
+
+ public static csvImportNextBtn() {
+ return cy.dataCy('csv-import-next-btn', { timeout: 10000 });
+ }
+
+ public static csvImportUploadBtn() {
+ return cy.dataCy('csv-import-upload-btn', { timeout: 10000 });
+ }
+
+ public static csvImportCloseBtn() {
+ return cy.dataCy('csv-import-close-btn', { timeout: 10000 });
+ }
+
+ public static csvImportSuccessTitle() {
+ return cy.dataCy('csv-import-success-title', { timeout: 10000 });
+ }
+
+ public static csvImportColumnScope(index: number) {
+ return cy
+ .dataCy('csv-import-column-scope', { timeout: 10000 })
+ .eq(index);
+ }
+
+ public static csvImportColumnScopeTimestamp() {
+ return cy.dataCy('csv-import-column-scope-timestamp', {
+ timeout: 10000,
+ });
+ }
+
+ public static csvImportTimestampFormat() {
+ return cy.dataCy('csv-import-timestamp-format', { timeout: 10000 });
+ }
+
+ public static csvImportSchemaMismatch() {
+ return cy.dataCy('csv-import-schema-mismatch', { timeout: 10000 });
+ }
+
+ public static csvImportSchemaMismatchList() {
+ return cy.dataCy('csv-import-schema-mismatch-list', {
+ timeout: 10000,
+ });
+ }
+
+ public static csvImportSchemaMismatchItems() {
+ return cy.dataCy('csv-import-schema-mismatch-item', {
+ timeout: 10000,
+ });
+ }
+
+ public static datasetRow(name: string) {
+ return cy.contains('[data-cy="datalake-settings"] tbody tr', name, {
+ timeout: 10000,
+ });
+ }
}
diff --git a/ui/cypress/support/utils/dataset/DatasetUtils.ts b/ui/cypress/support/utils/dataset/DatasetUtils.ts
index fbda2e8b20..f9513e29ce 100644
--- a/ui/cypress/support/utils/dataset/DatasetUtils.ts
+++ b/ui/cypress/support/utils/dataset/DatasetUtils.ts
@@ -20,6 +20,8 @@ import { PermissionUtils } from '../user/PermissionUtils';
import { DatasetBtns } from './DatasetBtns';
export class DatasetUtils {
+ private static readonly CSV_IMPORT_FIXTURE_PREFIX = 'cypress/fixtures/';
+
public static goToDatasets() {
cy.visit('#/datasets');
}
@@ -41,4 +43,110 @@ export class DatasetUtils {
DatasetUtils.goToDatasets();
PermissionUtils.authorizeUser(datasetname, email);
}
+
+ public static openCsvImportDialog() {
+ this.goToDatasets();
+ DatasetBtns.openCsvImportDialog().click();
+ }
+
+ public static uploadCsvImportFile(filePath: string) {
+ DatasetBtns.csvImportFileInput().selectFile(
+ this.CSV_IMPORT_FIXTURE_PREFIX + filePath,
+ { force: true },
+ );
+ }
+
+ public static createNewDatasetFromCsv(datasetName: string) {
+ DatasetBtns.csvImportTargetMode().click();
+ DatasetBtns.csvImportTargetModeNew().click();
+ DatasetBtns.csvImportNewMeasurement().clear().type(datasetName);
+ }
+
+ public static useExistingDatasetForCsvImport(datasetName: string) {
+ DatasetBtns.csvImportTargetMode().click();
+ DatasetBtns.csvImportTargetModeExisting().click();
+ DatasetBtns.csvImportExistingMeasurement().click();
+ DatasetBtns.csvImportExistingMeasurementOption(datasetName).click();
+ }
+
+ public static selectCsvImportDelimiterComma() {
+ DatasetBtns.csvImportDelimiter().click();
+ DatasetBtns.csvImportDelimiterComma().click();
+ cy.dataCy('csv-import-column-scope', { timeout: 10000 }).should(
+ 'have.length',
+ 7,
+ );
+ }
+
+ public static continueCsvImportToPreview() {
+ DatasetBtns.csvImportNextBtn().click();
+ cy.dataCy('csv-import-preview', { timeout: 10000 }).should(
+ 'be.visible',
+ );
+ }
+
+ public static selectCsvImportTimestampColumn(columnIndex: number) {
+ DatasetBtns.csvImportColumnScope(columnIndex).click();
+ DatasetBtns.csvImportColumnScopeTimestamp().click();
+ }
+
+ public static setCsvImportTimestampFormat(format: string) {
+ DatasetBtns.csvImportTimestampFormat().clear().type(format);
+ }
+
+ public static uploadCsvImport() {
+ DatasetBtns.csvImportUploadBtn().click();
+ DatasetBtns.csvImportSuccessTitle().should('be.visible');
+ DatasetBtns.csvImportCloseBtn().click();
+ }
+
+ public static expectCsvImportSchemaMismatch(
+ summary: string,
+ detailText?: string,
+ ) {
+ DatasetBtns.csvImportSchemaMismatch().should('be.visible');
+ cy.dataCy('exception-message-title').should('contain.text', summary);
+ if (detailText) {
+ DatasetBtns.csvImportSchemaMismatchList().should('be.visible');
+ DatasetBtns.csvImportSchemaMismatchItems().should(
+ 'contain.text',
+ detailText,
+ );
+ }
+ }
+
+ public static expectDatasetTotalEventCount(
+ datasetName: string,
+ expectedCount: string,
+ ) {
+ DatasetBtns.datasetRow(datasetName)
+ .should('be.visible')
+ .find('[data-cy="datalake-total-count-button"]')
+ .then($button => {
+ if ($button.length > 0) {
+ cy.wrap($button).click({ force: true });
+ }
+ });
+
+ DatasetBtns.datasetRow(datasetName)
+ .find('[data-cy="datalake-number-of-events"]', {
+ timeout: 10000,
+ })
+ .should($element => {
+ const text = $element.text().trim();
+ expect(text).to.equal(expectedCount);
+ });
+ }
+
+ public static expectDatasetSevenDayEventCount(
+ datasetName: string,
+ expectedCount: string,
+ ) {
+ DatasetBtns.datasetRow(datasetName)
+ .should('be.visible')
+ .should($row => {
+ const text = $row.children().eq(2).text().trim();
+ expect(text).to.contain(expectedCount);
+ });
+ }
}
diff --git a/ui/cypress/tests/chart/configuration.smoke.spec.ts b/ui/cypress/tests/chart/configuration.smoke.spec.ts
index 9080a80515..a09a831321 100644
--- a/ui/cypress/tests/chart/configuration.smoke.spec.ts
+++ b/ui/cypress/tests/chart/configuration.smoke.spec.ts
@@ -16,7 +16,6 @@
*
*/
-import { PipelineUtils } from '../../support/utils/pipeline/PipelineUtils';
import { ChartUtils } from '../../support/utils/chart/ChartUtils';
import { ChartBtns } from '../../support/utils/chart/ChartBtns';
import { GeneralUtils } from '../../support/utils/GeneralUtils';
@@ -56,7 +55,6 @@ describe('Delete data in datalake', () => {
before('Setup Test', () => {
cy.initStreamPipesTest();
ChartUtils.loadRandomDataSetIntoDataLake();
- PipelineUtils.deletePipeline('Persist prepared_data');
});
it('Perform Test', () => {
diff --git a/ui/cypress/tests/chart/filterNumericalStringProperties.spec.ts b/ui/cypress/tests/chart/filterNumericalStringProperties.spec.ts
index 2bd393488b..f09f65a856 100644
--- a/ui/cypress/tests/chart/filterNumericalStringProperties.spec.ts
+++ b/ui/cypress/tests/chart/filterNumericalStringProperties.spec.ts
@@ -19,30 +19,24 @@
import { ChartUtils } from '../../support/utils/chart/ChartUtils';
import { ChartWidgetTableUtils } from '../../support/utils/chart/ChartWidgetTableUtils';
import { DataLakeFilterConfig } from '../../support/model/DataLakeFilterConfig';
-import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
-import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
-import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
-import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
import { ChartWidget } from '../../support/model/ChartWidget';
+import { DataLakeSeedUtils } from '../../support/utils/dataset/DataLakeSeedUtils';
describe('Validate that filter works for numerical dimension property', () => {
beforeEach('Setup Test', () => {
cy.initStreamPipesTest();
-
- FileManagementUtils.addFile(
- 'datalake/filterNumericalStringProperties.csv',
- );
- const adapterInput = AdapterBuilder.create('File_Stream')
- .setName('Test Adapter')
- .setTimestampProperty('timestamp')
- .addDataTypeChange('dimensionKey', 'Integer')
- .addDimensionProperty('dimensionKey')
- .setStoreInDataLake()
- .setFormat('csv')
- .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
- .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check')
- .build();
- ConnectUtils.testAdapter(adapterInput);
+ DataLakeSeedUtils.importCsvFixture({
+ fixture: 'datalake/filterNumericalStringProperties.csv',
+ measurementName: 'Test Adapter',
+ delimiter: ';',
+ timestampColumn: 'timestamp',
+ columnOverrides: {
+ dimensionKey: {
+ runtimeType: 'LONG',
+ propertyScope: 'DIMENSION_PROPERTY',
+ },
+ },
+ });
});
it('Perform Test', () => {
diff --git a/ui/cypress/tests/dataset/csvImport.spec.ts b/ui/cypress/tests/dataset/csvImport.spec.ts
new file mode 100644
index 0000000000..f7ff816887
--- /dev/null
+++ b/ui/cypress/tests/dataset/csvImport.spec.ts
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { DatasetUtils } from '../../support/utils/dataset/DatasetUtils';
+
+describe('CSV import happy path', () => {
+ const datasetName = 'csv_machine_data_import';
+ const stringTimestampDatasetName = 'csv_machine_data_import_string_ts';
+ const existingDatasetName = 'csv_machine_data_existing_import';
+ const missingValuesDatasetName = 'csv_machine_data_missing_values';
+
+ beforeEach('Setup Test', () => {
+ cy.initStreamPipesTest();
+ });
+
+ it('Uploads a CSV file into a new dataset and shows the imported event count', () => {
+ DatasetUtils.openCsvImportDialog();
+ DatasetUtils.uploadCsvImportFile(
+ 'datalake/machine-data-simulator-import.csv',
+ );
+ DatasetUtils.createNewDatasetFromCsv(datasetName);
+ DatasetUtils.continueCsvImportToPreview();
+ DatasetUtils.selectCsvImportDelimiterComma();
+ DatasetUtils.selectCsvImportTimestampColumn(0);
+ DatasetUtils.uploadCsvImport();
+ DatasetUtils.expectDatasetTotalEventCount(datasetName, '7');
+ });
+
+ it('Uploads a CSV file with string timestamps and transforms them during import', () => {
+ DatasetUtils.openCsvImportDialog();
+ DatasetUtils.uploadCsvImportFile(
+ 'datalake/machine-data-simulator-import-string-timestamp.csv',
+ );
+ DatasetUtils.createNewDatasetFromCsv(stringTimestampDatasetName);
+ DatasetUtils.continueCsvImportToPreview();
+ DatasetUtils.selectCsvImportDelimiterComma();
+ DatasetUtils.selectCsvImportTimestampColumn(0);
+ DatasetUtils.setCsvImportTimestampFormat('yyyy-MM-dd HH:mm:ss');
+ DatasetUtils.uploadCsvImport();
+ DatasetUtils.expectDatasetTotalEventCount(
+ stringTimestampDatasetName,
+ '7',
+ );
+ });
+
+ it('Uploads a CSV file with missing values and still imports all rows', () => {
+ DatasetUtils.openCsvImportDialog();
+ DatasetUtils.uploadCsvImportFile(
+ 'datalake/machine-data-simulator-import-missing-values.csv',
+ );
+ DatasetUtils.createNewDatasetFromCsv(missingValuesDatasetName);
+ DatasetUtils.continueCsvImportToPreview();
+ DatasetUtils.selectCsvImportDelimiterComma();
+ DatasetUtils.selectCsvImportTimestampColumn(0);
+ DatasetUtils.uploadCsvImport();
+ DatasetUtils.expectDatasetTotalEventCount(
+ missingValuesDatasetName,
+ '7',
+ );
+ });
+
+ it('Appends matching data to an existing dataset and warns on mismatched timestamp schema', () => {
+ DatasetUtils.openCsvImportDialog();
+ DatasetUtils.uploadCsvImportFile(
+ 'datalake/machine-data-simulator-import.csv',
+ );
+ DatasetUtils.createNewDatasetFromCsv(existingDatasetName);
+ DatasetUtils.continueCsvImportToPreview();
+ DatasetUtils.selectCsvImportDelimiterComma();
+ DatasetUtils.selectCsvImportTimestampColumn(0);
+ DatasetUtils.uploadCsvImport();
+ DatasetUtils.expectDatasetSevenDayEventCount(existingDatasetName, '7');
+
+ DatasetUtils.openCsvImportDialog();
+ DatasetUtils.uploadCsvImportFile(
+ 'datalake/machine-data-simulator-import-later-timestamps.csv',
+ );
+ DatasetUtils.useExistingDatasetForCsvImport(existingDatasetName);
+ DatasetUtils.continueCsvImportToPreview();
+ DatasetUtils.selectCsvImportDelimiterComma();
+ DatasetUtils.selectCsvImportTimestampColumn(0);
+ DatasetUtils.uploadCsvImport();
+ DatasetUtils.expectDatasetSevenDayEventCount(existingDatasetName, '14');
+
+ DatasetUtils.openCsvImportDialog();
+ DatasetUtils.uploadCsvImportFile(
+ 'datalake/machine-data-simulator-import-mismatched-timestamp.csv',
+ );
+ DatasetUtils.useExistingDatasetForCsvImport(existingDatasetName);
+ DatasetUtils.continueCsvImportToPreview();
+ DatasetUtils.selectCsvImportDelimiterComma();
+ DatasetUtils.selectCsvImportTimestampColumn(0);
+ DatasetUtils.expectCsvImportSchemaMismatch(
+ 'Imported columns must exactly match the existing measurement schema.',
+ 'Timestamp column must be "timestamp" but is "event_time".',
+ );
+ });
+});
diff --git a/ui/cypress/tests/userManagement/testUserRoleDashboard.spec.ts b/ui/cypress/tests/userManagement/testUserRoleDashboard.spec.ts
index 1c904cab62..a572c676f2 100644
--- a/ui/cypress/tests/userManagement/testUserRoleDashboard.spec.ts
+++ b/ui/cypress/tests/userManagement/testUserRoleDashboard.spec.ts
@@ -18,14 +18,15 @@
import { UserRole } from '../../../src/app/core/auth/user-role.enum';
import { UserUtils } from '../../support/utils/UserUtils';
-import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
import { User } from '../../support/model/User';
import { ChartUtils } from '../../support/utils/chart/ChartUtils';
import { PermissionUtils } from '../../support/utils/user/PermissionUtils';
import { ChartBtns } from '../../support/utils/chart/ChartBtns';
+import { DataLakeSeedUtils } from '../../support/utils/dataset/DataLakeSeedUtils';
describe('Test User Roles for Dashboards', () => {
const dashboardName = 'test-dashboard';
+ const datasetName = 'simulator';
let dashboardUser1: User;
let dashboardAdmin1: User;
let dashboardAdmin2: User;
@@ -123,7 +124,12 @@ describe('Test User Roles for Dashboards', () => {
it('Test Dashboard and Charts Permissions', () => {
UserUtils.switchUser(dashboardAdmin1);
- ConnectUtils.addMachineDataSimulator('simulator', true);
+ DataLakeSeedUtils.importCsvFixture({
+ fixture: 'datalake/machine-data-simulator-import.csv',
+ measurementName: datasetName,
+ delimiter: ',',
+ timestampColumn: 'timestamp',
+ });
addChart('chart1');
cy.wait(1000);
addChart('chart2');
@@ -194,7 +200,7 @@ describe('Test User Roles for Dashboards', () => {
}
function addChart(chartName: string) {
- ChartUtils.addDataViewAndTableWidget(chartName, 'simulator', true);
+ ChartUtils.addDataViewAndTableWidget(chartName, datasetName, true);
ChartUtils.saveDataViewConfiguration();
}
});
diff --git a/ui/cypress/tests/userManagement/testUserRoleDataset.spec.ts b/ui/cypress/tests/userManagement/testUserRoleDataset.spec.ts
index a3aa42126f..b1ad06104a 100644
--- a/ui/cypress/tests/userManagement/testUserRoleDataset.spec.ts
+++ b/ui/cypress/tests/userManagement/testUserRoleDataset.spec.ts
@@ -18,16 +18,16 @@
import { UserRole } from '../../../src/app/core/auth/user-role.enum';
import { UserUtils } from '../../support/utils/UserUtils';
-import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
import { User } from '../../support/model/User';
import { ChartUtils } from '../../support/utils/chart/ChartUtils';
import { PermissionUtils } from '../../support/utils/user/PermissionUtils';
import { ChartBtns } from '../../support/utils/chart/ChartBtns';
import { DatasetUtils } from '../../support/utils/dataset/DatasetUtils';
import { GeneralUtils } from '../../support/utils/GeneralUtils';
+import { DataLakeSeedUtils } from '../../support/utils/dataset/DataLakeSeedUtils';
describe('Test Dataset Permissions', () => {
- const datasetName = 'Persist simulator';
+ const datasetName = 'simulator';
let datasetUser1: User;
let datasetAdmin1: User;
let datasetAdmin2: User;
@@ -171,15 +171,20 @@ describe('Test Dataset Permissions', () => {
if (!available) {
cy.get('sp-alert-banner').should('be.visible');
} else {
- ChartUtils.assertSelectDataSet('simulator');
- ChartUtils.addDataViewAndTableWidget('test', 'simulator', true);
+ ChartUtils.assertSelectDataSet(datasetName);
+ ChartUtils.addDataViewAndTableWidget('test', datasetName, true);
ChartUtils.saveDataViewConfiguration();
}
}
function generateDataset() {
UserUtils.switchUser(datasetAdmin1);
- ConnectUtils.addMachineDataSimulator('simulator', true);
+ DataLakeSeedUtils.importCsvFixture({
+ fixture: 'datalake/machine-data-simulator-import.csv',
+ measurementName: datasetName,
+ delimiter: ',',
+ timestampColumn: 'timestamp',
+ });
}
function generateDashboard(name: string) {
@@ -212,6 +217,6 @@ describe('Test Dataset Permissions', () => {
function authUserOnDataset(email: string) {
UserUtils.switchUser(datasetAdmin1);
- DatasetUtils.authorizeUserOnDataset('simulator', email);
+ DatasetUtils.authorizeUserOnDataset(datasetName, email);
}
});
diff --git a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
index c7f8aeec7a..4413701820 100644
--- a/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
+++ b/ui/projects/streampipes/platform-services/src/lib/apis/datalake-rest.service.ts
@@ -29,6 +29,14 @@ import { DataLakeMeasure, SpQueryResult } from '../model/gen/streampipes-model';
import { map } from 'rxjs/operators';
import { DatalakeQueryParameters } from '../model/datalake/DatalakeQueryParameters';
import { NGX_LOADING_BAR_IGNORED } from '@ngx-loading-bar/http-client';
+import {
+ CsvImportPreviewRequest,
+ CsvImportPreviewResult,
+ CsvImportRequest,
+ CsvImportResult,
+ CsvImportSchemaValidationRequest,
+ CsvImportSchemaValidationResult,
+} from '../model/datalake/csv-import.model';
@Injectable({
providedIn: 'root',
@@ -48,6 +56,10 @@ export class DatalakeRestService {
return this.baseUrl + '/api/v4/datalake/measure';
}
+ public get dataLakeImportUrl() {
+ return this.baseUrl + '/api/v4/datalake/import';
+ }
+
getMeasurementEntryCounts(
measurementNames: string[],
daysBack = -1,
@@ -233,6 +245,45 @@ export class DatalakeRestService {
return this.http.delete(url);
}
+ previewImport(
+ request: CsvImportPreviewRequest,
+ file?: File,
+ ): Observable<CsvImportPreviewResult> {
+ if (file) {
+ const formData = new FormData();
+ formData.append('file', file, file.name);
+ formData.append(
+ 'request',
+ new Blob([JSON.stringify(request)], {
+ type: 'application/json',
+ }),
+ );
+
+ return this.http.post<CsvImportPreviewResult>(
+ `${this.dataLakeImportUrl}/preview`,
+ formData,
+ );
+ }
+
+ return this.http.post<CsvImportPreviewResult>(
+ `${this.dataLakeImportUrl}/preview`,
+ request,
+ );
+ }
+
+ validateImportSchema(
+ request: CsvImportSchemaValidationRequest,
+ ): Observable<CsvImportSchemaValidationResult> {
+ return this.http.post<CsvImportSchemaValidationResult>(
+ `${this.dataLakeImportUrl}/validate-schema`,
+ request,
+ );
+ }
+
+ importCsvData(request: CsvImportRequest): Observable<CsvImportResult> {
+ return this.http.post<CsvImportResult>(this.dataLakeImportUrl, request);
+ }
+
dropSingleMeasurementSeries(index: string) {
const url = this.dataLakeUrl + '/measurements/' + index + '/drop';
return this.http.delete(url);
diff --git a/ui/projects/streampipes/platform-services/src/lib/model/datalake/csv-import.model.ts b/ui/projects/streampipes/platform-services/src/lib/model/datalake/csv-import.model.ts
new file mode 100644
index 0000000000..98edf75c48
--- /dev/null
+++ b/ui/projects/streampipes/platform-services/src/lib/model/datalake/csv-import.model.ts
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { EventSchema } from '../gen/streampipes-model';
+
+export type CsvImportTargetMode = 'NEW' | 'EXISTING';
+export type CsvRuntimeType = 'STRING' | 'BOOLEAN' | 'LONG' | 'FLOAT';
+
+export interface CsvImportConfiguration {
+ delimiter: string;
+ decimalSeparator: '.' | ',';
+ hasHeader: boolean;
+ timestampFormat?: string;
+}
+
+export interface CsvImportTarget {
+ mode: CsvImportTargetMode;
+ measurementName: string;
+}
+
+export interface CsvImportColumn {
+ csvColumn: string;
+ runtimeName: string;
+ runtimeType: CsvRuntimeType;
+ propertyScope?: string;
+ semanticType?: string;
+ label?: string;
+ description?: string;
+ inferredType?: CsvRuntimeType;
+ timestampCandidate?: boolean;
+}
+
+export interface CsvImportValidationMessage {
+ field: string;
+ message: string;
+}
+
+export interface CsvImportPreviewRequest {
+ uploadId?: string;
+ fileName?: string;
+ csvConfig: CsvImportConfiguration;
+ headers?: string[];
+ rows?: string[][];
+ target?: CsvImportTarget;
+}
+
+export interface CsvImportPreviewResult {
+ uploadId?: string;
+ headers: string[];
+ previewRows: string[][];
+ columns: CsvImportColumn[];
+ guessedEventSchema: EventSchema;
+ timestampCandidates: string[];
+ valid: boolean;
+ validationMessages: CsvImportValidationMessage[];
+}
+
+export interface CsvImportSchemaValidationRequest {
+ target: CsvImportTarget;
+ timestampColumn: string;
+ columns: CsvImportColumn[];
+}
+
+export type CsvImportSchemaIssueType =
+ | 'TIMESTAMP_COLUMN_MISMATCH'
+ | 'COLUMN_NAME_MISMATCH'
+ | 'COLUMN_TYPE_MISMATCH'
+ | 'COLUMN_SCOPE_MISMATCH';
+
+export interface CsvImportSchemaIssue {
+ type: CsvImportSchemaIssueType;
+ columnName?: string;
+ expected?: string | null;
+ actual?: string | null;
+}
+
+export interface CsvImportSchemaValidationResult {
+ valid: boolean;
+ validationMessages: CsvImportValidationMessage[];
+ issues: CsvImportSchemaIssue[];
+}
+
+export interface CsvImportRequest {
+ uploadId?: string;
+ csvConfig: CsvImportConfiguration;
+ headers?: string[];
+ rows?: string[][];
+ target: CsvImportTarget;
+ timestampColumn: string;
+ columns: CsvImportColumn[];
+}
+
+export interface CsvImportResult {
+ measurementName: string;
+ measurementId: string;
+ createdNewMeasurement: boolean;
+ importedRowCount: number;
+ validationMessages: CsvImportValidationMessage[];
+}
diff --git a/ui/projects/streampipes/platform-services/src/public-api.ts b/ui/projects/streampipes/platform-services/src/public-api.ts
index 3754b801ed..732732d5d7 100644
--- a/ui/projects/streampipes/platform-services/src/public-api.ts
+++ b/ui/projects/streampipes/platform-services/src/public-api.ts
@@ -62,6 +62,7 @@ export * from './lib/apis/export-provider.service';
export * from './lib/model/datalake/DateRange';
export * from './lib/model/datalake/DatalakeQueryParameters';
+export * from './lib/model/datalake/csv-import.model';
export * from './lib/model/dashboard/dashboard.model';
export * from './lib/model/config/email-config.model';
export * from './lib/model/config/general-config.model';
diff --git a/ui/projects/streampipes/shared-ui/src/lib/dialog/data-download-dialog/data-download-dialog.component.html b/ui/projects/streampipes/shared-ui/src/lib/dialog/data-download-dialog/data-download-dialog.component.html
index 723a79fbc3..ebd0780566 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/dialog/data-download-dialog/data-download-dialog.component.html
+++ b/ui/projects/streampipes/shared-ui/src/lib/dialog/data-download-dialog/data-download-dialog.component.html
@@ -53,7 +53,7 @@
</div>
<mat-divider></mat-divider>
- <div class="sp-dialog-actions actions-align-right">
+ <div class="sp-dialog-actions actions-align-left">
<button
mat-button
mat-flat-button
diff --git a/ui/projects/streampipes/shared-ui/src/lib/dialog/standard-dialog/standard-dialog.component.scss b/ui/projects/streampipes/shared-ui/src/lib/dialog/standard-dialog/standard-dialog.component.scss
index 4af62bebaf..9ce08157de 100644
--- a/ui/projects/streampipes/shared-ui/src/lib/dialog/standard-dialog/standard-dialog.component.scss
+++ b/ui/projects/streampipes/shared-ui/src/lib/dialog/standard-dialog/standard-dialog.component.scss
@@ -47,7 +47,8 @@ standard-dialog-container {
.dialog-panel-content {
height: 100%;
- overflow-y: auto;
+ overflow: auto;
+ min-width: 0;
}
.dialog-title {
diff --git a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
index db6509a8b5..1c4f55be91 100644
--- a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
+++ b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.html
@@ -28,6 +28,15 @@
[title]="'Datasets' | translate"
></sp-basic-header-title-component>
<span fxFlex></span>
+ <button
+ *ngIf="writeAccess"
+ color="accent"
+ mat-flat-button
+ data-cy="open-csv-import-dialog"
+ (click)="openCsvImportDialog()"
+ >
+ {{ 'Import CSV' | translate }}
+ </button>
<button
color="accent"
mat-icon-button
diff --git a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
index 8e8e8b61b0..7393d27797 100644
--- a/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
+++ b/ui/src/app/dataset/components/datalake-configuration/datalake-configuration.component.ts
@@ -73,6 +73,7 @@ import { ExportProviderConnectionTestComponent } from '../../dialog/export-provi
import { DataRetentionLogDialogComponent } from '../../dialog/data-retention-log-dialog/data-retention-log-dialog.component';
import { UserPrivilege } from '../../../core/auth/user-privilege.enum';
import { UserRole } from '../../../core/auth/user-role.enum';
+import { CsvImportDialogComponent } from '../../dialog/csv-import-dialog/csv-import-dialog.component';
import {
FlexDirective,
FlexOrderDirective,
@@ -443,6 +444,29 @@ export class DatalakeConfigurationComponent implements OnInit, AfterViewInit {
});
}
+ openCsvImportDialog() {
+ const dialogRef: DialogRef<CsvImportDialogComponent> =
+ this.dialogService.open(CsvImportDialogComponent, {
+ panelType: PanelType.SLIDE_IN_PANEL,
+ title: this.translateService.instant('Import CSV'),
+ width: '60vw',
+ data: {
+ measurementNames: this.availableMeasurements.map(
+ measurement => measurement.name,
+ ),
+ },
+ });
+
+ dialogRef.afterClosed().subscribe(refresh => {
+ const importCompleted =
+ dialogRef.componentInstance?.instance?.hasImportResult?.() ===
+ true;
+ if (refresh || importCompleted) {
+ this.loadAvailableMeasurements();
+ }
+ });
+ }
+
queryEntryCounts(
measurements: string[],
targetField: string,
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.html b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.html
new file mode 100644
index 0000000000..9ba8fea63c
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.html
@@ -0,0 +1,324 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div class="sp-dialog-container csv-import-shell">
+ <div class="sp-dialog-content p-15 csv-import-content">
+ @if (topMessages().length > 0) {
+ <div class="message-box" data-cy="csv-import-messages">
+ @for (message of topMessages(); track message.message) {
+ <div class="message-line">{{ message.message }}</div>
+ }
+ </div>
+ }
+
+ <mat-horizontal-stepper #csvImportStepper>
+ <mat-step>
+ <sp-split-section
+ [level]="3"
+ [title]="'Upload CSV file' | translate"
+ >
+ <div class="upload-row">
+ <label
+ class="file-label"
+ data-cy="csv-import-file-label"
+ >
+ <span>{{
+ fileName() || ('Select CSV file' | translate)
+ }}</span>
+ <input
+ type="file"
+ accept=".csv,text/csv"
+ (change)="onFileSelected($event)"
+ data-cy="csv-import-file-input"
+ />
+ </label>
+ @if (fileName()) {
+ <div class="file-name">{{ fileName() }}</div>
+ }
+ </div>
+ </sp-split-section>
+
+ <sp-split-section
+ [level]="3"
+ [title]="'Choose target dataset' | translate"
+ >
+ <div class="grid">
+ <sp-form-field
+ [level]="3"
+ [label]="'Import target' | translate"
+ >
+ <mat-form-field appearance="outline" class="w-100">
+ <mat-select
+ [formControl]="targetForm.controls.mode"
+ data-cy="csv-import-target-mode"
+ >
+ <mat-option
+ value="NEW"
+ data-cy="csv-import-target-mode-new"
+ >
+ {{ 'Create new dataset' | translate }}
+ </mat-option>
+ <mat-option
+ value="EXISTING"
+ data-cy="csv-import-target-mode-existing"
+ >
+ {{ 'Use existing dataset' | translate }}
+ </mat-option>
+ </mat-select>
+ </mat-form-field>
+ </sp-form-field>
+
+ @if (targetMode() === 'NEW') {
+ <sp-form-field
+ [level]="3"
+ [label]="'Dataset name' | translate"
+ >
+ <mat-form-field
+ appearance="outline"
+ class="w-100"
+ >
+ <input
+ matInput
+ [formControl]="
+ targetForm.controls
+ .newMeasurementName
+ "
+ data-cy="csv-import-new-measurement"
+ />
+ </mat-form-field>
+ </sp-form-field>
+ } @else {
+ <sp-form-field
+ [level]="3"
+ [label]="'Existing dataset' | translate"
+ >
+ <mat-form-field
+ appearance="outline"
+ class="w-100"
+ >
+ <mat-select
+ [formControl]="
+ targetForm.controls
+ .existingMeasurementName
+ "
+ data-cy="csv-import-existing-measurement"
+ >
+ @for (
+ measurementName of measurementNames;
+ track measurementName
+ ) {
+ <mat-option
+ [value]="measurementName"
+ [attr.data-cy]="
+ 'csv-import-existing-measurement-option-' +
+ measurementName
+ "
+ >
+ {{ measurementName }}
+ </mat-option>
+ }
+ </mat-select>
+ </mat-form-field>
+ </sp-form-field>
+ }
+ </div>
+ </sp-split-section>
+
+ <ng-template matStepLabel>{{
+ 'Input' | translate
+ }}</ng-template>
+ </mat-step>
+
+ <mat-step>
+ <sp-split-section
+ [level]="3"
+ [title]="'CSV configuration' | translate"
+ >
+ @if (previewLoading()) {
+ <div section-actions>
+ <mat-spinner [diameter]="24"></mat-spinner>
+ </div>
+ }
+
+ <div class="grid">
+ <sp-form-field
+ [level]="3"
+ [label]="'Delimiter' | translate"
+ >
+ <mat-form-field appearance="outline" class="w-100">
+ <mat-select
+ [formControl]="parseForm.controls.delimiter"
+ data-cy="csv-import-delimiter"
+ >
+ <mat-option
+ value=";"
+ data-cy="csv-import-delimiter-semicolon"
+ >;</mat-option
+ >
+ <mat-option
+ value=","
+ data-cy="csv-import-delimiter-comma"
+ >,</mat-option
+ >
+ <mat-option
+ value="|"
+ data-cy="csv-import-delimiter-pipe"
+ >|</mat-option
+ >
+ <mat-option
+ value="\t"
+ data-cy="csv-import-delimiter-tab"
+ >Tab</mat-option
+ >
+ </mat-select>
+ </mat-form-field>
+ </sp-form-field>
+
+ <sp-form-field
+ [level]="3"
+ [label]="'Decimal separator' | translate"
+ >
+ <mat-form-field appearance="outline" class="w-100">
+ <mat-select
+ [formControl]="
+ parseForm.controls.decimalSeparator
+ "
+ data-cy="csv-import-decimal-separator"
+ >
+ <mat-option
+ value=","
+ data-cy="csv-import-decimal-separator-comma"
+ >,</mat-option
+ >
+ <mat-option
+ value="."
+ data-cy="csv-import-decimal-separator-dot"
+ >.</mat-option
+ >
+ </mat-select>
+ </mat-form-field>
+ </sp-form-field>
+ </div>
+
+ <mat-checkbox
+ [formControl]="parseForm.controls.hasHeader"
+ data-cy="csv-import-has-header"
+ >
+ {{ 'First row contains headers' | translate }}
+ </mat-checkbox>
+ </sp-split-section>
+
+ <sp-split-section
+ [level]="3"
+ [title]="'Live preview' | translate"
+ >
+ <sp-csv-import-preview-table
+ [hasPreview]="hasPreview()"
+ [previewRows]="previewRows()"
+ [columnModels]="columnModels()"
+ [timestampFormat]="timestampFormat()"
+ [hasSchemaMismatch]="hasSchemaMismatch()"
+ [schemaMismatchSummary]="schemaMismatchSummary()"
+ [schemaMismatchDetails]="schemaMismatchDetails()"
+ [showTimestampWarning]="showTimestampWarning()"
+ (columnTypeChange)="
+ setColumnType($event.model, $event.type)
+ "
+ (columnRoleChange)="
+ setColumnRole($event.model, $event.role)
+ "
+ (timestampFormatChange)="updateTimestampFormat($event)"
+ ></sp-csv-import-preview-table>
+ </sp-split-section>
+
+ <ng-template matStepLabel>{{
+ 'Preview' | translate
+ }}</ng-template>
+ </mat-step>
+
+ <mat-step>
+ <sp-split-section [level]="3" [title]="'Upload' | translate">
+ <sp-csv-import-upload-state
+ [importLoading]="importLoading()"
+ [hasImportResult]="hasImportResult()"
+ [importResult]="importResult()"
+ [uploadErrors]="uploadMessages()"
+ ></sp-csv-import-upload-state>
+ </sp-split-section>
+
+ <ng-template matStepLabel>{{
+ 'Upload' | translate
+ }}</ng-template>
+ </mat-step>
+ </mat-horizontal-stepper>
+ </div>
+
+ <mat-divider></mat-divider>
+
+ <div class="sp-dialog-actions actions-align-left">
+ <button
+ mat-button
+ mat-flat-button
+ class="mat-basic"
+ style="margin-right: 10px"
+ (click)="close(hasImportResult())"
+ data-cy="csv-import-close-btn"
+ >
+ {{
+ hasImportResult()
+ ? ('Close' | translate)
+ : ('Cancel' | translate)
+ }}
+ </button>
+ @if (csvImportStepper.selectedIndex === 1 && !importLoading()) {
+ <button
+ mat-button
+ mat-flat-button
+ class="mat-basic"
+ style="margin-right: 10px"
+ (click)="previousStep()"
+ >
+ {{ 'Previous' | translate }}
+ </button>
+ }
+ @if (csvImportStepper.selectedIndex === 0) {
+ <button
+ mat-button
+ mat-flat-button
+ color="accent"
+ (click)="nextStep()"
+ [disabled]="!canProceedToConfiguration()"
+ data-cy="csv-import-next-btn"
+ >
+ {{ 'Next' | translate }}
+ </button>
+ }
+ @if (csvImportStepper.selectedIndex === 1) {
+ <button
+ mat-button
+ mat-flat-button
+ color="accent"
+ (click)="startUpload()"
+ [disabled]="!canImport() || importLoading()"
+ data-cy="csv-import-upload-btn"
+ >
+ {{ 'Upload' | translate }}
+ </button>
+ }
+ </div>
+</div>
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.scss b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.scss
new file mode 100644
index 0000000000..fa03b88623
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.scss
@@ -0,0 +1,95 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+.csv-import-shell {
+ min-height: 0;
+}
+
+.csv-import-content {
+ display: flex;
+ flex-direction: column;
+ gap: 18px;
+ max-height: 78vh;
+ overflow: auto;
+}
+
+:host {
+ display: block;
+ width: 100%;
+ min-width: 0;
+}
+
+:host ::ng-deep sp-split-section {
+ display: block;
+ width: 100%;
+ min-width: 0;
+ max-width: 100%;
+}
+
+:host ::ng-deep sp-split-section .section-outer,
+:host ::ng-deep sp-split-section .section-body,
+:host ::ng-deep sp-split-section .section-body-compact {
+ width: 100%;
+ min-width: 0;
+ max-width: 100%;
+}
+
+.upload-row {
+ display: flex;
+ align-items: center;
+ gap: 12px;
+ flex-wrap: wrap;
+}
+
+.file-label {
+ display: inline-flex;
+ align-items: center;
+ justify-content: center;
+ min-height: 48px;
+ padding: 0 16px;
+ border: 1px dashed var(--color-primary, #24176e);
+ border-radius: 8px;
+ cursor: pointer;
+ font-weight: 500;
+}
+
+.file-label input {
+ display: none;
+}
+
+.file-name {
+ color: var(--color-text-200, #5a6673);
+ font-size: 14px;
+}
+
+.grid {
+ display: grid;
+ grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
+ gap: 16px;
+}
+
+.message-box {
+ border-radius: 12px;
+ background: #fff4e5;
+ color: #8a4b00;
+ padding: 12px 14px;
+}
+
+.message-line + .message-line {
+ margin-top: 6px;
+}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.ts b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.ts
new file mode 100644
index 0000000000..2ec7073007
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-dialog.component.ts
@@ -0,0 +1,786 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { CommonModule } from '@angular/common';
+import {
+ Component,
+ computed,
+ inject,
+ Input,
+ signal,
+ viewChild,
+} from '@angular/core';
+import { toSignal, takeUntilDestroyed } from '@angular/core/rxjs-interop';
+import {
+ FormBuilder,
+ FormsModule,
+ ReactiveFormsModule,
+ Validators,
+} from '@angular/forms';
+import { MatButton } from '@angular/material/button';
+import { MatCheckbox } from '@angular/material/checkbox';
+import { MatDivider } from '@angular/material/divider';
+import { MatFormField } from '@angular/material/form-field';
+import { MatInput } from '@angular/material/input';
+import { MatOption, MatSelect } from '@angular/material/select';
+import { MatProgressSpinner } from '@angular/material/progress-spinner';
+import { MatStep, MatStepLabel, MatStepper } from '@angular/material/stepper';
+import { TranslatePipe } from '@ngx-translate/core';
+import {
+ CsvImportColumn,
+ CsvImportConfiguration,
+ CsvImportPreviewRequest,
+ CsvImportPreviewResult,
+ CsvImportRequest,
+ CsvImportResult,
+ CsvImportSchemaIssue,
+ CsvImportSchemaValidationRequest,
+ CsvImportSchemaValidationResult,
+ CsvImportTarget,
+ CsvImportValidationMessage,
+ CsvRuntimeType,
+ DataType,
+ DatalakeRestService,
+ EventPropertyPrimitive,
+ SemanticType,
+} from '@streampipes/platform-services';
+import {
+ DialogRef,
+ FormFieldComponent,
+ SplitSectionComponent,
+} from '@streampipes/shared-ui';
+import { startWith } from 'rxjs';
+import { CsvImportColumnModel, CsvImportColumnRole } from './csv-import.model';
+import { CsvImportPreviewTableComponent } from './csv-import-preview-table/csv-import-preview-table.component';
+import { CsvImportUploadStateComponent } from './csv-import-upload-state/csv-import-upload-state.component';
+
+@Component({
+ selector: 'sp-csv-import-dialog',
+ templateUrl: './csv-import-dialog.component.html',
+ styleUrls: ['./csv-import-dialog.component.scss'],
+ imports: [
+ CommonModule,
+ FormsModule,
+ ReactiveFormsModule,
+ MatButton,
+ MatCheckbox,
+ FormFieldComponent,
+ SplitSectionComponent,
+ MatFormField,
+ MatInput,
+ MatSelect,
+ MatOption,
+ MatDivider,
+ MatProgressSpinner,
+ MatStepper,
+ MatStep,
+ MatStepLabel,
+ TranslatePipe,
+ CsvImportPreviewTableComponent,
+ CsvImportUploadStateComponent,
+ ],
+})
+export class CsvImportDialogComponent {
+ @Input()
+ measurementNames: string[] = [];
+
+ readonly stepper = viewChild<MatStepper>('csvImportStepper');
+
+ private readonly fb = inject(FormBuilder);
+ private readonly dialogRef = inject(DialogRef<CsvImportDialogComponent>);
+ private readonly datalakeRestService = inject(DatalakeRestService);
+
+ private previewReloadTimeout?: ReturnType<typeof setTimeout>;
+ private schemaValidationTimeout?: ReturnType<typeof setTimeout>;
+
+ readonly selectedFile = signal<File | undefined>(undefined);
+ readonly uploadId = signal<string | undefined>(undefined);
+ readonly fileName = signal('');
+ readonly timestampFormat = signal('');
+ readonly previewResult = signal<CsvImportPreviewResult | undefined>(
+ undefined,
+ );
+ readonly schemaValidationResult = signal<
+ CsvImportSchemaValidationResult | undefined
+ >(undefined);
+ readonly importResult = signal<CsvImportResult | undefined>(undefined);
+ readonly columnModels = signal<CsvImportColumnModel[]>([]);
+ readonly previewLoading = signal(false);
+ readonly importLoading = signal(false);
+ readonly localMessages = signal<CsvImportValidationMessage[]>([]);
+ readonly uploadMessages = signal<CsvImportValidationMessage[]>([]);
+
+ readonly parseForm = this.fb.group({
+ delimiter: [',' as ',' | ';' | '|' | '\\t', Validators.required],
+ decimalSeparator: ['.' as ',' | '.', Validators.required],
+ hasHeader: [true, Validators.required],
+ });
+
+ readonly targetForm = this.fb.group({
+ mode: ['NEW', Validators.required],
+ newMeasurementName: [''],
+ existingMeasurementName: [''],
+ });
+
+ private readonly parseFormValue = toSignal(
+ this.parseForm.valueChanges.pipe(
+ startWith(this.parseForm.getRawValue()),
+ ),
+ { initialValue: this.parseForm.getRawValue() },
+ );
+
+ private readonly targetFormValue = toSignal(
+ this.targetForm.valueChanges.pipe(
+ startWith(this.targetForm.getRawValue()),
+ ),
+ { initialValue: this.targetForm.getRawValue() },
+ );
+
+ readonly topMessages = computed(() => [
+ ...(this.previewResult()?.validationMessages?.filter(
+ message =>
+ message.field !== 'columns' &&
+ message.field !== 'timestampColumn' &&
+ message.field !== 'schemaDetails',
+ ) ?? []),
+ ...this.localMessages(),
+ ]);
+
+ readonly hasPreview = computed(() => !!this.previewResult());
+ readonly hasImportResult = computed(
+ () => !!this.importResult()?.measurementName,
+ );
+ readonly previewRows = computed(
+ () => this.previewResult()?.previewRows ?? [],
+ );
+ readonly targetMode = computed(
+ () => this.targetFormValue().mode as 'NEW' | 'EXISTING',
+ );
+ readonly selectedTimestampColumn = computed(
+ () =>
+ this.columnModels().find(model =>
+ SemanticType.isTimestamp(model.eventProperty),
+ )?.column.runtimeName,
+ );
+ readonly canProceedToConfiguration = computed(
+ () => this.isTargetValid() && !!this.selectedFile(),
+ );
+ readonly currentTarget = computed<CsvImportTarget | undefined>(() => {
+ const formValue = this.targetFormValue();
+
+ if (this.targetMode() === 'NEW') {
+ const measurementName = formValue.newMeasurementName?.trim();
+ return measurementName
+ ? { mode: 'NEW', measurementName }
+ : undefined;
+ }
+
+ const measurementName = formValue.existingMeasurementName?.trim();
+ return measurementName
+ ? { mode: 'EXISTING', measurementName }
+ : undefined;
+ });
+
+ readonly canImport = computed(
+ () =>
+ this.previewResult()?.valid === true &&
+ !!this.currentTarget() &&
+ !!this.selectedTimestampColumn() &&
+ (this.targetMode() !== 'EXISTING' ||
+ this.schemaValidationResult()?.valid === true),
+ );
+
+ readonly showTimestampWarning = computed(
+ () => this.hasPreview() && !this.selectedTimestampColumn(),
+ );
+ readonly isTargetValid = computed(
+ () => this.validateLocalTarget().length === 0,
+ );
+ readonly hasSchemaMismatch = computed(
+ () =>
+ this.targetMode() === 'EXISTING' &&
+ !!this.schemaValidationResult()?.issues?.length,
+ );
+ readonly schemaMismatchSummary = computed(
+ () =>
+ 'Imported columns must exactly match the existing measurement schema.',
+ );
+ readonly schemaMismatchDetails = computed(() =>
+ (this.schemaValidationResult()?.issues ?? []).map(issue =>
+ this.toSchemaIssueText(issue),
+ ),
+ );
+
+ private readonly currentCsvConfig = computed<CsvImportConfiguration>(() => {
+ const formValue = this.parseFormValue();
+
+ return {
+ delimiter: formValue.delimiter ?? ',',
+ decimalSeparator: (formValue.decimalSeparator as ',' | '.') ?? '.',
+ hasHeader: formValue.hasHeader ?? true,
+ timestampFormat: this.timestampFormat().trim() || undefined,
+ };
+ });
+
+ constructor() {
+ this.parseForm.valueChanges.pipe(takeUntilDestroyed()).subscribe(() => {
+ this.invalidatePreview();
+ if (this.stepper()?.selectedIndex === 1) {
+ this.schedulePreviewReload();
+ }
+ });
+
+ this.targetForm.controls.mode.valueChanges
+ .pipe(takeUntilDestroyed())
+ .subscribe(mode => {
+ if (mode === 'NEW') {
+ this.targetForm.controls.existingMeasurementName.setValue(
+ '',
+ );
+ } else {
+ this.targetForm.controls.newMeasurementName.setValue('');
+ }
+
+ this.invalidatePreview();
+ });
+
+ this.targetForm.controls.newMeasurementName.valueChanges
+ .pipe(takeUntilDestroyed())
+ .subscribe(() => {
+ this.invalidatePreview();
+ if (this.stepper()?.selectedIndex === 1) {
+ this.schedulePreviewReload();
+ }
+ });
+
+ this.targetForm.controls.existingMeasurementName.valueChanges
+ .pipe(takeUntilDestroyed())
+ .subscribe(() => {
+ this.invalidatePreview();
+ if (this.stepper()?.selectedIndex === 1) {
+ this.schedulePreviewReload();
+ }
+ });
+ }
+
+ onFileSelected(event: Event): void {
+ const input = event.target as HTMLInputElement;
+ const file = input.files?.[0];
+ if (!file) {
+ return;
+ }
+
+ this.fileName.set(file.name);
+ this.selectedFile.set(file);
+ this.uploadId.set(undefined);
+ this.timestampFormat.set('');
+ this.invalidatePreview();
+ }
+
+ nextStep(): void {
+ if (this.stepper()?.selectedIndex !== 0) {
+ return;
+ }
+
+ if (!this.canProceedToConfiguration()) {
+ const messages = this.validateLocalTarget();
+ if (!this.selectedFile()) {
+ messages.push({
+ field: 'file',
+ message: 'Please select a CSV file first.',
+ });
+ }
+ this.localMessages.set(messages);
+ return;
+ }
+
+ this.stepper()?.next();
+ this.loadPreview();
+ }
+
+ previousStep(): void {
+ this.stepper()?.previous();
+ }
+
+ startUpload(): void {
+ this.uploadMessages.set([]);
+
+ const messages = this.validateLocalTarget();
+ if (!this.selectedTimestampColumn()) {
+ messages.push({
+ field: 'timestampColumn',
+ message: 'Please select exactly one timestamp column.',
+ });
+ }
+
+ this.localMessages.set(messages);
+ if (messages.length > 0 || !this.canImport()) {
+ return;
+ }
+
+ this.stepper()?.next();
+ this.importData();
+ }
+
+ loadPreview(): void {
+ this.localMessages.set([]);
+ this.clearImportResult();
+
+ if (!this.isTargetValid() || !this.currentTarget()) {
+ this.localMessages.set([
+ {
+ field: 'target.measurementName',
+ message:
+ 'Please complete the target dataset selection first.',
+ },
+ ]);
+ return;
+ }
+
+ if (!this.selectedFile() && !this.uploadId()) {
+ this.localMessages.set([
+ {
+ field: 'file',
+ message: 'Please select a CSV file first.',
+ },
+ ]);
+ return;
+ }
+
+ this.previewLoading.set(true);
+ const useMultipartUpload = !!this.selectedFile() && !this.uploadId();
+
+ this.datalakeRestService
+ .previewImport(
+ this.buildPreviewRequest(this.currentTarget()),
+ useMultipartUpload ? this.selectedFile() : undefined,
+ )
+ .subscribe({
+ next: preview => {
+ this.previewResult.set(preview);
+ this.uploadId.set(preview.uploadId ?? this.uploadId());
+ this.columnModels.set(
+ preview.columns.map(column =>
+ this.toColumnModel(column),
+ ),
+ );
+ this.schemaValidationResult.set(undefined);
+ this.previewLoading.set(false);
+ this.validateSchema();
+ },
+ error: error => {
+ this.previewLoading.set(false);
+ this.previewResult.set(undefined);
+ this.schemaValidationResult.set(undefined);
+ this.localMessages.set([
+ {
+ field: 'preview',
+ message:
+ error?.error?.message ??
+ 'Preview could not be generated.',
+ },
+ ]);
+ },
+ });
+ }
+
+ setColumnType(model: CsvImportColumnModel, type: CsvRuntimeType): void {
+ model.eventProperty.runtimeType = this.toRuntimeType(type);
+ model.column.runtimeType = type;
+
+ if (this.isTimestampColumn(model)) {
+ model.eventProperty.runtimeType = DataType.LONG;
+ model.column.runtimeType = 'LONG';
+ }
+
+ this.syncColumn(model);
+ this.columnModels.update(models => [...models]);
+ this.scheduleSchemaValidation();
+ }
+
+ setColumnRole(
+ model: CsvImportColumnModel,
+ role: CsvImportColumnRole,
+ ): void {
+ if (role === 'TIMESTAMP') {
+ this.columnModels().forEach(other => {
+ if (SemanticType.isTimestamp(other.eventProperty)) {
+ other.eventProperty.semanticType = undefined;
+ if (
+ other.eventProperty.propertyScope === 'HEADER_PROPERTY'
+ ) {
+ other.eventProperty.propertyScope =
+ 'MEASUREMENT_PROPERTY';
+ }
+ if (other.column.inferredType) {
+ other.eventProperty.runtimeType = this.toRuntimeType(
+ other.column.inferredType,
+ );
+ }
+ this.syncColumn(other);
+ }
+ });
+
+ model.eventProperty.semanticType = SemanticType.TIMESTAMP;
+ model.eventProperty.propertyScope = 'HEADER_PROPERTY';
+ model.eventProperty.runtimeType = DataType.LONG;
+ model.column.runtimeType = 'LONG';
+ } else {
+ model.eventProperty.semanticType = undefined;
+ model.eventProperty.propertyScope = role;
+ model.eventProperty.runtimeType = this.toRuntimeType(
+ model.column.runtimeType ||
+ model.column.inferredType ||
+ 'STRING',
+ );
+ }
+
+ this.syncColumn(model);
+ this.columnModels.update(models => [...models]);
+ this.scheduleSchemaValidation();
+ }
+
+ updateTimestampFormat(timestampFormat: string): void {
+ this.timestampFormat.set(timestampFormat);
+ this.scheduleSchemaValidation();
+ }
+
+ importData(): void {
+ const messages = this.validateLocalTarget();
+ this.uploadMessages.set([]);
+
+ if (!this.selectedTimestampColumn()) {
+ messages.push({
+ field: 'timestampColumn',
+ message: 'Please select exactly one timestamp column.',
+ });
+ }
+
+ this.localMessages.set(messages);
+ if (messages.length > 0) {
+ return;
+ }
+
+ this.importLoading.set(true);
+ this.datalakeRestService
+ .importCsvData(this.buildImportRequest())
+ .subscribe({
+ next: result => {
+ this.importResult.set(result);
+ this.importLoading.set(false);
+ },
+ error: error => {
+ this.importLoading.set(false);
+ const result = error?.error as CsvImportResult | undefined;
+ this.importResult.set(result);
+ this.uploadMessages.set(
+ result?.validationMessages?.length
+ ? result.validationMessages
+ : [
+ {
+ field: 'import',
+ message:
+ error?.error?.message ??
+ 'CSV import failed.',
+ },
+ ],
+ );
+ },
+ });
+ }
+
+ close(refresh = false): void {
+ this.dialogRef.close(refresh);
+ }
+
+ private buildPreviewRequest(
+ target?: CsvImportTarget,
+ ): CsvImportPreviewRequest {
+ return {
+ uploadId: this.uploadId(),
+ fileName: this.fileName(),
+ csvConfig: this.currentCsvConfig(),
+ target,
+ };
+ }
+
+ private buildImportRequest(): CsvImportRequest {
+ return {
+ uploadId: this.uploadId(),
+ csvConfig: this.currentCsvConfig(),
+ target: this.currentTarget()!,
+ timestampColumn: this.selectedTimestampColumn()!,
+ columns: this.columnModels().map(model => model.column),
+ };
+ }
+
+ private buildSchemaValidationRequest():
+ | CsvImportSchemaValidationRequest
+ | undefined {
+ if (!this.currentTarget() || !this.selectedTimestampColumn()) {
+ return undefined;
+ }
+
+ return {
+ target: this.currentTarget()!,
+ timestampColumn: this.selectedTimestampColumn()!,
+ columns: this.columnModels().map(model => model.column),
+ };
+ }
+
+ private validateLocalTarget(): CsvImportValidationMessage[] {
+ const messages: CsvImportValidationMessage[] = [];
+ const targetFormValue = this.targetFormValue();
+
+ if (this.targetMode() === 'NEW') {
+ const name = targetFormValue.newMeasurementName?.trim();
+
+ if (!name) {
+ messages.push({
+ field: 'target.measurementName',
+ message: 'Please provide a new dataset name.',
+ });
+ } else if (this.measurementNames.includes(name)) {
+ messages.push({
+ field: 'target.measurementName',
+ message: 'A dataset with this name already exists.',
+ });
+ }
+ } else if (!targetFormValue.existingMeasurementName) {
+ messages.push({
+ field: 'target.measurementName',
+ message: 'Please select an existing dataset.',
+ });
+ }
+
+ return messages;
+ }
+
+ private toColumnModel(column: CsvImportColumn): CsvImportColumnModel {
+ const initialType = this.toDefaultColumnType(
+ column.runtimeType || column.inferredType || 'STRING',
+ );
+ const property = new EventPropertyPrimitive();
+ property['@class'] =
+ 'org.apache.streampipes.model.schema.EventPropertyPrimitive';
+ property.runtimeName = column.runtimeName;
+ property.runtimeType = this.toRuntimeType(initialType);
+ property.propertyScope = 'MEASUREMENT_PROPERTY';
+ property.semanticType = undefined;
+ property.label = column.label || '';
+ property.description = column.description || '';
+ property.additionalMetadata = {};
+
+ return {
+ column: {
+ ...column,
+ runtimeType: initialType,
+ propertyScope: 'MEASUREMENT_PROPERTY',
+ semanticType: undefined,
+ },
+ eventProperty: property,
+ };
+ }
+
+ private syncColumn(model: CsvImportColumnModel): void {
+ model.column.runtimeName = model.eventProperty.runtimeName;
+ model.column.runtimeType = this.fromRuntimeType(
+ model.eventProperty.runtimeType,
+ );
+ model.column.propertyScope = model.eventProperty.propertyScope;
+ model.column.semanticType = model.eventProperty.semanticType;
+ model.column.label = model.eventProperty.label;
+ model.column.description = model.eventProperty.description;
+ }
+
+ private isTimestampColumn(model: CsvImportColumnModel): boolean {
+ return SemanticType.isTimestamp(model.eventProperty);
+ }
+
+ private toRuntimeType(type: string): string {
+ switch (type) {
+ case 'BOOLEAN':
+ return DataType.BOOLEAN;
+ case 'LONG':
+ return DataType.LONG;
+ case 'FLOAT':
+ return DataType.FLOAT;
+ default:
+ return DataType.STRING;
+ }
+ }
+
+ private fromRuntimeType(
+ type: string,
+ ): 'STRING' | 'BOOLEAN' | 'LONG' | 'FLOAT' {
+ if (type === DataType.BOOLEAN) {
+ return 'BOOLEAN';
+ } else if (type === DataType.LONG || type === DataType.INTEGER) {
+ return 'LONG';
+ } else if (type === DataType.FLOAT || type === DataType.DOUBLE) {
+ return 'FLOAT';
+ }
+ return 'STRING';
+ }
+
+ private toDefaultColumnType(
+ type: string,
+ ): 'STRING' | 'BOOLEAN' | 'LONG' | 'FLOAT' {
+ if (type === 'BOOLEAN') {
+ return 'BOOLEAN';
+ } else if (type === 'STRING') {
+ return 'STRING';
+ }
+ return 'FLOAT';
+ }
+
+ private invalidatePreview(): void {
+ if (this.previewReloadTimeout) {
+ clearTimeout(this.previewReloadTimeout);
+ }
+
+ if (this.schemaValidationTimeout) {
+ clearTimeout(this.schemaValidationTimeout);
+ }
+
+ this.previewResult.set(undefined);
+ this.schemaValidationResult.set(undefined);
+ this.columnModels.set([]);
+ this.clearImportResult();
+ }
+
+ private clearImportResult(): void {
+ this.importResult.set(undefined);
+ this.uploadMessages.set([]);
+ }
+
+ private schedulePreviewReload(): void {
+ if (!this.currentTarget() || this.previewLoading()) {
+ return;
+ }
+
+ if (!this.selectedFile() && !this.uploadId()) {
+ return;
+ }
+
+ if (this.previewReloadTimeout) {
+ clearTimeout(this.previewReloadTimeout);
+ }
+
+ this.previewReloadTimeout = setTimeout(() => {
+ this.loadPreview();
+ }, 250);
+ }
+
+ private validateSchema(): void {
+ const request = this.buildSchemaValidationRequest();
+ if (!request) {
+ this.schemaValidationResult.set(undefined);
+ return;
+ }
+
+ this.datalakeRestService.validateImportSchema(request).subscribe({
+ next: result => {
+ this.schemaValidationResult.set(result);
+ },
+ error: error => {
+ const result = error?.error as
+ | CsvImportSchemaValidationResult
+ | undefined;
+
+ if (
+ result?.validationMessages?.length ||
+ result?.issues?.length
+ ) {
+ this.schemaValidationResult.set(result);
+ } else {
+ this.schemaValidationResult.set({
+ valid: false,
+ validationMessages: [
+ {
+ field: 'columns',
+ message:
+ 'Schema validation could not be completed.',
+ },
+ ],
+ issues: [],
+ });
+ }
+ },
+ });
+ }
+
+ private scheduleSchemaValidation(): void {
+ if (
+ !this.hasPreview() ||
+ this.targetMode() !== 'EXISTING' ||
+ !this.selectedTimestampColumn()
+ ) {
+ this.schemaValidationResult.set(undefined);
+ return;
+ }
+
+ if (this.schemaValidationTimeout) {
+ clearTimeout(this.schemaValidationTimeout);
+ }
+
+ this.schemaValidationTimeout = setTimeout(() => {
+ this.validateSchema();
+ }, 250);
+ }
+
+ private toSchemaIssueText(issue: CsvImportSchemaIssue): string {
+ switch (issue.type) {
+ case 'TIMESTAMP_COLUMN_MISMATCH':
+ return `Timestamp column must be "${issue.expected}" but is "${issue.actual}".`;
+ case 'COLUMN_NAME_MISMATCH':
+ return `Column "${issue.actual || issue.columnName}" is not part of the existing measurement schema.`;
+ case 'COLUMN_TYPE_MISMATCH':
+ return `Column "${issue.columnName}" must use type "${this.formatSchemaValue(issue.expected)}" instead of "${this.formatSchemaValue(issue.actual)}".`;
+ case 'COLUMN_SCOPE_MISMATCH':
+ return `Column "${issue.columnName}" must use role "${this.formatScope(issue.expected)}" instead of "${this.formatScope(issue.actual)}".`;
+ default:
+ return 'Imported columns must exactly match the existing measurement schema.';
+ }
+ }
+
+ private formatSchemaValue(value?: string | null): string {
+ if (!value) {
+ return '-';
+ }
+
+ if (value.includes('#')) {
+ return value.substring(value.lastIndexOf('#') + 1).toUpperCase();
+ }
+
+ return value;
+ }
+
+ private formatScope(value?: string | null): string {
+ switch (value) {
+ case 'HEADER_PROPERTY':
+ return 'Header';
+ case 'DIMENSION_PROPERTY':
+ return 'Dimension';
+ case 'MEASUREMENT_PROPERTY':
+ return 'Measurement';
+ default:
+ return value || '-';
+ }
+ }
+}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.html b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.html
new file mode 100644
index 0000000000..a7235d3c54
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.html
@@ -0,0 +1,186 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+@if (hasSchemaMismatch()) {
+ <sp-alert-banner
+ type="error"
+ data-cy="csv-import-schema-mismatch"
+ [title]="'Error' | translate"
+ [description]="schemaMismatchSummary()"
+ ></sp-alert-banner>
+
+ @if (schemaMismatchDetails().length > 0) {
+ <ul
+ class="schema-mismatch-list"
+ data-cy="csv-import-schema-mismatch-list"
+ >
+ @for (detail of schemaMismatchDetails(); track detail) {
+ <li
+ class="schema-mismatch-item"
+ data-cy="csv-import-schema-mismatch-item"
+ >
+ {{ detail }}
+ </li>
+ }
+ </ul>
+ }
+}
+
+@if (showTimestampWarning()) {
+ <sp-alert-banner
+ type="warning"
+ data-cy="csv-import-timestamp-warning"
+ [title]="'Warning' | translate"
+ [description]="
+ 'Please select exactly one timestamp column before uploading the CSV.'
+ | translate
+ "
+ ></sp-alert-banner>
+}
+
+@if (hasPreview()) {
+ <div class="preview-table-wrapper" data-cy="csv-import-preview">
+ <table class="preview-table preview-config-table">
+ <thead>
+ <tr>
+ @for (
+ model of columnModels();
+ track model.column.csvColumn
+ ) {
+ <th>
+ <div class="header-cell">
+ <div class="header-title">
+ {{ model.column.csvColumn }}
+ </div>
+ @if (isTimestampColumn(model)) {
+ <mat-form-field
+ appearance="outline"
+ class="w-100"
+ >
+ <input
+ matInput
+ [ngModel]="timestampFormat()"
+ [ngModelOptions]="{
+ standalone: true,
+ }"
+ (ngModelChange)="
+ timestampFormatChange.emit(
+ $event
+ )
+ "
+ placeholder="yyyy-MM-dd HH:mm:ss"
+ data-cy="csv-import-timestamp-format"
+ />
+ </mat-form-field>
+ } @else {
+ <mat-form-field
+ appearance="outline"
+ class="w-100"
+ >
+ <mat-select
+ [value]="
+ model.column.runtimeType ||
+ model.column.inferredType ||
+ 'STRING'
+ "
+ (selectionChange)="
+ onColumnTypeChange(
+ model,
+ $event.value
+ )
+ "
+ data-cy="csv-import-column-type"
+ >
+ <mat-option value="STRING">
+ STRING
+ </mat-option>
+ <mat-option value="BOOLEAN">
+ BOOLEAN
+ </mat-option>
+ <mat-option value="LONG">
+ LONG
+ </mat-option>
+ <mat-option value="FLOAT">
+ FLOAT
+ </mat-option>
+ </mat-select>
+ </mat-form-field>
+ }
+ <mat-form-field
+ appearance="outline"
+ class="w-100"
+ >
+ <mat-select
+ [value]="getColumnRole(model)"
+ (selectionChange)="
+ onColumnRoleChange(
+ model,
+ $event.value
+ )
+ "
+ data-cy="csv-import-column-scope"
+ >
+ <mat-option
+ value="TIMESTAMP"
+ [disabled]="
+ isTimestampSelectionDisabled(
+ model
+ )
+ "
+ data-cy="csv-import-column-scope-timestamp"
+ >
+ {{ 'Timestamp' | translate }}
+ </mat-option>
+ <mat-option
+ value="DIMENSION_PROPERTY"
+ data-cy="csv-import-column-scope-dimension"
+ >
+ {{ 'Dimension' | translate }}
+ </mat-option>
+ <mat-option
+ value="MEASUREMENT_PROPERTY"
+ data-cy="csv-import-column-scope-measurement"
+ >
+ {{ 'Measurement' | translate }}
+ </mat-option>
+ </mat-select>
+ </mat-form-field>
+ </div>
+ </th>
+ }
+ </tr>
+ </thead>
+ <tbody>
+ @for (row of previewRows(); track $index) {
+ <tr>
+ @for (cell of row; track $index) {
+ <td>{{ cell }}</td>
+ }
+ </tr>
+ }
+ </tbody>
+ </table>
+ </div>
+} @else {
+ <div class="empty-state">
+ {{
+ 'Preview is being prepared from the current CSV configuration.'
+ | translate
+ }}
+ </div>
+}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.scss b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.scss
new file mode 100644
index 0000000000..abfd4de237
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.scss
@@ -0,0 +1,77 @@
+/*!
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+.schema-mismatch-list {
+ margin: 10px 0 0;
+ padding-left: 20px;
+ color: var(--color-text-200, #5a6673);
+}
+
+.schema-mismatch-item + .schema-mismatch-item {
+ margin-top: 6px;
+}
+
+.preview-table-wrapper {
+ overflow: auto;
+ border: 1px solid var(--color-border, #d8dee4);
+ border-radius: 14px;
+}
+
+.preview-table {
+ width: max-content;
+ min-width: 100%;
+ border-collapse: collapse;
+}
+
+.preview-table th,
+.preview-table td {
+ padding: 10px 12px;
+ border-bottom: 1px solid var(--color-border, #d8dee4);
+ text-align: left;
+ vertical-align: top;
+ white-space: nowrap;
+ font-size: 13px;
+}
+
+.preview-table th {
+ position: sticky;
+ top: 0;
+ background: #f8fafc;
+ z-index: 1;
+}
+
+.header-cell {
+ display: flex;
+ flex-direction: column;
+ gap: 10px;
+ min-width: 220px;
+}
+
+.header-title {
+ font-weight: 700;
+ font-size: 14px;
+ color: #111827;
+}
+
+.empty-state {
+ padding: 20px;
+ border-radius: 12px;
+ background: #f8fafc;
+ color: var(--color-text-200, #5a6673);
+ font-size: 14px;
+}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.ts b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.ts
new file mode 100644
index 0000000000..514560a300
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-preview-table/csv-import-preview-table.component.ts
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { CommonModule } from '@angular/common';
+import { Component, computed, input, output } from '@angular/core';
+import { FormsModule } from '@angular/forms';
+import { MatFormField } from '@angular/material/form-field';
+import { MatInput } from '@angular/material/input';
+import { MatOption, MatSelect } from '@angular/material/select';
+import { TranslatePipe } from '@ngx-translate/core';
+import { SemanticType } from '@streampipes/platform-services';
+import { SpAlertBannerComponent } from '@streampipes/shared-ui';
+import {
+ CsvImportColumnModel,
+ CsvImportColumnRole,
+ CsvImportColumnRoleChange,
+ CsvImportColumnTypeChange,
+} from '../csv-import.model';
+
+@Component({
+ selector: 'sp-csv-import-preview-table',
+ templateUrl: './csv-import-preview-table.component.html',
+ styleUrls: ['./csv-import-preview-table.component.scss'],
+ imports: [
+ CommonModule,
+ FormsModule,
+ MatFormField,
+ MatInput,
+ MatSelect,
+ MatOption,
+ TranslatePipe,
+ SpAlertBannerComponent,
+ ],
+})
+export class CsvImportPreviewTableComponent {
+ readonly hasPreview = input(false);
+ readonly previewRows = input<string[][]>([]);
+ readonly columnModels = input<CsvImportColumnModel[]>([]);
+ readonly timestampFormat = input('');
+ readonly hasSchemaMismatch = input(false);
+ readonly schemaMismatchSummary = input('');
+ readonly schemaMismatchDetails = input<string[]>([]);
+ readonly showTimestampWarning = input(false);
+
+ readonly columnTypeChange = output<CsvImportColumnTypeChange>();
+ readonly columnRoleChange = output<CsvImportColumnRoleChange>();
+ readonly timestampFormatChange = output<string>();
+
+ private readonly selectedTimestampColumnModel = computed(() =>
+ this.columnModels().find(model => this.isTimestampColumn(model)),
+ );
+
+ isTimestampColumn(model: CsvImportColumnModel): boolean {
+ return SemanticType.isTimestamp(model.eventProperty);
+ }
+
+ isTimestampSelectionDisabled(model: CsvImportColumnModel): boolean {
+ return (
+ !!this.selectedTimestampColumnModel() &&
+ !this.isTimestampColumn(model)
+ );
+ }
+
+ getColumnRole(model: CsvImportColumnModel): CsvImportColumnRole {
+ if (this.isTimestampColumn(model)) {
+ return 'TIMESTAMP';
+ } else if (model.column.propertyScope === 'DIMENSION_PROPERTY') {
+ return 'DIMENSION_PROPERTY';
+ }
+
+ return 'MEASUREMENT_PROPERTY';
+ }
+
+ onColumnTypeChange(
+ model: CsvImportColumnModel,
+ type: CsvImportColumnTypeChange['type'],
+ ): void {
+ this.columnTypeChange.emit({ model, type });
+ }
+
+ onColumnRoleChange(
+ model: CsvImportColumnModel,
+ role: CsvImportColumnRole,
+ ): void {
+ this.columnRoleChange.emit({ model, role });
+ }
+}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.html b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.html
new file mode 100644
index 0000000000..5acd93e07f
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.html
@@ -0,0 +1,56 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ ~
+ -->
+
+<div class="upload-state">
+ @if (importLoading()) {
+ <mat-spinner [diameter]="56"></mat-spinner>
+ <div class="upload-title">
+ {{ 'Uploading CSV data' | translate }}
+ </div>
+ <div class="upload-copy">
+ {{
+ 'The data is currently being written to the data lake.'
+ | translate
+ }}
+ </div>
+ } @else if (hasImportResult()) {
+ <mat-icon class="success-icon">check_circle</mat-icon>
+ <div class="upload-title" data-cy="csv-import-success-title">
+ {{ 'Import completed' | translate }}
+ </div>
+ <div class="result-name">
+ {{ importResult()?.measurementName }}
+ </div>
+ <div class="result-meta">
+ {{ importResult()?.importedRowCount }}
+ {{ 'rows imported' | translate }}
+ </div>
+ } @else if (uploadErrors().length > 0) {
+ <sp-alert-banner
+ type="error"
+ [title]="'Error' | translate"
+ [description]="'The CSV import failed.' | translate"
+ data-cy="csv-import-upload-error"
+ ></sp-alert-banner>
+ <div class="upload-errors" data-cy="csv-import-upload-error-details">
+ @for (message of uploadErrors(); track message.message) {
+ <div class="upload-error-line">{{ message.message }}</div>
+ }
+ </div>
+ }
+</div>
diff --git a/ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.scss
similarity index 55%
copy from ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts
copy to ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.scss
index 1f71ce8d8f..3b740c3d52 100644
--- a/ui/cypress/support/utils/chart/ChartWidgetTableUtils.ts
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.scss
@@ -1,4 +1,4 @@
-/*
+/*!
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -16,18 +16,43 @@
*
*/
-export class ChartWidgetTableUtils {
- public static chartTableRowTimestamp() {
- return cy.dataCy('data-explorer-table-row-timestamp', {
- timeout: 10000,
- });
- }
+.upload-state {
+ min-height: 280px;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ justify-content: center;
+ gap: 14px;
+ text-align: center;
+}
+
+.upload-title {
+ font-size: 20px;
+ font-weight: 600;
+}
+
+.upload-copy,
+.result-meta {
+ color: var(--color-text-200, #5a6673);
+ max-width: 420px;
+}
+
+.success-icon {
+ color: var(--color-success, #2e7d32);
+ font-size: 48px;
+ width: 48px;
+ height: 48px;
+}
+
+.result-name {
+ font-weight: 600;
+}
+
+.upload-errors {
+ max-width: 520px;
+ color: var(--color-text-200, #5a6673);
+}
- /**
- * Checks how many rows are visible within the table widget in charts
- * @param amount of expected rows
- */
- public static checkAmountOfRows(amount: number) {
- this.chartTableRowTimestamp().should('have.length', amount);
- }
+.upload-error-line + .upload-error-line {
+ margin-top: 6px;
}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.ts b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.ts
new file mode 100644
index 0000000000..9963c3a1e6
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import-upload-state/csv-import-upload-state.component.ts
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { CommonModule } from '@angular/common';
+import { Component, input } from '@angular/core';
+import { MatIcon } from '@angular/material/icon';
+import { MatProgressSpinner } from '@angular/material/progress-spinner';
+import { TranslatePipe } from '@ngx-translate/core';
+import {
+ CsvImportResult,
+ CsvImportValidationMessage,
+} from '@streampipes/platform-services';
+import { SpAlertBannerComponent } from '@streampipes/shared-ui';
+
+@Component({
+ selector: 'sp-csv-import-upload-state',
+ templateUrl: './csv-import-upload-state.component.html',
+ styleUrls: ['./csv-import-upload-state.component.scss'],
+ imports: [
+ CommonModule,
+ MatIcon,
+ MatProgressSpinner,
+ TranslatePipe,
+ SpAlertBannerComponent,
+ ],
+})
+export class CsvImportUploadStateComponent {
+ readonly importLoading = input(false);
+ readonly hasImportResult = input(false);
+ readonly importResult = input<CsvImportResult | undefined>(undefined);
+ readonly uploadErrors = input<CsvImportValidationMessage[]>([]);
+}
diff --git a/ui/src/app/dataset/dialog/csv-import-dialog/csv-import.model.ts b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import.model.ts
new file mode 100644
index 0000000000..298d81a6ff
--- /dev/null
+++ b/ui/src/app/dataset/dialog/csv-import-dialog/csv-import.model.ts
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import {
+ CsvImportColumn,
+ CsvRuntimeType,
+ EventPropertyPrimitive,
+} from '@streampipes/platform-services';
+
+export type CsvImportColumnRole =
+ | 'TIMESTAMP'
+ | 'DIMENSION_PROPERTY'
+ | 'MEASUREMENT_PROPERTY';
+
+export interface CsvImportColumnModel {
+ column: CsvImportColumn;
+ eventProperty: EventPropertyPrimitive;
+}
+
+export interface CsvImportColumnTypeChange {
+ model: CsvImportColumnModel;
+ type: CsvRuntimeType;
+}
+
+export interface CsvImportColumnRoleChange {
+ model: CsvImportColumnModel;
+ role: CsvImportColumnRole;
+}