You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/01/16 16:27:48 UTC
[streampipes] branch hotfix-fix-nightly-build created (now 73fa98d74)
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a change to branch hotfix-fix-nightly-build
in repository https://gitbox.apache.org/repos/asf/streampipes.git
at 73fa98d74 [hotfix] Fix failing e2e tests
This branch includes the following new commits:
new 73fa98d74 [hotfix] Fix failing e2e tests
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[streampipes] 01/01: [hotfix] Fix failing e2e tests
Posted by ze...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch hotfix-fix-nightly-build
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 73fa98d7466b2e952824c7ecb3ad5eae0ee95e28
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Jan 16 17:27:29 2023 +0100
[hotfix] Fix failing e2e tests
---
.../connect/adapter/format/csv/CsvParser.java | 9 +-
.../connect/adapter/util/DatatypeUtils.java | 30 +++--
.../connect/adapter/format/csv/CsvParserTest.java | 45 +++++++
.../connect/adapter/util/DatatypeUtilsTest.java | 130 +++++++++++++++++++++
.../event-schema-preview.component.html | 1 +
5 files changed, 196 insertions(+), 19 deletions(-)
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
index a0a68d963..12724b38a 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
@@ -108,11 +108,11 @@ public class CsvParser extends Parser {
EventSchema resultSchema = new EventSchema();
for (int i = 0; i < keys.length; i++) {
EventPropertyPrimitive p = new EventPropertyPrimitive();
- var runtimeType = DatatypeUtils.getXsdDatatype(data[i], false);
+ var runtimeType = DatatypeUtils.getXsdDatatype(data[i], true);
var convertedValue = DatatypeUtils.convertValue(data[i], runtimeType);
p.setRuntimeName(keys[i]);
p.setRuntimeType(runtimeType);
- sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], false), convertedValue));
+ sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], true), convertedValue));
resultSchema.addEventProperty(p);
}
@@ -165,11 +165,6 @@ public class CsvParser extends Parser {
inQuotes = true;
- //Fixed : allow "" in empty quote enclosed
-// if (chars[0] != '"' && customQuote == '\"') {
-// curVal.append('"');
-// }
-
//double quotes in column will hit this!
if (startCollectChar) {
curVal.append('"');
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
index 20655f112..4e1933ee0 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
@@ -81,17 +81,29 @@ public class DatatypeUtils {
}
public static Class<?> getTypeClass(String value,
- boolean preferFloat) {
+ boolean preferFloatingPointNumber) {
+ var targetClass = String.class;
if (NumberUtils.isParsable(value)) {
+ Class<?> numberClass;
try {
- Integer.parseInt(value);
- return preferFloat ? Float.class : Integer.class;
+ long longValue = Long.parseLong(value);
+ numberClass = longValue > Integer.MAX_VALUE ? Long.class : Integer.class;
+ if (preferFloatingPointNumber) {
+ return numberClass == Long.class ? Double.class : Float.class;
+ } else {
+ return numberClass;
+ }
} catch (NumberFormatException ignored) {
}
try {
- Long.parseLong(value);
- return preferFloat ? Float.class : Long.class;
+ double doubleValue = Double.parseDouble(value);
+ numberClass = doubleValue > Float.MAX_VALUE ? Double.class : Float.class;
+ if (preferFloatingPointNumber) {
+ return numberClass == Double.class ? Double.class : Float.class;
+ } else {
+ return numberClass;
+ }
} catch (NumberFormatException ignored) {
}
@@ -107,13 +119,7 @@ public class DatatypeUtils {
return Boolean.class;
}
- return String.class;
+ return targetClass;
}
- public static void main(String[] args) {
- long max = Long.MAX_VALUE;
- String className = getCanonicalTypeClassName(String.valueOf(max), true);
- System.out.println(className);
- System.out.println(convertValue(max, className));
- }
}
diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java
new file mode 100644
index 000000000..f525319ad
--- /dev/null
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.management.connect.adapter.format.csv;
+
+
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class CsvParserTest {
+
+ @Test
+ public void getSchemaAndSampleTimestamp() {
+ var csvValue = List.of(
+ "timestamp,value".getBytes(StandardCharsets.UTF_8),
+ "1667904471000,1".getBytes(StandardCharsets.UTF_8)
+ );
+
+ var csvParser = new CsvParser(",", true);
+ var result = csvParser.getSchemaAndSample(csvValue);
+
+ assertEquals(result.getEventPreview().size(), 1);
+ assertEquals(1667904471000.0, result.getEventPreview().get(0).get("timestamp").getValue());
+ }
+
+}
\ No newline at end of file
diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java
new file mode 100644
index 000000000..c5d273e07
--- /dev/null
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.management.connect.adapter.util;
+
+import org.apache.streampipes.vocabulary.XSD;
+
+import org.junit.Test;
+
+import java.util.Locale;
+
+import static org.junit.Assert.assertEquals;
+
+public class DatatypeUtilsTest {
+
+ @Test
+ /**
+ * This test ensures that timestamps represented as strings are correctly parsed.
+ * Often they are first parsed into floating point number before transformed back to long.
+ * The data type for those values should be Double and not Float, because the transformation to Float might change
+ * the value
+ */
+ public void convertTimestampValue() {
+ var inputValue = "1667904471000";
+
+ var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString());
+ var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString());
+
+ assertEquals(Long.parseLong(inputValue), longValue);
+ }
+
+ String booleanInputValue = "true";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointBoolean() {
+ var result = DatatypeUtils.getTypeClass(booleanInputValue, false);
+ assertEquals(Boolean.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointBoolean() {
+ var result = DatatypeUtils.getTypeClass(booleanInputValue, true);
+ assertEquals(Boolean.class, result);
+ }
+
+ String integerInputValue = "1";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointInteger() {
+ var result = DatatypeUtils.getTypeClass(integerInputValue, false);
+ assertEquals(Integer.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointInteger() {
+ var result = DatatypeUtils.getTypeClass(integerInputValue, true);
+ assertEquals(Float.class, result);
+ }
+
+ String floatInputValue = "1.0";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointFloat() {
+ var result = DatatypeUtils.getTypeClass(floatInputValue, false);
+ assertEquals(Float.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointFloat() {
+ var result = DatatypeUtils.getTypeClass(floatInputValue, true);
+ assertEquals(Float.class, result);
+ }
+
+
+ String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE);
+ @Test
+ public void getTypeClassNoPrefereFloatingPointDouble() {
+ var result = DatatypeUtils.getTypeClass(doubleInputValue, false);
+ assertEquals(Double.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointDouble() {
+ var result = DatatypeUtils.getTypeClass(doubleInputValue, true);
+ assertEquals(Double.class, result);
+ }
+
+
+ String longInputValue = "1667904471000";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointLong() {
+ var result = DatatypeUtils.getTypeClass(longInputValue, false);
+ assertEquals(Long.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointLong() {
+ var result = DatatypeUtils.getTypeClass(longInputValue, true);
+ assertEquals(Double.class, result);
+ }
+
+ String stringInputValue = "one";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointString() {
+ var result = DatatypeUtils.getTypeClass(stringInputValue, false);
+
+ assertEquals(String.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointString() {
+ var result = DatatypeUtils.getTypeClass(stringInputValue, true);
+
+ assertEquals(String.class, result);
+ }
+
+
+}
\ No newline at end of file
diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
index a55ad96ce..6bb8b51e3 100644
--- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
+++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
@@ -22,6 +22,7 @@
<pre
[innerHTML]="originalField | jsonpretty"
class="preview-text"
+ data-cy="schema-preview-original-event"
></pre>
</sp-basic-inner-panel>
</div>