You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streampipes.apache.org by ze...@apache.org on 2023/01/16 16:27:49 UTC
[streampipes] 01/01: [hotfix] Fix failing e2e tests
This is an automated email from the ASF dual-hosted git repository.
zehnder pushed a commit to branch hotfix-fix-nightly-build
in repository https://gitbox.apache.org/repos/asf/streampipes.git
commit 73fa98d7466b2e952824c7ecb3ad5eae0ee95e28
Author: Philipp Zehnder <te...@users.noreply.github.com>
AuthorDate: Mon Jan 16 17:27:29 2023 +0100
[hotfix] Fix failing e2e tests
---
.../connect/adapter/format/csv/CsvParser.java | 9 +-
.../connect/adapter/util/DatatypeUtils.java | 30 +++--
.../connect/adapter/format/csv/CsvParserTest.java | 45 +++++++
.../connect/adapter/util/DatatypeUtilsTest.java | 130 +++++++++++++++++++++
.../event-schema-preview.component.html | 1 +
5 files changed, 196 insertions(+), 19 deletions(-)
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
index a0a68d963..12724b38a 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParser.java
@@ -108,11 +108,11 @@ public class CsvParser extends Parser {
EventSchema resultSchema = new EventSchema();
for (int i = 0; i < keys.length; i++) {
EventPropertyPrimitive p = new EventPropertyPrimitive();
- var runtimeType = DatatypeUtils.getXsdDatatype(data[i], false);
+ var runtimeType = DatatypeUtils.getXsdDatatype(data[i], true);
var convertedValue = DatatypeUtils.convertValue(data[i], runtimeType);
p.setRuntimeName(keys[i]);
p.setRuntimeType(runtimeType);
- sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], false), convertedValue));
+ sample.put(keys[i], new GuessTypeInfo(DatatypeUtils.getCanonicalTypeClassName(data[i], true), convertedValue));
resultSchema.addEventProperty(p);
}
@@ -165,11 +165,6 @@ public class CsvParser extends Parser {
inQuotes = true;
- //Fixed : allow "" in empty quote enclosed
-// if (chars[0] != '"' && customQuote == '\"') {
-// curVal.append('"');
-// }
-
//double quotes in column will hit this!
if (startCollectChar) {
curVal.append('"');
diff --git a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
index 20655f112..4e1933ee0 100644
--- a/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
+++ b/streampipes-extensions-management/src/main/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtils.java
@@ -81,17 +81,29 @@ public class DatatypeUtils {
}
public static Class<?> getTypeClass(String value,
- boolean preferFloat) {
+ boolean preferFloatingPointNumber) {
+ var targetClass = String.class;
if (NumberUtils.isParsable(value)) {
+ Class<?> numberClass;
try {
- Integer.parseInt(value);
- return preferFloat ? Float.class : Integer.class;
+ long longValue = Long.parseLong(value);
+ numberClass = longValue > Integer.MAX_VALUE ? Long.class : Integer.class;
+ if (preferFloatingPointNumber) {
+ return numberClass == Long.class ? Double.class : Float.class;
+ } else {
+ return numberClass;
+ }
} catch (NumberFormatException ignored) {
}
try {
- Long.parseLong(value);
- return preferFloat ? Float.class : Long.class;
+ double doubleValue = Double.parseDouble(value);
+ numberClass = doubleValue > Float.MAX_VALUE ? Double.class : Float.class;
+ if (preferFloatingPointNumber) {
+ return numberClass == Double.class ? Double.class : Float.class;
+ } else {
+ return numberClass;
+ }
} catch (NumberFormatException ignored) {
}
@@ -107,13 +119,7 @@ public class DatatypeUtils {
return Boolean.class;
}
- return String.class;
+ return targetClass;
}
- public static void main(String[] args) {
- long max = Long.MAX_VALUE;
- String className = getCanonicalTypeClassName(String.valueOf(max), true);
- System.out.println(className);
- System.out.println(convertValue(max, className));
- }
}
diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java
new file mode 100644
index 000000000..f525319ad
--- /dev/null
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/format/csv/CsvParserTest.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.management.connect.adapter.format.csv;
+
+
+import org.junit.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class CsvParserTest {
+
+ @Test
+ public void getSchemaAndSampleTimestamp() {
+ var csvValue = List.of(
+ "timestamp,value".getBytes(StandardCharsets.UTF_8),
+ "1667904471000,1".getBytes(StandardCharsets.UTF_8)
+ );
+
+ var csvParser = new CsvParser(",", true);
+ var result = csvParser.getSchemaAndSample(csvValue);
+
+ assertEquals(result.getEventPreview().size(), 1);
+ assertEquals(1667904471000.0, result.getEventPreview().get(0).get("timestamp").getValue());
+ }
+
+}
\ No newline at end of file
diff --git a/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java
new file mode 100644
index 000000000..c5d273e07
--- /dev/null
+++ b/streampipes-extensions-management/src/test/java/org/apache/streampipes/extensions/management/connect/adapter/util/DatatypeUtilsTest.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.streampipes.extensions.management.connect.adapter.util;
+
+import org.apache.streampipes.vocabulary.XSD;
+
+import org.junit.Test;
+
+import java.util.Locale;
+
+import static org.junit.Assert.assertEquals;
+
+public class DatatypeUtilsTest {
+
+ @Test
+ /**
+ * This test ensures that timestamps represented as strings are correctly parsed.
+ * Often they are first parsed into floating point number before transformed back to long.
+ * The data type for those values should be Double and not Float, because the transformation to Float might change
+ * the value
+ */
+ public void convertTimestampValue() {
+ var inputValue = "1667904471000";
+
+ var floatValue = DatatypeUtils.convertValue(inputValue, XSD.DOUBLE.toString());
+ var longValue = DatatypeUtils.convertValue(floatValue, XSD.LONG.toString());
+
+ assertEquals(Long.parseLong(inputValue), longValue);
+ }
+
+ String booleanInputValue = "true";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointBoolean() {
+ var result = DatatypeUtils.getTypeClass(booleanInputValue, false);
+ assertEquals(Boolean.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointBoolean() {
+ var result = DatatypeUtils.getTypeClass(booleanInputValue, true);
+ assertEquals(Boolean.class, result);
+ }
+
+ String integerInputValue = "1";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointInteger() {
+ var result = DatatypeUtils.getTypeClass(integerInputValue, false);
+ assertEquals(Integer.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointInteger() {
+ var result = DatatypeUtils.getTypeClass(integerInputValue, true);
+ assertEquals(Float.class, result);
+ }
+
+ String floatInputValue = "1.0";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointFloat() {
+ var result = DatatypeUtils.getTypeClass(floatInputValue, false);
+ assertEquals(Float.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointFloat() {
+ var result = DatatypeUtils.getTypeClass(floatInputValue, true);
+ assertEquals(Float.class, result);
+ }
+
+
+ String doubleInputValue = String.format(Locale.US, "%.2f", Double.MAX_VALUE);
+ @Test
+ public void getTypeClassNoPrefereFloatingPointDouble() {
+ var result = DatatypeUtils.getTypeClass(doubleInputValue, false);
+ assertEquals(Double.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointDouble() {
+ var result = DatatypeUtils.getTypeClass(doubleInputValue, true);
+ assertEquals(Double.class, result);
+ }
+
+
+ String longInputValue = "1667904471000";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointLong() {
+ var result = DatatypeUtils.getTypeClass(longInputValue, false);
+ assertEquals(Long.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointLong() {
+ var result = DatatypeUtils.getTypeClass(longInputValue, true);
+ assertEquals(Double.class, result);
+ }
+
+ String stringInputValue = "one";
+ @Test
+ public void getTypeClassNoPrefereFloatingPointString() {
+ var result = DatatypeUtils.getTypeClass(stringInputValue, false);
+
+ assertEquals(String.class, result);
+ }
+
+ @Test
+ public void getTypeClassWithPrefereFloatingPointString() {
+ var result = DatatypeUtils.getTypeClass(stringInputValue, true);
+
+ assertEquals(String.class, result);
+ }
+
+
+}
\ No newline at end of file
diff --git a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
index a55ad96ce..6bb8b51e3 100644
--- a/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
+++ b/ui/src/app/connect/components/adapter-configuration/schema-editor/event-schema-preview/event-schema-preview.component.html
@@ -22,6 +22,7 @@
<pre
[innerHTML]="originalField | jsonpretty"
class="preview-text"
+ data-cy="schema-preview-original-event"
></pre>
</sp-basic-inner-panel>
</div>