You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/06/21 04:27:15 UTC
incubator-atlas git commit: ATLAS-1876: fix to handle large
float/double values during ser-de, export/import
Repository: incubator-atlas
Updated Branches:
refs/heads/master 242b5585c -> f053fd59b
ATLAS-1876: fix to handle large float/double values during ser-de, export/import
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/f053fd59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/f053fd59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/f053fd59
Branch: refs/heads/master
Commit: f053fd59b5337429d96d441a07ad3d44eac65a5d
Parents: 242b558
Author: ashutoshm <am...@hortonworks.com>
Authored: Tue Jun 20 20:09:39 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jun 20 21:13:13 2017 -0700
----------------------------------------------------------------------
.../apache/atlas/type/AtlasBuiltInTypes.java | 14 ++++-
.../java/org/apache/atlas/type/AtlasType.java | 5 +-
.../type/TestAtlasBuiltInTypesFloatDouble.java | 63 +++++++++++++++++++
.../atlas/repository/impexp/ZipSource.java | 26 +-------
.../atlas/repository/impexp/ZipSourceTest.java | 22 +++++++
repository/src/test/resources/stocks-float.zip | Bin 0 -> 5986 bytes
6 files changed, 103 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
index f08a601..139d4ec 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasBuiltInTypes.java
@@ -283,7 +283,12 @@ public class AtlasBuiltInTypes {
return ((Number) obj).floatValue();
} else {
try {
- return Float.valueOf(obj.toString());
+ Float f = Float.valueOf(obj.toString());
+ if(!Float.isInfinite(f)) {
+ return f;
+ } else {
+ return null;
+ }
} catch (NumberFormatException excp) {
// ignore
}
@@ -329,7 +334,12 @@ public class AtlasBuiltInTypes {
return ((Number) obj).doubleValue();
} else {
try {
- return Double.valueOf(obj.toString());
+ Double d = Double.valueOf(obj.toString());
+ if(!Double.isInfinite(d)) {
+ return d;
+ } else {
+ return null;
+ }
} catch (NumberFormatException excp) {
// ignore
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/intg/src/main/java/org/apache/atlas/type/AtlasType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasType.java b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
index f05cfd6..86072fe 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasType.java
@@ -21,9 +21,9 @@ package org.apache.atlas.type;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.ObjectMapper;
-
import java.io.IOException;
import java.util.List;
@@ -36,7 +36,8 @@ import java.util.List;
public abstract class AtlasType {
- private static final ObjectMapper mapper = new ObjectMapper();
+ private static final ObjectMapper mapper = new ObjectMapper()
+ .configure(DeserializationConfig.Feature.USE_BIG_DECIMAL_FOR_FLOATS, true);
private final String typeName;
private final TypeCategory typeCategory;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java
new file mode 100644
index 0000000..573164f
--- /dev/null
+++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasBuiltInTypesFloatDouble.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.type;
+
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+public class TestAtlasBuiltInTypesFloatDouble {
+
+ @Test
+ public void floatRangeCheck() {
+ assertFloatChecks(String.valueOf("-1.E-45"), true);
+ assertFloatChecks(String.valueOf(Float.MAX_VALUE), true);
+ assertFloatChecks("3.4028235E32", true);
+ assertFloatChecks("-3.4028235E32", true);
+ assertFloatChecks(String.valueOf(Float.MIN_VALUE), true);
+ assertFloatChecks("4028235e+555", false);
+ assertFloatChecks("-4028235e+555", false);
+ }
+
+ @Test
+ public void doubleRangeCheck() {
+ assertDoubleChecks(String.valueOf(Double.MAX_VALUE), true);
+ assertDoubleChecks("3.4028235E32", true);
+ assertDoubleChecks(String.valueOf(Double.MIN_VALUE), true);
+ assertDoubleChecks("4028235e+55555", false);
+ assertDoubleChecks("-4028235e+55555", false);
+ }
+
+ private void assertFloatChecks(String v, boolean notNull) {
+ assertNullNotNull(notNull, new AtlasBuiltInTypes.AtlasFloatType().getNormalizedValue(v));
+ }
+
+
+ private void assertDoubleChecks(String v, boolean notNull) {
+ assertNullNotNull(notNull, new AtlasBuiltInTypes.AtlasDoubleType().getNormalizedValue(v));
+ }
+
+ private void assertNullNotNull(boolean notNull, Object f) {
+ if(notNull) {
+ assertNotNull(f);
+ } else {
+ assertNull(f);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index aa1477f..edb816f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -23,9 +23,8 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.graph.v1.EntityImportStream;
+import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.type.TypeReference;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,8 +38,6 @@ import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
-import static org.apache.atlas.AtlasErrorCode.JSON_ERROR_OBJECT_MAPPER_NULL_RETURNED;
-
public class ZipSource implements EntityImportStream {
private static final Logger LOG = LoggerFactory.getLogger(ZipSource.class);
@@ -90,7 +87,7 @@ public class ZipSource implements EntityImportStream {
try {
String s = getFromCache(fileName);
- this.creationOrder = convertFromJson(new TypeReference<List<String>>(){}, s);
+ this.creationOrder = convertFromJson(List.class, s);
this.iterator = this.creationOrder.iterator();
} catch (AtlasBaseException e) {
LOG.error(String.format("Error retrieving '%s' from zip.", fileName), e);
@@ -137,26 +134,9 @@ public class ZipSource implements EntityImportStream {
return entityWithExtInfo;
}
- private <T> T convertFromJson(TypeReference clazz, String jsonData) throws AtlasBaseException {
- try {
- ObjectMapper mapper = new ObjectMapper();
-
- T ret = mapper.readValue(jsonData, clazz);
- if(ret == null) {
- throw new AtlasBaseException(JSON_ERROR_OBJECT_MAPPER_NULL_RETURNED, clazz.toString());
- }
-
- return ret;
- } catch (Exception e) {
- throw new AtlasBaseException("Error converting file to JSON.", e);
- }
- }
-
private <T> T convertFromJson(Class<T> clazz, String jsonData) throws AtlasBaseException {
try {
- ObjectMapper mapper = new ObjectMapper();
-
- return mapper.readValue(jsonData, clazz);
+ return AtlasType.fromJson(jsonData, clazz);
} catch (Exception e) {
throw new AtlasBaseException("Error converting file to JSON.", e);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
index 8a57dcd..1c1c68f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -44,6 +44,13 @@ public class ZipSourceTest {
return new Object[][] {{ new ZipSource(fs) }};
}
+ @DataProvider(name = "zipFileStocksFloat")
+ public static Object[][] getDataFromZipFileWithLongFloats() throws IOException {
+ FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream("stocks-float.zip");
+
+ return new Object[][] {{ new ZipSource(fs) }};
+ }
+
@DataProvider(name = "sales")
public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException {
return getZipSource("sales-v1-full.zip");
@@ -139,6 +146,21 @@ public class ZipSourceTest {
assertTrue(zipSource.hasNext());
}
+ @Test(dataProvider = "zipFileStocksFloat")
+ public void attemptToSerializeLongFloats(ZipSource zipSource) throws IOException, AtlasBaseException {
+ Assert.assertTrue(zipSource.hasNext());
+ assertTrue(zipSource.hasNext());
+ assertTrue(zipSource.hasNext());
+
+ AtlasEntity.AtlasEntityWithExtInfo e = zipSource.getNextEntityWithExtInfo();
+ assertNotNull(e);
+ assertTrue(e.getEntity().getClassifications().size() > 0);
+ assertNotNull(e.getEntity().getClassifications().get(0).getAttribute("fv"));
+ assertEquals(e.getEntity().getClassifications().get(0).getAttribute("fv").toString(), "3.4028235E+38");
+
+ assertTrue(zipSource.hasNext());
+ }
+
@Test(dataProvider = "zipFileStocks")
public void applyTransformation(ZipSource zipSource) throws IOException, AtlasBaseException {
ImportTransforms transforms = getTransformForHiveDB();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/f053fd59/repository/src/test/resources/stocks-float.zip
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/stocks-float.zip b/repository/src/test/resources/stocks-float.zip
new file mode 100644
index 0000000..ccef1e0
Binary files /dev/null and b/repository/src/test/resources/stocks-float.zip differ