You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/10/04 03:59:23 UTC
[1/2] atlas git commit: ATLAS-2897: Elegant handling of empty zip
files. Part 2.
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 360be1480 -> a0986f302
ATLAS-2897: Elegant handling of empty zip files. Part 2.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a490fcdf
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a490fcdf
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a490fcdf
Branch: refs/heads/branch-0.8
Commit: a490fcdf6fe7b2d4524d33088957bfa3b45426ba
Parents: 360be14
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Oct 3 09:01:07 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Wed Oct 3 13:10:30 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/ZipSource.java | 19 +++++++++++--------
.../repository/impexp/ExportServiceTest.java | 1 -
.../impexp/ZipFileResourceTestUtils.java | 20 ++++++++------------
.../atlas/web/resources/AdminResource.java | 19 ++++++++++++++++++-
4 files changed, 37 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/a490fcdf/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index 691f933..a974249 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -64,13 +64,20 @@ public class ZipSource implements EntityImportStream {
this.importTransform = importTransform;
updateGuidZipEntryMap();
- if (MapUtils.isEmpty(guidEntityJsonMap)) {
+ if (isZipFileEmpty()) {
throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP, "Attempting to import empty ZIP.");
}
setCreationOrder();
}
+ private boolean isZipFileEmpty() {
+ return MapUtils.isEmpty(guidEntityJsonMap) ||
+ (guidEntityJsonMap.containsKey(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()) &&
+ (guidEntityJsonMap.get(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()) == null)
+ );
+ }
+
public ImportTransforms getImportTransform() { return this.importTransform; }
public void setImportTransform(ImportTransforms importTransform) {
@@ -136,7 +143,7 @@ public class ZipSource implements EntityImportStream {
zipInputStream.close();
}
- public List<String> getCreationOrder() throws AtlasBaseException {
+ public List<String> getCreationOrder() {
return this.creationOrder;
}
@@ -228,12 +235,8 @@ public class ZipSource implements EntityImportStream {
@Override
public void reset() {
- try {
- getCreationOrder();
- this.iterator = this.creationOrder.iterator();
- } catch (AtlasBaseException e) {
- LOG.error("reset", e);
- }
+ getCreationOrder();
+ this.iterator = this.creationOrder.iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/a490fcdf/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 0d13a5b..6f168d1 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -294,7 +294,6 @@ public class ExportServiceTest extends ExportImportTestBase {
assertNotNull(zipSource.getCreationOrder());
assertEquals(zipSource.getCreationOrder().size(), 0);
- assertEquals(AtlasExportResult.OperationStatus.FAIL, zipSource.getExportResult().getOperationStatus());
}
@Test
http://git-wip-us.apache.org/repos/asf/atlas/blob/a490fcdf/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index 124853e..c4480ad 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -194,19 +194,15 @@ public class ZipFileResourceTestUtils {
public static AtlasEntity.AtlasEntityWithExtInfo getEntities(ZipSource source, int expectedCount) {
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo();
- try {
- int count = 0;
- for (String s : source.getCreationOrder()) {
- AtlasEntity entity = source.getByGuid(s);
- entityWithExtInfo.addReferredEntity(s, entity);
- count++;
- }
-
- assertEquals(count, expectedCount);
- return entityWithExtInfo;
- } catch (AtlasBaseException e) {
- throw new SkipException("getEntities: failed!");
+ int count = 0;
+ for (String s : source.getCreationOrder()) {
+ AtlasEntity entity = source.getByGuid(s);
+ entityWithExtInfo.addReferredEntity(s, entity);
+ count++;
}
+
+ assertEquals(count, expectedCount);
+ return entityWithExtInfo;
}
public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
http://git-wip-us.apache.org/repos/asf/atlas/blob/a490fcdf/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 26b2774..b53ac69 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -382,7 +382,7 @@ public class AdminResource {
}
acquireExportImportLock("import");
- AtlasImportResult result;
+ AtlasImportResult result = null;
try {
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
@@ -391,6 +391,15 @@ public class AdminResource {
result = importService.run(zipSource, request, Servlets.getUserName(httpServletRequest),
Servlets.getHostName(httpServletRequest),
AtlasAuthorizationUtils.getRequestIpAddress(httpServletRequest));
+ } catch (AtlasBaseException excp) {
+ if (excp.getAtlasErrorCode().equals(AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP)) {
+ LOG.info(excp.getMessage());
+ } else {
+ LOG.error("importData(binary) failed", excp);
+ }
+
+ throw excp;
+
} catch (Exception excp) {
LOG.error("importData(binary) failed", excp);
@@ -423,6 +432,14 @@ public class AdminResource {
result = importService.run(request, Servlets.getUserName(httpServletRequest),
Servlets.getHostName(httpServletRequest),
AtlasAuthorizationUtils.getRequestIpAddress(httpServletRequest));
+ } catch (AtlasBaseException excp) {
+ if (excp.getAtlasErrorCode().getErrorCode().equals(AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP)) {
+ LOG.info(excp.getMessage());
+ } else {
+ LOG.error("importData(binary) failed", excp);
+ }
+
+ throw excp;
} catch (Exception excp) {
LOG.error("importFile() failed", excp);
[2/2] atlas git commit: ATLAS-2906: Allow transforms to be applied
when entity-level transforms are present.
Posted by am...@apache.org.
ATLAS-2906: Allow transforms to be applied when entity-level transforms are present.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a0986f30
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a0986f30
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a0986f30
Branch: refs/heads/branch-0.8
Commit: a0986f302e9e89454fb51e684bac1455618cf556
Parents: a490fcd
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Oct 3 18:08:51 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Wed Oct 3 18:08:51 2018 -0700
----------------------------------------------------------------------
.../entitytransform/HiveStorageDescriptorEntityHandler.java | 9 ++++++++-
.../atlas/entitytransform/TransformationConstants.java | 1 +
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/a0986f30/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
index dc4edfb..38de206 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
@@ -26,7 +26,7 @@ import java.util.List;
import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
- static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE, HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE);
public HiveStorageDescriptorEntityHandler(List<AtlasEntityTransformer> transformers) {
@@ -90,6 +90,9 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
return clusterName;
+
+ case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
+ return location;
}
return super.getAttribute(attribute);
@@ -116,6 +119,10 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
isCustomAttributeUpdated = true;
break;
+ case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
+ location = attributeValue;
+ break;
+
default:
super.setAttribute(attribute, attributeValue);
break;
http://git-wip-us.apache.org/repos/asf/atlas/blob/a0986f30/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java b/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
index 51c3ace..247de73 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
@@ -37,6 +37,7 @@ public final class TransformationConstants {
public static final String HDFS_PATH_NAME_ATTRIBUTE = "hdfs_path.name";
public static final String HDFS_PATH_PATH_ATTRIBUTE = "hdfs_path.path";
public static final String HDFS_CLUSTER_NAME_ATTRIBUTE = "hdfs_path.clusterName";
+ public static final String HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE = "hive_storagedesc.location";
public static final char TYPE_NAME_ATTRIBUTE_NAME_SEP = '.';
public static final char CLUSTER_DELIMITER = '@';