You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2020/06/02 20:36:11 UTC
[atlas] branch branch-2.0 updated (84f579e -> 98e3d12)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from 84f579e ATLAS-3621: updated HiveHook to not save query-string in multiple attributes - queryText and name - #2 (avoid excessive logs)
new dee69ac ATLAS-3649: Table-level Imports: Deleting Tables Differing By Requests Fails
new 599e7c4 ATLAS-3773: Shadow Attributes Update Post-Migration from 0.8.
new 1ae8dc9 ATLAS-3595, ATLAS-3603: Auto-start migration import for ZIP file-based migration.
new 0631676 ATLAS-3641: Import Service: Support zipDirect format of import. Test files
new fd39e4c ATLAS-3643: PC Fx: StatusReporter: Introduce Status Reporting to Be used With PC Framework
new 09f907b ATLAS-3641: Import Service: Support zipDirect format of import.
new a82db67 ATLAS-3642: PC fx: WorkItemManager getResults Modification.
new 1678b9b ATLAS-3320: Migration Import implementation.
new 206c3ed ATLAS-3663: ZipFileMigrator: Automatic Resume During Migration.
new 8023946 ATLAS-3674: ZipFileMigrationImporter: Set Shell Entity Creation.
new 0030bc2 ATLAS-3735: CustomAttribute length validation bypass for keys with special prefix.
new f096f93 ATLAS-3737: Refactoring: Support multiple instances of AtlasGraph
new 43c9e32 ATLAS-3762: Improve Edge creator using Genuine iterator.
new fe5fa7f ATLAS-3762: Improve edge creation using genuine iterator. Part 2
new f1297e3 ATLAS-3796: Status Reporter display fix.
new 671e7f6 ATLAS-3798: Import percentage display fix.
new 9a1b2cc ATLAS-3799: EntityConumer only adds entity GUIDs that are added when they were produced.
new 3294324 ATLAS-3797: Refactoring: Improve Edge Creation.
new bd9e0c9 ATLAS-3722: ZipFileMigrationImporter: Allow AtlasPatchService to Run Before Migration Starts
new 98e3d12 Revert "ATLAS-3758: support sort params in FreeTextSearchProcessor - #2"
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/atlas/service/Services.java | 17 +-
.../atlas/repository/graphdb/AtlasGraph.java | 13 +-
.../atlas/repository/graphdb/AtlasVertex.java | 10 +
.../repository/graphdb/janus/AtlasJanusGraph.java | 67 +++-
.../repository/graphdb/janus/AtlasJanusVertex.java | 17 ++
.../graphdb/janus/query/NativeJanusGraphQuery.java | 2 +-
.../java/org/apache/atlas/AtlasConfiguration.java | 2 +
.../atlas/model/impexp/AtlasImportRequest.java | 44 ++-
.../MigrationImportStatus.java} | 46 ++-
.../java/org/apache/atlas/pc/StatusReporter.java | 118 ++++++++
.../java/org/apache/atlas/pc/WorkItemConsumer.java | 11 +-
.../java/org/apache/atlas/pc/WorkItemManager.java | 9 +-
.../java/org/apache/atlas/utils/AtlasJson.java | 4 +
...ithResultsTest.java => StatusReporterTest.java} | 107 +++----
.../apache/atlas/GraphTransactionInterceptor.java | 4 +
.../atlas/discovery/EntityDiscoveryService.java | 2 +-
.../atlas/discovery/EntityLineageService.java | 6 +-
.../converters/AtlasInstanceConverter.java | 7 +-
.../atlas/repository/graph/FullTextMapperV2.java | 14 +-
.../repository/graph/GraphBackedSearchIndexer.java | 69 +++++
.../apache/atlas/repository/graph/GraphHelper.java | 146 ++++-----
.../atlas/repository/graph/IFullTextMapper.java | 45 +++
.../atlas/repository/impexp/ExportService.java | 8 +-
.../atlas/repository/impexp/ImportService.java | 26 +-
.../impexp/MigrationProgressService.java | 42 ++-
.../impexp/TableReplicationRequestProcessor.java | 18 +-
.../repository/impexp/ZipExportFileNames.java | 4 +
.../atlas/repository/impexp/ZipSourceDirect.java | 337 +++++++++++++++++++++
.../repository/migration/DataMigrationService.java | 19 +-
.../migration/DataMigrationStatusService.java | 245 +++++++++++++++
.../atlas/repository/migration/FileWatcher.java | 149 +++++++++
.../migration/ZipFileMigrationImporter.java | 163 ++++++++++
.../repository/patches/UniqueAttributePatch.java | 43 ++-
.../repository/store/graph/AtlasEntityStore.java | 8 +
.../store/graph/v1/DeleteHandlerDelegate.java | 11 +-
.../repository/store/graph/v1/DeleteHandlerV1.java | 15 +-
.../store/graph/v1/HardDeleteHandlerV1.java | 5 +-
.../store/graph/v1/SoftDeleteHandlerV1.java | 5 +-
.../graph/v2/AtlasEntityGraphDiscoveryV2.java | 9 +-
.../store/graph/v2/AtlasEntityStoreV2.java | 76 ++---
.../store/graph/v2/AtlasGraphUtilsV2.java | 108 ++++---
.../store/graph/v2/AtlasRelationshipStoreV2.java | 56 ++--
.../store/graph/v2/BulkImporterImpl.java | 231 +++-----------
.../store/graph/v2/ClassificationAssociator.java | 25 +-
.../store/graph/v2/EntityGraphMapper.java | 104 ++++---
.../store/graph/v2/EntityGraphRetriever.java | 30 +-
.../store/graph/v2/EntityStateChecker.java | 10 +-
.../store/graph/v2/IDBasedEntityResolver.java | 11 +-
.../graph/v2/UniqAttrBasedEntityResolver.java | 9 +-
.../graph/v2/bulkimport/FullTextMapperV2Nop.java | 57 ++++
.../bulkimport/ImportStrategy.java} | 11 +-
.../store/graph/v2/bulkimport/MigrationImport.java | 143 +++++++++
.../RegularImport.java} | 83 ++---
.../graph/v2/bulkimport/pc/EntityConsumer.java | 270 +++++++++++++++++
.../v2/bulkimport/pc/EntityConsumerBuilder.java | 63 ++++
.../v2/bulkimport/pc/EntityCreationManager.java | 129 ++++++++
.../test/java/org/apache/atlas/TestModules.java | 5 +
.../impexp/DataMigrationStatusServiceTest.java | 66 ++++
.../repository/impexp/ExportSkipLineageTest.java | 6 +-
.../atlas/repository/impexp/ImportServiceTest.java | 15 +
.../atlas/repository/impexp/ZipDirectTest.java | 83 +++++
.../repository/migration/HiveParititionTest.java | 6 +-
.../atlas/repository/migration/HiveStocksTest.java | 6 +-
.../repository/migration/MigrationBaseAsserts.java | 22 +-
.../store/graph/v2/AtlasEntityStoreV2Test.java | 2 +-
.../store/graph/v2/AtlasEntityTestBase.java | 2 +-
.../graph/v2/AtlasRelationshipStoreV2Test.java | 8 +-
.../store/graph/v2/BulkImportPercentTest.java | 15 +-
.../graph/v2/ClassificationAssociatorTest.java | 25 +-
repository/src/test/resources/zip-direct-1.zip | Bin 0 -> 25511 bytes
repository/src/test/resources/zip-direct-2.zip | Bin 0 -> 26248 bytes
repository/src/test/resources/zip-direct-3.zip | Bin 0 -> 26165 bytes
.../resources/solr/core-template/solrconfig.xml | 2 +-
73 files changed, 2823 insertions(+), 723 deletions(-)
copy intg/src/main/java/org/apache/atlas/model/{discovery/AtlasSuggestionsResult.java => migration/MigrationImportStatus.java} (61%)
create mode 100644 intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
copy intg/src/test/java/org/apache/atlas/pc/{WorkItemManagerWithResultsTest.java => StatusReporterTest.java} (51%)
create mode 100644 repository/src/main/java/org/apache/atlas/repository/graph/IFullTextMapper.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/migration/FileWatcher.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/FullTextMapperV2Nop.java
copy repository/src/main/java/org/apache/atlas/repository/store/graph/{EntityResolver.java => v2/bulkimport/ImportStrategy.java} (66%)
create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
copy repository/src/main/java/org/apache/atlas/repository/store/graph/v2/{BulkImporterImpl.java => bulkimport/RegularImport.java} (78%)
create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java
create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
create mode 100644 repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
create mode 100644 repository/src/test/java/org/apache/atlas/repository/impexp/ZipDirectTest.java
create mode 100644 repository/src/test/resources/zip-direct-1.zip
create mode 100644 repository/src/test/resources/zip-direct-2.zip
create mode 100644 repository/src/test/resources/zip-direct-3.zip
[atlas] 16/20: ATLAS-3798: Import percentage display fix.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 671e7f65baa923a2b9b8d8be47752b21a5f8c0df
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Fri May 15 11:50:33 2020 -0700
ATLAS-3798: Import percentage display fix.
(cherry picked from commit 214f2a760a1a4c8c3a83b5afaa42fdc36055ff98)
---
.../atlas/repository/store/graph/v2/BulkImporterImpl.java | 4 ++--
.../repository/store/graph/v2/BulkImportPercentTest.java | 15 ++++++++++++---
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
index 9898fe8..8e17fd4 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
@@ -89,11 +89,11 @@ public class BulkImporterImpl implements BulkImporter {
}
@VisibleForTesting
- public static float updateImportProgress(Logger log, int currentIndex, int streamSize, float currentPercent, String additionalInfo) {
+ public static float updateImportProgress(Logger log, long currentIndex, long streamSize, float currentPercent, String additionalInfo) {
final double tolerance = 0.000001;
final int MAX_PERCENT = 100;
- int maxSize = (currentIndex <= streamSize) ? streamSize : currentIndex;
+ long maxSize = (currentIndex <= streamSize) ? streamSize : currentIndex;
if (maxSize <= 0) {
return currentPercent;
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkImportPercentTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkImportPercentTest.java
index 2fa4c91..1ae98ce 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkImportPercentTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/BulkImportPercentTest.java
@@ -39,7 +39,7 @@ public class BulkImportPercentTest {
private List<Integer> percentHolder;
private Logger log;
- public void setupPercentHolder(int max) {
+ public void setupPercentHolder(long max) {
percentHolder = new ArrayList<>();
}
@@ -127,6 +127,15 @@ public class BulkImportPercentTest {
}
@Test
+ public void percentTest_Equal100M() throws Exception {
+ long streamSize = 100000000;
+ double[] expected = fillPercentHolderWith100();
+
+ runWithSize(streamSize);
+ assertEqualsForPercentHolder(expected);
+ }
+
+ @Test
public void percentTest_Equal4323() throws Exception {
int streamSize = 4323;
@@ -153,7 +162,7 @@ public class BulkImportPercentTest {
assertTrue((f - MAX_PERCENT_FLOAT) <= 0.0001);
}
- private void runWithSize(int streamSize) throws Exception {
+ private void runWithSize(long streamSize) throws Exception {
float currentPercent = 0;
setupPercentHolder(streamSize);
for (int currentIndex = 0; currentIndex < streamSize; currentIndex++) {
@@ -161,7 +170,7 @@ public class BulkImportPercentTest {
}
}
- private float invokeBulkImportProgress(int currentIndex, int streamSize, float currentPercent) throws Exception {
+ private float invokeBulkImportProgress(int currentIndex, long streamSize, float currentPercent) throws Exception {
return BulkImporterImpl.updateImportProgress(log, currentIndex, streamSize, currentPercent, "additional info");
}
[atlas] 07/20: ATLAS-3642: PC fx: WorkItemManager getResults
Modification.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit a82db67aba71e4604a900499e06987b726aa93a7
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon Mar 2 09:17:41 2020 -0800
ATLAS-3642: PC fx: WorkItemManager getResults Modification.
(cherry picked from commit cc601d7371fae1dbc16b55d1ca84f06b745700dc)
---
intg/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java | 11 ++++-------
intg/src/main/java/org/apache/atlas/pc/WorkItemManager.java | 9 +++++----
2 files changed, 9 insertions(+), 11 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java b/intg/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java
index 9ba4bf4..dd76697 100644
--- a/intg/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java
+++ b/intg/src/main/java/org/apache/atlas/pc/WorkItemConsumer.java
@@ -21,6 +21,7 @@ package org.apache.atlas.pc;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -37,7 +38,7 @@ public abstract class WorkItemConsumer<T> implements Runnable {
private final AtomicBoolean isDirty = new AtomicBoolean(false);
private final AtomicLong maxCommitTimeInMs = new AtomicLong(DEFAULT_COMMIT_TIME_IN_MS);
private CountDownLatch countdownLatch;
- private BlockingQueue<Object> results;
+ private Queue<Object> results;
public WorkItemConsumer(BlockingQueue<T> queue) {
this.queue = queue;
@@ -101,11 +102,7 @@ public abstract class WorkItemConsumer<T> implements Runnable {
protected abstract void processItem(T item);
protected void addResult(Object value) {
- try {
- results.put(value);
- } catch (InterruptedException e) {
- LOG.error("Interrupted while adding result: {}", value);
- }
+ results.add(value);
}
protected void updateCommitTime(long commitTime) {
@@ -118,7 +115,7 @@ public abstract class WorkItemConsumer<T> implements Runnable {
this.countdownLatch = countdownLatch;
}
- public <V> void setResults(BlockingQueue<Object> queue) {
+ public <V> void setResults(Queue<Object> queue) {
this.results = queue;
}
}
diff --git a/intg/src/main/java/org/apache/atlas/pc/WorkItemManager.java b/intg/src/main/java/org/apache/atlas/pc/WorkItemManager.java
index a7ba67c..351421e 100644
--- a/intg/src/main/java/org/apache/atlas/pc/WorkItemManager.java
+++ b/intg/src/main/java/org/apache/atlas/pc/WorkItemManager.java
@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
+import java.util.Queue;
import java.util.concurrent.*;
public class WorkItemManager<T, U extends WorkItemConsumer> {
@@ -33,7 +34,7 @@ public class WorkItemManager<T, U extends WorkItemConsumer> {
private final ExecutorService service;
private final List<U> consumers = new ArrayList<>();
private CountDownLatch countdownLatch;
- private BlockingQueue<Object> resultsQueue;
+ private Queue<Object> resultsQueue;
public WorkItemManager(WorkItemBuilder builder, String namePrefix, int batchSize, int numWorkers, boolean collectResults) {
this.numWorkers = numWorkers;
@@ -49,13 +50,13 @@ public class WorkItemManager<T, U extends WorkItemConsumer> {
this(builder, "workItemConsumer", batchSize, numWorkers, false);
}
- public void setResultsCollection(BlockingQueue<Object> resultsQueue) {
+ public void setResultsCollection(Queue<Object> resultsQueue) {
this.resultsQueue = resultsQueue;
}
private void createConsumers(WorkItemBuilder builder, int numWorkers, boolean collectResults) {
if (collectResults) {
- setResultsCollection(new LinkedBlockingQueue<>());
+ setResultsCollection(new ConcurrentLinkedQueue<>());
}
for (int i = 0; i < numWorkers; i++) {
@@ -124,7 +125,7 @@ public class WorkItemManager<T, U extends WorkItemConsumer> {
LOG.info("WorkItemManager: Shutdown done!");
}
- public BlockingQueue getResults() {
+ public Queue getResults() {
return this.resultsQueue;
}
[atlas] 04/20: ATLAS-3641: Import Service: Support zipDirect format
of import. Test files
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 06316761621af3d1f9081e8da0b21bda80b96807
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon Mar 2 08:03:00 2020 -0800
ATLAS-3641: Import Service: Support zipDirect format of import. Test files
(cherry picked from commit 6a49d94faf9613d3b10c33373500ca5e09b8dc7f)
---
repository/src/test/resources/zip-direct-1.zip | Bin 0 -> 25511 bytes
repository/src/test/resources/zip-direct-2.zip | Bin 0 -> 26248 bytes
repository/src/test/resources/zip-direct-3.zip | Bin 0 -> 26165 bytes
3 files changed, 0 insertions(+), 0 deletions(-)
diff --git a/repository/src/test/resources/zip-direct-1.zip b/repository/src/test/resources/zip-direct-1.zip
new file mode 100644
index 0000000..5907cbc
Binary files /dev/null and b/repository/src/test/resources/zip-direct-1.zip differ
diff --git a/repository/src/test/resources/zip-direct-2.zip b/repository/src/test/resources/zip-direct-2.zip
new file mode 100644
index 0000000..9c05b72
Binary files /dev/null and b/repository/src/test/resources/zip-direct-2.zip differ
diff --git a/repository/src/test/resources/zip-direct-3.zip b/repository/src/test/resources/zip-direct-3.zip
new file mode 100644
index 0000000..1a4d877
Binary files /dev/null and b/repository/src/test/resources/zip-direct-3.zip differ
[atlas] 06/20: ATLAS-3641: Import Service: Support zipDirect format
of import.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 09f907b6497aa527047456d9cefea0a170123af5
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon Mar 2 08:14:04 2020 -0800
ATLAS-3641: Import Service: Support zipDirect format of import.
(cherry picked from commit e8661ecbc49e750b3e962aa999f11b2a94d8fe27)
---
.../atlas/model/impexp/AtlasImportRequest.java | 20 +-
.../java/org/apache/atlas/utils/AtlasJson.java | 4 +
.../atlas/repository/impexp/ImportService.java | 17 +-
.../repository/impexp/ZipExportFileNames.java | 4 +
.../atlas/repository/impexp/ZipSourceDirect.java | 333 +++++++++++++++++++++
.../migration/ZipFileMigrationImporter.java | 8 +-
.../atlas/repository/impexp/ImportServiceTest.java | 15 +
.../atlas/repository/impexp/ZipDirectTest.java | 83 +++++
8 files changed, 474 insertions(+), 10 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 0b3ede9..3362bf1 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -44,10 +44,13 @@ public class AtlasImportRequest implements Serializable {
public static final String TRANSFORMS_KEY = "transforms";
public static final String TRANSFORMERS_KEY = "transformers";
public static final String OPTION_KEY_REPLICATED_FROM = "replicatedFrom";
+ public static final String OPTION_KEY_FORMAT = "format";
+ public static final String OPTION_KEY_FORMAT_ZIP_DIRECT = "zipDirect";
private static final String START_POSITION_KEY = "startPosition";
private static final String START_GUID_KEY = "startGuid";
private static final String FILE_NAME_KEY = "fileName";
private static final String UPDATE_TYPE_DEFINITION_KEY = "updateTypeDefinition";
+ private static final String OPTION_KEY_STREAM_SIZE = "size";
private Map<String, String> options;
@@ -108,7 +111,7 @@ public class AtlasImportRequest implements Serializable {
return null;
}
- return (String) this.options.get(key);
+ return this.options.get(key);
}
@JsonIgnore
@@ -127,4 +130,17 @@ public class AtlasImportRequest implements Serializable {
options = new HashMap<>();
}
options.put(key, value);
- }}
+ }
+
+ public void setSizeOption(int size) {
+ setOption(OPTION_KEY_STREAM_SIZE, Integer.toString(size));
+ }
+
+ public int getSizeOption() {
+ if (!this.options.containsKey(OPTION_KEY_STREAM_SIZE)) {
+ return 1;
+ }
+
+ return Integer.valueOf(this.options.get(OPTION_KEY_STREAM_SIZE));
+ }
+}
diff --git a/intg/src/main/java/org/apache/atlas/utils/AtlasJson.java b/intg/src/main/java/org/apache/atlas/utils/AtlasJson.java
index 1c13860..abeddf6 100644
--- a/intg/src/main/java/org/apache/atlas/utils/AtlasJson.java
+++ b/intg/src/main/java/org/apache/atlas/utils/AtlasJson.java
@@ -251,6 +251,10 @@ public class AtlasJson {
return ret;
}
+ public static ObjectCodec getMapper() {
+ return mapper;
+ }
+
static class DateSerializer extends JsonSerializer<Date> {
@Override
public void serialize(Date value, JsonGenerator jgen, SerializerProvider provider) throws IOException {
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index 27001e3..1964ade 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -92,7 +92,7 @@ public class ImportService {
request = new AtlasImportRequest();
}
- EntityImportStream source = createZipSource(inputStream, AtlasConfiguration.IMPORT_TEMP_DIRECTORY.getString());
+ EntityImportStream source = createZipSource(request, inputStream, AtlasConfiguration.IMPORT_TEMP_DIRECTORY.getString());
return run(source, request, userName, hostName, requestingIP);
}
@@ -248,8 +248,13 @@ public class ImportService {
return (int) (endTime - startTime);
}
- private EntityImportStream createZipSource(InputStream inputStream, String configuredTemporaryDirectory) throws AtlasBaseException {
+ private EntityImportStream createZipSource(AtlasImportRequest request, InputStream inputStream, String configuredTemporaryDirectory) throws AtlasBaseException {
try {
+ if (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_FORMAT) &&
+ request.getOptions().get(AtlasImportRequest.OPTION_KEY_FORMAT).equals(AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT) ) {
+ return getZipDirectEntityImportStream(request, inputStream);
+ }
+
if (StringUtils.isEmpty(configuredTemporaryDirectory)) {
return new ZipSource(inputStream);
}
@@ -260,9 +265,15 @@ public class ImportService {
}
}
+ private EntityImportStream getZipDirectEntityImportStream(AtlasImportRequest request, InputStream inputStream) throws IOException, AtlasBaseException {
+ ZipSourceDirect zipSourceDirect = new ZipSourceDirect(inputStream, request.getSizeOption());
+ LOG.info("Using ZipSourceDirect: Size: {} entities", zipSourceDirect.size());
+ return zipSourceDirect;
+ }
+
@VisibleForTesting
boolean checkHiveTableIncrementalSkipLineage(AtlasImportRequest importRequest, AtlasExportRequest exportRequest) {
- if (CollectionUtils.isEmpty(exportRequest.getItemsToExport())) {
+ if (exportRequest == null || CollectionUtils.isEmpty(exportRequest.getItemsToExport())) {
return false;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipExportFileNames.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipExportFileNames.java
index 351b475..8347b91 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipExportFileNames.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipExportFileNames.java
@@ -31,4 +31,8 @@ public enum ZipExportFileNames {
public String toString() {
return this.name;
}
+
+ public String toEntryFileName() {
+ return this.name + ".json";
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
new file mode 100644
index 0000000..cb5a7ac
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
@@ -0,0 +1,333 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.atlas.entitytransform.BaseEntityHandler;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.utils.AtlasJson;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.apache.atlas.AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP;
+
+public class ZipSourceDirect implements EntityImportStream {
+ private static final Logger LOG = LoggerFactory.getLogger(ZipSourceDirect.class);
+ private static final String ZIP_ENTRY_ENTITIES = "entities.json";
+
+ private final ZipInputStream zipInputStream;
+ private int currentPosition;
+
+ private ImportTransforms importTransform;
+ private List<BaseEntityHandler> entityHandlers;
+ private AtlasTypesDef typesDef;
+ private int streamSize = 1;
+
+ EntitiesArrayParser entitiesArrayParser;
+
+ public ZipSourceDirect(InputStream inputStream, int streamSize) throws IOException, AtlasBaseException {
+ this.zipInputStream = new ZipInputStream(inputStream);
+ this.streamSize = streamSize;
+ prepareStreamForFetch();
+ }
+
+ @Override
+ public ImportTransforms getImportTransform() {
+ return this.importTransform;
+ }
+
+ @Override
+ public void setImportTransform(ImportTransforms importTransform) {
+ this.importTransform = importTransform;
+ }
+
+ @Override
+ public List<BaseEntityHandler> getEntityHandlers() {
+ return entityHandlers;
+ }
+
+ @Override
+ public void setEntityHandlers(List<BaseEntityHandler> entityHandlers) {
+ this.entityHandlers = entityHandlers;
+ }
+
+ @Override
+ public AtlasTypesDef getTypesDef() throws AtlasBaseException {
+ return this.typesDef;
+ }
+
+ @Override
+ public AtlasExportResult getExportResult() throws AtlasBaseException {
+ return new AtlasExportResult();
+ }
+
+ @Override
+ public List<String> getCreationOrder() {
+ return new ArrayList<>();
+ }
+
+ @Override
+ public int getPosition() {
+ return currentPosition;
+ }
+
+ @Override
+ public AtlasEntity.AtlasEntityWithExtInfo getEntityWithExtInfo(String json) throws AtlasBaseException {
+ if (StringUtils.isEmpty(json)) {
+ return null;
+ }
+
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = convertFromJson(AtlasEntity.AtlasEntityWithExtInfo.class, json);
+
+ if (importTransform != null) {
+ entityWithExtInfo = importTransform.apply(entityWithExtInfo);
+ }
+
+ if (entityHandlers != null) {
+ applyTransformers(entityWithExtInfo);
+ }
+
+ return entityWithExtInfo;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return (this.entitiesArrayParser != null && entitiesArrayParser.hasNext());
+ }
+
+ @Override
+ public AtlasEntity next() {
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = getNextEntityWithExtInfo();
+
+ return entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
+ }
+
+ @Override
+ public AtlasEntity.AtlasEntityWithExtInfo getNextEntityWithExtInfo() {
+ try {
+ if (hasNext()) {
+ String json = moveNext();
+ return getEntityWithExtInfo(json);
+ }
+ } catch (AtlasBaseException e) {
+ LOG.error("getNextEntityWithExtInfo", e);
+ }
+ return null;
+ }
+
+ @Override
+ public void reset() {
+ currentPosition = 0;
+ }
+
+ @Override
+ public AtlasEntity getByGuid(String guid) {
+ try {
+ return getEntity(guid);
+ } catch (AtlasBaseException e) {
+ LOG.error("getByGuid: {} failed!", guid, e);
+ return null;
+ }
+ }
+
+ @Override
+ public void onImportComplete(String guid) {
+ }
+
+ @Override
+ public void setPosition(int index) {
+ try {
+ for (int i = 0; i < index; i++) {
+ moveNextEntry();
+ }
+ } catch (IOException e) {
+ LOG.error("Error setting position: {}. Position may be beyond the stream size.", index);
+ }
+ }
+
+ @Override
+ public void setPositionUsingEntityGuid(String guid) {
+ }
+
+ @Override
+ public void close() {
+ if (this.entitiesArrayParser != null) {
+ this.entitiesArrayParser.close();
+ }
+ }
+
+ private void applyTransformers(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ if (entityWithExtInfo == null) {
+ return;
+ }
+
+ transform(entityWithExtInfo.getEntity());
+
+ if (MapUtils.isNotEmpty(entityWithExtInfo.getReferredEntities())) {
+ for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) {
+ transform(e);
+ }
+ }
+ }
+
+ private void transform(AtlasEntity e) {
+ for (BaseEntityHandler handler : entityHandlers) {
+ handler.transform(e);
+ }
+ }
+
+ private <T> T convertFromJson(Class<T> clazz, String jsonData) throws AtlasBaseException {
+ try {
+ return AtlasType.fromJson(jsonData, clazz);
+
+ } catch (Exception e) {
+ throw new AtlasBaseException("Error converting file to JSON.", e);
+ }
+ }
+
+ private AtlasEntity getEntity(String guid) throws AtlasBaseException {
+ AtlasEntity.AtlasEntityWithExtInfo extInfo = getEntityWithExtInfo(guid);
+ return (extInfo != null) ? extInfo.getEntity() : null;
+ }
+
+ public int size() {
+ return this.streamSize;
+ }
+
+ private String moveNext() {
+ try {
+ moveNextEntry();
+ return entitiesArrayParser.next();
+ } catch (IOException e) {
+ LOG.error("moveNext failed!", e);
+ }
+
+ return null;
+ }
+
+ private void moveNextEntry() throws IOException {
+ this.currentPosition++;
+ }
+
+ private void prepareStreamForFetch() throws AtlasBaseException, IOException {
+ ZipEntry zipEntryNext = zipInputStream.getNextEntry();
+ if (zipEntryNext == null) {
+ throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP, "Attempting to import empty ZIP.");
+ }
+
+ if (zipEntryNext.getName().equals(ZipExportFileNames.ATLAS_TYPESDEF_NAME.toEntryFileName())) {
+ String json = getJsonPayloadFromZipEntryStream(this.zipInputStream);
+ this.typesDef = AtlasType.fromJson(json, AtlasTypesDef.class);
+ }
+
+ zipEntryNext = zipInputStream.getNextEntry();
+ if (zipEntryNext.getName().equals(ZIP_ENTRY_ENTITIES)) {
+ this.entitiesArrayParser = new EntitiesArrayParser(zipInputStream);
+ } else {
+ throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP, "Attempting to import empty ZIP. " + ZIP_ENTRY_ENTITIES + " could not be found!");
+ }
+ }
+
+ private String getJsonPayloadFromZipEntryStream(ZipInputStream zipInputStream) {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ try {
+ IOUtils.copy(zipInputStream, bos);
+ } catch (IOException e) {
+ LOG.error("Streaming copying failed!", e);
+ return null;
+ }
+ return bos.toString();
+ }
+
+ static class EntitiesArrayParser {
+ private static final String EMPTY_OBJECT = "{}";
+
+ private final JsonFactory factory;
+ private final JsonParser parser;
+ private boolean hasNext;
+
+ public EntitiesArrayParser(InputStream inputStream) throws IOException {
+ this.factory = AtlasJson.getMapper().getFactory();
+ this.parser = factory.createParser(inputStream);
+
+ parseNext();
+ }
+
+ public String next() throws IOException {
+ JsonToken jsonToken = parseNext();
+ if (!hasNext) {
+ return null;
+ }
+
+ if (jsonToken != null && jsonToken == JsonToken.START_OBJECT) {
+ JsonNode node = parser.readValueAsTree();
+ return validate(node.toString());
+ }
+ return null;
+
+ }
+
+ private JsonToken parseNext() throws IOException {
+ JsonToken jsonToken = this.parser.nextToken();
+ hasNext = (jsonToken != null) && (jsonToken != JsonToken.END_ARRAY);
+ return jsonToken;
+ }
+
+ private String validate(String payload) {
+ if (payload.equals(EMPTY_OBJECT)) {
+ hasNext = false;
+ close();
+ return null;
+ }
+
+ return payload;
+ }
+
+ public boolean hasNext() {
+ return hasNext;
+ }
+
+ public void close() {
+ try {
+ this.parser.close();
+ } catch (IOException e) {
+ LOG.error("Error closing parser!", e);
+ }
+ }
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
index ca0bc41..f552525 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
@@ -84,10 +84,8 @@ public class ZipFileMigrationImporter implements Runnable {
}
private AtlasImportRequest getImportRequest() throws AtlasException {
- return new AtlasImportRequest();
- }
-
- private String getPropertyValue(String property, String defaultValue) throws AtlasException {
- return ApplicationProperties.get().getString(property, defaultValue);
+ AtlasImportRequest request = new AtlasImportRequest();
+ request.setOption(AtlasImportRequest.OPTION_KEY_FORMAT, AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT);
+ return request;
}
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index 570b1f8..a37f651 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -187,6 +187,11 @@ public class ImportServiceTest extends AtlasTestBase {
return getZipSource("salesNewTypeAttrs-next.zip");
}
+ @DataProvider(name = "zip-direct-3")
+ public static Object[][] getZipDirect3(ITestContext context) throws IOException, AtlasBaseException {
+ return getZipSource("zip-direct-3.zip");
+ }
+
@Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
public void importDB5(InputStream inputStream) throws AtlasBaseException, IOException {
final String newEnumDefName = "database_action";
@@ -347,6 +352,16 @@ public class ImportServiceTest extends AtlasTestBase {
}
}
+ @Test(dataProvider = "zip-direct-3", expectedExceptions = AtlasBaseException.class)
+ public void zipDirectSample(InputStream inputStream) throws IOException, AtlasBaseException {
+ loadBaseModel();
+ loadFsModel();
+
+ AtlasImportRequest request = new AtlasImportRequest();
+ request.setOption(AtlasImportRequest.OPTION_KEY_FORMAT, AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT);
+ runImportWithParameters(importService, request, inputStream);
+ }
+
@DataProvider(name = "relationshipLineage")
public static Object[][] getImportWithRelationships(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("rel-lineage.zip");
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipDirectTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipDirectTest.java
new file mode 100644
index 0000000..faa31c3
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipDirectTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+
+public class ZipDirectTest {
+ @Test(expectedExceptions = AtlasBaseException.class)
+ public void loadFileEmpty() throws IOException, AtlasBaseException {
+ InputStream inputStream = ZipFileResourceTestUtils.getFileInputStream("zip-direct-1.zip");
+ new ZipSourceDirect(inputStream, 1);
+ }
+
+ @Test
+ public void loadFile() throws IOException, AtlasBaseException {
+ final int EXPECTED_ENTITY_COUNT = 3;
+
+ InputStream inputStream = ZipFileResourceTestUtils.getFileInputStream("zip-direct-2.zip");
+ ZipSourceDirect zipSourceDirect = new ZipSourceDirect(inputStream, EXPECTED_ENTITY_COUNT);
+
+ assertNotNull(zipSourceDirect);
+ assertNotNull(zipSourceDirect.getTypesDef());
+ assertTrue(zipSourceDirect.getTypesDef().getEntityDefs().size() > 0);
+ assertNotNull(zipSourceDirect.getExportResult());
+
+ int count = 0;
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo;
+ while((entityWithExtInfo = zipSourceDirect.getNextEntityWithExtInfo()) != null) {
+ assertNotNull(entityWithExtInfo);
+ count++;
+ }
+
+ assertEquals(count, EXPECTED_ENTITY_COUNT);
+ }
+
+ @Test
+ public void entitiesParserTest() throws IOException {
+ String object1 = "{\"type\":\"hdfs_path\"}";
+ String object2 = "{\"type\":\"hive_db\"}";
+ String entities = "[" + object1 + "," + object2 + ",{}]";
+ InputStream inputStream = new ByteArrayInputStream(entities.getBytes());
+ ZipSourceDirect.EntitiesArrayParser entitiesArrayParser = new ZipSourceDirect.EntitiesArrayParser(inputStream);
+
+ Object o = entitiesArrayParser.next();
+
+ assertNotNull(o);
+ assertEquals(o, object1);
+
+ o = entitiesArrayParser.next();
+ assertEquals(o, object2);
+
+ o = entitiesArrayParser.next();
+ assertNull(o);
+ }
+}
[atlas] 13/20: ATLAS-3762: Improve Edge creator using Genuine
iterator.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 43c9e32c89263a8e98c98dccc47f6b5ae7a3ceba
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Wed Apr 29 16:32:29 2020 -0700
ATLAS-3762: Improve Edge creator using Genuine iterator.
(cherry picked from commit 25f3002e0e84927eb39cebb5708d77ef81755d79)
---
.../atlas/repository/graphdb/AtlasVertex.java | 10 +++++
.../repository/graphdb/janus/AtlasJanusGraph.java | 36 +++++++++++++++---
.../repository/graphdb/janus/AtlasJanusVertex.java | 17 +++++++++
.../apache/atlas/repository/graph/GraphHelper.java | 25 ++++++++++++-
.../store/graph/v2/AtlasRelationshipStoreV2.java | 43 ++++++++++++++++------
5 files changed, 111 insertions(+), 20 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
index 9406e26..20e6177 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasVertex.java
@@ -45,6 +45,16 @@ public interface AtlasVertex<V, E> extends AtlasElement {
*/
Iterable<AtlasEdge<V, E>> getEdges(AtlasEdgeDirection direction, String[] edgeLabels);
+ long getEdgesCount(AtlasEdgeDirection direction, String edgeLabel);
+
+ /**
+ * Does vertex have edges specified by the direction and label
+ * @param dir
+ * @param edgeLabel
+ * @return
+ */
+ boolean hasEdges(AtlasEdgeDirection dir, String edgeLabel);
+
/**
* Gets the edges associated with this vertex going the
* specified direction.
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index eb02062..a30dbc7 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.janus;
import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.atlas.ApplicationProperties;
@@ -25,7 +26,19 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
-import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasGraphTraversal;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQueryParameter;
+import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.GraphIndexQueryParameters;
+import org.apache.atlas.repository.graphdb.GremlinVersion;
import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.type.AtlasType;
@@ -62,9 +75,12 @@ import javax.script.ScriptEngine;
import javax.script.ScriptException;
import java.io.IOException;
import java.io.OutputStream;
-import java.util.*;
-import java.util.stream.Collectors;
-import java.util.stream.StreamSupport;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
@@ -403,7 +419,11 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterable<? extends Vertex> it) {
- return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)).collect(Collectors.toList());
+
+ return Iterables.transform(it,
+ (Function<Vertex, AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>>) input ->
+ GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input));
+
}
public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> wrapEdges(Iterator<? extends Edge> it) {
@@ -413,7 +433,11 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> wrapEdges(Iterable<? extends Edge> it) {
- return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createEdge(AtlasJanusGraph.this, input)).collect(Collectors.toList());
+
+ return Iterables.transform(it,
+ (Function<Edge, AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>>) input ->
+ GraphDbObjectFactory.createEdge(AtlasJanusGraph.this, input));
+
}
public void addMultiProperties(Set<String> names) {
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
index fdc9fd0..b6e2c26 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusVertex.java
@@ -20,12 +20,14 @@ package org.apache.atlas.repository.graphdb.janus;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
+import java.util.stream.StreamSupport;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
+import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -78,6 +80,21 @@ public class AtlasJanusVertex extends AtlasJanusElement<Vertex> implements Atlas
return graph.wrapEdges(edges);
}
+ @Override
+ public long getEdgesCount(AtlasEdgeDirection dir, String edgeLabel) {
+ Direction direction = AtlasJanusObjectFactory.createDirection(dir);
+ Iterator<Edge> it = getWrappedElement().edges(direction, edgeLabel);
+ IteratorToIterableAdapter<Edge> iterable = new IteratorToIterableAdapter<>(it);
+ return StreamSupport.stream(iterable.spliterator(), true).count();
+ }
+
+ @Override
+ public boolean hasEdges(AtlasEdgeDirection dir, String edgeLabel) {
+ Direction direction = AtlasJanusObjectFactory.createDirection(dir);
+ Iterator<Edge> edges = getWrappedElement().edges(direction, edgeLabel);
+ return edges.hasNext();
+ }
+
private JanusGraphVertex getAsJanusVertex() {
return (JanusGraphVertex)getWrappedElement();
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 2b8227a..b9e3a5e 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -35,7 +35,6 @@ import org.apache.atlas.repository.graphdb.AtlasVertexQuery;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
-import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.utils.AtlasPerfMetrics;
import org.apache.atlas.v1.model.instance.Id;
import org.apache.atlas.v1.model.instance.Referenceable;
@@ -72,7 +71,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.UUID;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
@@ -269,6 +267,21 @@ public final class GraphHelper {
return ret;
}
+ public static long getAdjacentEdgesCountByLabel(AtlasVertex instanceVertex, AtlasEdgeDirection direction, final String edgeLabel) {
+ AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAdjacentEdgesCountByLabel");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Finding edges for {} with label {}", string(instanceVertex), edgeLabel);
+ }
+
+ long ret = 0;
+ if(instanceVertex != null && edgeLabel != null) {
+ ret = instanceVertex.getEdgesCount(direction, edgeLabel);
+ }
+
+ RequestContext.get().endMetricRecord(metric);
+ return ret;
+ }
+
public static boolean isPropagationEnabled(AtlasVertex classificationVertex) {
boolean ret = false;
@@ -437,6 +450,14 @@ public final class GraphHelper {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
}
+ public static long getOutGoingEdgesCountByLabel(AtlasVertex instanceVertex, String edgeLabel) {
+ return getAdjacentEdgesCountByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
+ }
+
+ public static long getInComingEdgesCountByLabel(AtlasVertex instanceVertex, String edgeLabel) {
+ return getAdjacentEdgesCountByLabel(instanceVertex, AtlasEdgeDirection.IN, edgeLabel);
+ }
+
public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel, AtlasRelationshipEdgeDirection edgeDirection) {
AtlasEdge ret;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index d1c1f12..ab431bc 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -19,6 +19,7 @@
package org.apache.atlas.repository.store.graph.v2;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.authorize.AtlasPrivilege;
@@ -49,6 +50,7 @@ import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.utils.AtlasPerfMetrics;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
@@ -67,24 +69,25 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
+import java.util.function.Function;
+import static org.apache.atlas.AtlasConfiguration.NOTIFICATION_RELATIONSHIPS_ENABLED;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
-import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.*;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.BOTH;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.NONE;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.HOME_ID_KEY;
import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY;
-import static org.apache.atlas.AtlasConfiguration.NOTIFICATION_RELATIONSHIPS_ENABLED;
-
-
import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassificationIds;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications;
-import static org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTypeName;
@@ -776,23 +779,39 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
}
public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String relationshipLabel) {
- AtlasEdge ret = null;
- Iterator<AtlasEdge> edgesIterator = getIncomingEdgesByLabel(toVertex, relationshipLabel);
+ AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getRelationshipEdge");
+
+ AtlasEdge ret = null;
+
+ if (toVertex.hasEdges(AtlasEdgeDirection.IN, relationshipLabel) && fromVertex.hasEdges(AtlasEdgeDirection.OUT, relationshipLabel)) {
+ long fromVertexOutgoingEdgeCount = graphHelper.getOutGoingEdgesCountByLabel(fromVertex, relationshipLabel);
+ long toVertexIncomingEdgeCount = graphHelper.getInComingEdgesCountByLabel(toVertex, relationshipLabel);
+ if (toVertexIncomingEdgeCount < fromVertexOutgoingEdgeCount) {
+ Iterator<AtlasEdge> edgesIteratorIn = graphHelper.getIncomingEdgesByLabel(toVertex, relationshipLabel);
+ ret = getActiveEdgeFromList(edgesIteratorIn, fromVertex.getId(), e -> e.getOutVertex().getId());
+ } else {
+ Iterator<AtlasEdge> edgesIteratorOut = graphHelper.getOutGoingEdgesByLabel(fromVertex, relationshipLabel);
+ ret = getActiveEdgeFromList(edgesIteratorOut, toVertex.getId(), e -> e.getInVertex().getId());
+ }
+ }
+ RequestContext.get().endMetricRecord(metric);
+ return ret;
+ }
+
+ private AtlasEdge getActiveEdgeFromList(Iterator<AtlasEdge> edgesIterator, Object vertexIdToCompare, Function<AtlasEdge, Object> edgeIdFn) {
while (edgesIterator != null && edgesIterator.hasNext()) {
AtlasEdge edge = edgesIterator.next();
-
if (edge != null) {
Status status = graphHelper.getStatus(edge);
- if ((status == null || status == ACTIVE) && edge.getOutVertex().getId().equals(fromVertex.getId())) {
- ret = edge;
- break;
+ if ((status == null || status == ACTIVE) && edgeIdFn.apply(edge).equals(vertexIdToCompare)) {
+ return edge;
}
}
}
- return ret;
+ return null;
}
private Long getRelationshipVersion(AtlasRelationship relationship) {
[atlas] 10/20: ATLAS-3674: ZipFileMigrationImporter: Set Shell
Entity Creation.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 80239468dc78e52e5edece828f7495a5abde3273
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Fri Mar 20 13:06:01 2020 -0700
ATLAS-3674: ZipFileMigrationImporter: Set Shell Entity Creation.
(cherry picked from commit dff690a00e3ef690ffd584b07d243586d64b31b7)
---
.../atlas/repository/impexp/ZipSourceDirect.java | 8 ++-----
.../migration/DataMigrationStatusService.java | 4 ++--
.../graph/v2/bulkimport/pc/EntityConsumer.java | 7 +++++-
.../v2/bulkimport/pc/EntityCreationManager.java | 28 ++++++++++------------
.../impexp/DataMigrationStatusServiceTest.java | 2 +-
5 files changed, 23 insertions(+), 26 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
index 75b8e9e..04342fa 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
@@ -175,12 +175,8 @@ public class ZipSourceDirect implements EntityImportStream {
@Override
public void setPosition(int index) {
- try {
- for (int i = 0; i < index; i++) {
- moveNextEntry();
- }
- } catch (IOException e) {
- LOG.error("Error setting position: {}. Position may be beyond the stream size.", index);
+ for (int i = 0; i < index; i++) {
+ moveNext();
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
index 8c7a3a8..b891684 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
@@ -99,8 +99,8 @@ public class DataMigrationStatusService {
this.status = null;
}
- public void savePosition(String position) {
- this.status.setCurrentIndex(Long.valueOf(position));
+ public void savePosition(Long position) {
+ this.status.setCurrentIndex(position);
this.migrationStatusVertexManagement.updateVertexPartial(this.status);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
index d0fac10..fc7d392 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
@@ -90,6 +90,8 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
private void processEntity(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, long currentCount) {
try {
RequestContext.get().setImportInProgress(true);
+ RequestContext.get().setCreateShellEntityForNonExistingReference(true);
+
AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
LOG.debug("Processing: {}", currentCount);
@@ -163,7 +165,10 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
}
private void retryProcessEntity(int retryCount) {
- LOG.info("Replaying: Starting!: Buffer: {}: Retry count: {}", entityBuffer.size(), retryCount);
+ if (LOG.isDebugEnabled() || retryCount > 1) {
+ LOG.info("Replaying: Starting!: Buffer: {}: Retry count: {}", entityBuffer.size(), retryCount);
+ }
+
for (AtlasEntity.AtlasEntityWithExtInfo e : entityBuffer) {
processEntity(e, counter.get());
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
index 177b563..734add6 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
@@ -35,7 +35,7 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
private static final String WORKER_PREFIX = "migration-import";
private static final long STATUS_REPORT_TIMEOUT_DURATION = 1 * 60 * 1000; // 5 min
- private final StatusReporter<String, String> statusReporter;
+ private final StatusReporter<String, Long> statusReporter;
private final AtlasImportResult importResult;
private final DataMigrationStatusService dataMigrationStatusService;
private String currentTypeName;
@@ -51,7 +51,7 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
}
public long read(EntityImportStream entityStream) {
- long currentIndex = this.dataMigrationStatusService.getStatus().getCurrentIndex();
+ long currentIndex = entityStream.getPosition();
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo;
this.entityImportStream = entityStream;
this.dataMigrationStatusService.setStatus("IN_PROGRESS");
@@ -68,6 +68,8 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
break;
}
}
+
+ this.dataMigrationStatusService.setStatus("DONE");
return currentIndex;
}
@@ -83,7 +85,7 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
}
setCurrentTypeName(typeName);
- statusReporter.produced(entityWithExtInfo.getEntity().getGuid(), String.format("%s:%s", entityWithExtInfo.getEntity().getTypeName(), currentIndex));
+ statusReporter.produced(entityWithExtInfo.getEntity().getGuid(), currentIndex);
super.checkProduce(entityWithExtInfo);
extractResults();
}
@@ -98,25 +100,19 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
}
private void logStatus() {
- String ack = statusReporter.ack();
- if (StringUtils.isEmpty(ack)) {
- return;
- }
-
- String[] split = ack.split(":");
- if (split.length == 0 || split.length < 2) {
+ Long ack = statusReporter.ack();
+ if (ack == null) {
return;
}
- importResult.incrementMeticsCounter(split[0]);
- String currentPosition = split[1];
- dataMigrationStatusService.savePosition(currentPosition);
- this.currentPercent = updateImportMetrics(split[0], Integer.parseInt(currentPosition), this.entityImportStream.size(), getCurrentPercent());
+ importResult.incrementMeticsCounter(getCurrentTypeName());
+ dataMigrationStatusService.savePosition(ack);
+ this.currentPercent = updateImportMetrics(getCurrentTypeName(), ack, this.entityImportStream.size(), getCurrentPercent());
}
- private static float updateImportMetrics(String typeNameGuid, int currentIndex, int streamSize, float currentPercent) {
+ private static float updateImportMetrics(String typeNameGuid, long currentIndex, int streamSize, float currentPercent) {
String lastEntityImported = String.format("entity:last-imported:%s:(%s)", typeNameGuid, currentIndex);
- return BulkImporterImpl.updateImportProgress(LOG, currentIndex, streamSize, currentPercent, lastEntityImported);
+ return BulkImporterImpl.updateImportProgress(LOG, (int) currentIndex, streamSize, currentPercent, lastEntityImported);
}
private String getCurrentTypeName() {
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
index bf1d9a0..5f1cd0b 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
@@ -54,7 +54,7 @@ public class DataMigrationStatusServiceTest {
assertEquals(ret.getTotalCount(), expected.getTotalCount());
assertEquals(ret.getCurrentIndex(), expected.getCurrentIndex());
- dataMigrationStatusService.savePosition("100");
+ dataMigrationStatusService.savePosition(100l);
assertNotNull(dataMigrationStatusService.getStatus());
assertNotNull(dataMigrationStatusService.getStatus().getCurrentIndex(), "100");
assertNotNull(dataMigrationStatusService.getCreate(expected).getCurrentIndex(), "100");
[atlas] 03/20: ATLAS-3595,
ATLAS-3603: Auto-start migration import for ZIP file-based
migration.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 1ae8dc9f9adc233ca36f56304ef624610b18e050
Author: Nikhil Bonte <ni...@freestoneinfotech.com>
AuthorDate: Fri Feb 14 09:07:42 2020 -0800
ATLAS-3595, ATLAS-3603: Auto-start migration import for ZIP file-based migration.
Signed-off-by: Ashutosh Mestry <am...@cloudera.com>
(cherry picked from commit 3e035c45a9dd695471d0927bbcdd72108094c828)
---
.../java/org/apache/atlas/service/Services.java | 16 ++-
.../repository/migration/DataMigrationService.java | 19 ++-
.../atlas/repository/migration/FileWatcher.java | 149 +++++++++++++++++++++
.../migration/ZipFileMigrationImporter.java | 93 +++++++++++++
4 files changed, 271 insertions(+), 6 deletions(-)
diff --git a/common/src/main/java/org/apache/atlas/service/Services.java b/common/src/main/java/org/apache/atlas/service/Services.java
index 7b36db5..75eba03 100644
--- a/common/src/main/java/org/apache/atlas/service/Services.java
+++ b/common/src/main/java/org/apache/atlas/service/Services.java
@@ -40,10 +40,12 @@ import static org.apache.atlas.AtlasConstants.ATLAS_SERVICES_ENABLED;
public class Services {
public static final Logger LOG = LoggerFactory.getLogger(Services.class);
private static final String DATA_MIGRATION_CLASS_NAME_DEFAULT = "DataMigrationService";
+ private static final String FILE_EXTENSION_ZIP = ".zip";
private final List<Service> services;
private final String dataMigrationClassName;
private final boolean servicesEnabled;
+ private final String migrationDirName;
private final boolean migrationEnabled;
@Inject
@@ -51,7 +53,8 @@ public class Services {
this.services = services;
this.dataMigrationClassName = configuration.getString("atlas.migration.class.name", DATA_MIGRATION_CLASS_NAME_DEFAULT);
this.servicesEnabled = configuration.getBoolean(ATLAS_SERVICES_ENABLED, true);
- this.migrationEnabled = StringUtils.isNotEmpty(configuration.getString(ATLAS_MIGRATION_MODE_FILENAME));
+ this.migrationDirName = configuration.getString(ATLAS_MIGRATION_MODE_FILENAME);
+ this.migrationEnabled = StringUtils.isNotEmpty(migrationDirName);
}
@PostConstruct
@@ -92,11 +95,22 @@ public class Services {
private boolean isServiceUsed(Service service) {
if (isDataMigrationService(service)) {
return migrationEnabled;
+ } else if (isZipFileMigration()) {
+ return isNeededForZipFileMigration(service);
} else {
return !migrationEnabled && servicesEnabled;
}
}
+ private boolean isZipFileMigration() {
+ return migrationEnabled && StringUtils.endsWithIgnoreCase(migrationDirName, FILE_EXTENSION_ZIP);
+ }
+
+ private boolean isNeededForZipFileMigration(Service svc) {
+ return svc.getClass().getSuperclass().getSimpleName().equals("AbstractStorageBasedAuditRepository") ||
+ svc.getClass().getSuperclass().getSimpleName().equals("AbstractNotification");
+ }
+
private boolean isDataMigrationService(Service svc) {
return svc.getClass().getSimpleName().equals(dataMigrationClassName);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
index 1701361..0a2257e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
@@ -18,20 +18,21 @@
package org.apache.atlas.repository.migration;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.GraphDBMigrator;
+import org.apache.atlas.repository.impexp.ImportService;
import org.apache.atlas.repository.impexp.ImportTypeDefProcessor;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.service.Service;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.configuration.Configuration;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.service.Service;
import org.apache.commons.io.FileUtils;
-import org.apache.solr.common.StringUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -48,6 +49,8 @@ import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME;
public class DataMigrationService implements Service {
private static final Logger LOG = LoggerFactory.getLogger(DataMigrationService.class);
+ private static final String FILE_EXTENSION_ZIP = ".zip";
+
private static String ATLAS_MIGRATION_DATA_NAME = "atlas-migration-data.json";
private static String ATLAS_MIGRATION_TYPESDEF_NAME = "atlas-migration-typesdef.json";
@@ -57,9 +60,15 @@ public class DataMigrationService implements Service {
@Inject
public DataMigrationService(GraphDBMigrator migrator, AtlasTypeDefStore typeDefStore, Configuration configuration,
GraphBackedSearchIndexer indexer, AtlasTypeDefStoreInitializer storeInitializer,
- AtlasTypeRegistry typeRegistry) {
+ AtlasTypeRegistry typeRegistry, ImportService importService) {
this.configuration = configuration;
- this.thread = new Thread(new FileImporter(migrator, typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
+
+
+ String fileName = getFileName();
+ boolean zipFileBasedMigrationImport = StringUtils.endsWithIgnoreCase(fileName, FILE_EXTENSION_ZIP);
+ this.thread = (zipFileBasedMigrationImport)
+ ? new Thread(new ZipFileMigrationImporter(importService, fileName), "zipFileBasedMigrationImporter")
+ : new Thread(new FileImporter(migrator, typeDefStore, typeRegistry, storeInitializer, fileName, indexer));
}
@Override
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/FileWatcher.java b/repository/src/main/java/org/apache/atlas/repository/migration/FileWatcher.java
new file mode 100644
index 0000000..d40015a
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/FileWatcher.java
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.migration;
+
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.nio.file.StandardWatchEventKinds;
+import java.nio.file.WatchEvent;
+import java.nio.file.WatchKey;
+import java.nio.file.WatchService;
+
+public class FileWatcher {
+ private static final Logger LOG = LoggerFactory.getLogger(FileWatcher.class);
+
+ private final static int MAX_TIMES_PAUSE = 10;
+ private final static int PAUSE_INTERVAL = 5000; // 5 secs
+
+ private int checkIncrement;
+ private final File fileToWatch;
+
+ public FileWatcher(String filePath) {
+ this.checkIncrement = 1;
+ this.fileToWatch = new File(filePath);
+ }
+
+ public void start() throws IOException {
+ if (existsAndReadyCheck()) {
+ return;
+ }
+
+ WatchService watcher = FileSystems.getDefault().newWatchService();
+ Path pathToWatch = FileSystems.getDefault().getPath(fileToWatch.getParent());
+ register(watcher, pathToWatch);
+
+ try {
+ LOG.info(String.format("Migration File Watcher: Watching: %s", fileToWatch.toString()));
+ startWatching(watcher);
+ } catch (InterruptedException ex) {
+ LOG.error("Migration File Watcher: Interrupted!");
+ } finally {
+ watcher.close();
+ }
+ }
+
+ private void startWatching(WatchService watcher) throws InterruptedException {
+ while (true) {
+ WatchKey watchKey = watcher.take();
+ if (watchKey == null) {
+ continue;
+ }
+
+ for (WatchEvent event : watchKey.pollEvents()) {
+ if (checkIfFileAvailableAndReady(event)) {
+ return;
+ }
+ }
+
+ watchKey.reset();
+ }
+ }
+
+ private void register(WatchService watcher, Path path) throws IOException {
+ try {
+ path.register(watcher, StandardWatchEventKinds.ENTRY_CREATE, StandardWatchEventKinds.ENTRY_MODIFY);
+ } catch (IOException e) {
+ LOG.error("Migration File Watcher: Error while registering event {}!", path);
+ throw e;
+ }
+ }
+
+ private boolean checkIfFileAvailableAndReady(WatchEvent event) {
+ WatchEvent<Path> watchEvent = event;
+ Path path = watchEvent.context();
+
+ if (!path.toString().equals(fileToWatch.getName())) {
+ return false;
+ }
+
+ return existsAndReadyCheck();
+ }
+
+ private boolean existsAndReadyCheck() {
+ boolean ret = fileToWatch.exists() && fileToWatch.canRead();
+ if (ret) {
+ try {
+ return isReadyForUse(fileToWatch);
+ } catch (InterruptedException e) {
+ LOG.error("Migration File Watcher: Interrupted {}!", fileToWatch);
+ return false;
+ }
+ } else {
+ LOG.info(String.format("Migration File Watcher: File does not exist!: %s", fileToWatch.getAbsolutePath()));
+ }
+
+ return ret;
+ }
+
+ private boolean isReadyForUse(File file) throws InterruptedException {
+ Long fileSizeBefore = file.length();
+ Thread.sleep(getCheckInterval());
+ Long fileSizeAfter = file.length();
+ boolean ret = fileSizeBefore.equals(fileSizeAfter);
+
+ if (ret) {
+ LOG.info(String.format("Migration File Watcher: %s: File is ready for use!", file.getAbsolutePath()));
+ } else {
+ incrementCheckCounter();
+ LOG.info(
+ String.format("Migration File Watcher: File is being written: Pause: %,d secs: New size: %,d."
+ , getCheckInterval() / 1000
+ , fileSizeAfter));
+ }
+
+ return ret;
+ }
+
+ private int getCheckInterval() {
+ return (PAUSE_INTERVAL * (checkIncrement));
+ }
+
+ private int incrementCheckCounter() {
+ if (checkIncrement > MAX_TIMES_PAUSE) {
+ checkIncrement = 1;
+ }
+
+ return (PAUSE_INTERVAL * (checkIncrement++));
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
new file mode 100644
index 0000000..ca0bc41
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.repository.impexp.ImportService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+
+public class ZipFileMigrationImporter implements Runnable {
+ private static final Logger LOG = LoggerFactory.getLogger(ZipFileMigrationImporter.class);
+
+ private static String ENV_USER_NAME = "user.name";
+
+ private final ImportService importService;
+ private final String fileToImport;
+
+ public ZipFileMigrationImporter(ImportService importService, String fileName) {
+ this.importService = importService;
+ this.fileToImport = fileName;
+ }
+
+ @Override
+ public void run() {
+ try {
+ FileWatcher fileWatcher = new FileWatcher(fileToImport);
+ fileWatcher.start();
+
+ performImport(new FileInputStream(new File(fileToImport)));
+ } catch (IOException e) {
+ LOG.error("Migration Import: IO Error!", e);
+ } catch (AtlasBaseException e) {
+ LOG.error("Migration Import: Error!", e);
+ }
+ }
+
+ private void performImport(InputStream fs) throws AtlasBaseException {
+ try {
+ LOG.info("Migration Import: {}: Starting...", fileToImport);
+
+ RequestContext.get().setUser(getUserNameFromEnvironment(), null);
+
+ importService.run(fs, getImportRequest(),
+ getUserNameFromEnvironment(),
+ InetAddress.getLocalHost().getHostName(),
+ InetAddress.getLocalHost().getHostAddress());
+
+ } catch (Exception ex) {
+ LOG.error("Error loading zip for migration", ex);
+ throw new AtlasBaseException(ex);
+ } finally {
+ LOG.info("Migration Import: {}: Done!", fileToImport);
+ }
+ }
+
+ private String getUserNameFromEnvironment() {
+ return System.getProperty(ENV_USER_NAME);
+ }
+
+ private AtlasImportRequest getImportRequest() throws AtlasException {
+ return new AtlasImportRequest();
+ }
+
+ private String getPropertyValue(String property, String defaultValue) throws AtlasException {
+ return ApplicationProperties.get().getString(property, defaultValue);
+ }
+}
[atlas] 05/20: ATLAS-3643: PC Fx: StatusReporter: Introduce Status
Reporting to Be used With PC Framework
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit fd39e4cf1dffd5535236263dedfb87e42fe62eb5
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon Mar 2 08:05:55 2020 -0800
ATLAS-3643: PC Fx: StatusReporter: Introduce Status Reporting to Be used With PC Framework
(cherry picked from commit ed2755dcce37318ffce21e5ecd7692416aa49a1d)
---
.../java/org/apache/atlas/pc/StatusReporter.java | 75 +++++++++++++++++
.../org/apache/atlas/pc/StatusReporterTest.java | 94 ++++++++++++++++++++++
2 files changed, 169 insertions(+)
diff --git a/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java b/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
new file mode 100644
index 0000000..f84e8d0
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.pc;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class StatusReporter<T, U> {
+ private Map<T,U> producedItems = new LinkedHashMap<>();
+ private Set<T> processedSet = new HashSet<>();
+
+ public void produced(T item, U index) {
+ this.producedItems.put(item, index);
+ }
+
+ public void processed(T item) {
+ this.processedSet.add(item);
+ }
+
+ public void processed(T[] index) {
+ this.processedSet.addAll(Arrays.asList(index));
+ }
+
+ public U ack() {
+ U ack = null;
+ U ret;
+ do {
+ ret = completionIndex(getFirstElement(this.producedItems));
+ if (ret != null) {
+ ack = ret;
+ }
+ } while(ret != null);
+
+ return ack;
+ }
+
+ private Map.Entry<T, U> getFirstElement(Map<T, U> map) {
+ if (map.isEmpty()) {
+ return null;
+ }
+
+ return map.entrySet().iterator().next();
+ }
+
+ private U completionIndex(Map.Entry<T, U> lookFor) {
+ U ack = null;
+ if (lookFor == null || !processedSet.contains(lookFor.getKey())) {
+ return ack;
+ }
+
+ ack = lookFor.getValue();
+ producedItems.remove(lookFor.getKey());
+ processedSet.remove(lookFor);
+ return ack;
+ }
+}
diff --git a/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java b/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
new file mode 100644
index 0000000..3e50562
--- /dev/null
+++ b/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.pc;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.testng.annotations.Test;
+
+import java.util.concurrent.BlockingQueue;
+
+import static org.testng.Assert.assertEquals;
+
+public class StatusReporterTest {
+ private static class IntegerConsumer extends WorkItemConsumer<Integer> {
+ private static ThreadLocal<Integer> payload = new ThreadLocal<Integer>();
+ private Integer current;
+
+ public IntegerConsumer(BlockingQueue<Integer> queue) {
+ super(queue);
+ }
+
+ @Override
+ protected void doCommit() {
+ addResult(current);
+ }
+
+ @Override
+ protected void processItem(Integer item) {
+ try {
+ this.current = item;
+ Thread.sleep(20 + RandomUtils.nextInt(5, 7));
+ super.commit();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private class IntegerConsumerBuilder implements WorkItemBuilder<IntegerConsumer, Integer> {
+ @Override
+ public IntegerConsumer build(BlockingQueue<Integer> queue) {
+ return new IntegerConsumer(queue);
+ }
+ }
+
+ private WorkItemManager<Integer, WorkItemConsumer> getWorkItemManger(IntegerConsumerBuilder cb, int numWorkers) {
+ return new WorkItemManager<>(cb, "IntegerConsumer", 5, numWorkers, true);
+ }
+
+ @Test
+ public void statusReporting() throws InterruptedException {
+ final int maxItems = 50;
+
+ IntegerConsumerBuilder cb = new IntegerConsumerBuilder();
+ WorkItemManager<Integer, WorkItemConsumer> wi = getWorkItemManger(cb, 5);
+ StatusReporter<Integer, Integer> statusReporter = new StatusReporter<>();
+
+ for (int i = 0; i < maxItems; i++) {
+ wi.produce(i);
+ statusReporter.produced(i, i);
+
+ extractResults(wi, statusReporter);
+ }
+
+ wi.drain();
+ extractResults(wi, statusReporter);
+ assertEquals(statusReporter.ack().intValue(), (maxItems - 1));
+ wi.shutdown();
+ }
+
+ private void extractResults(WorkItemManager<Integer, WorkItemConsumer> wi, StatusReporter<Integer, Integer> statusReporter) {
+ Object result = null;
+ while((result = wi.getResults().poll()) != null) {
+ if (result == null || !(result instanceof Integer)) {
+ continue;
+ }
+ statusReporter.processed((Integer) result);
+ }
+ }
+}
[atlas] 12/20: ATLAS-3737: Refactoring: Support multiple instances
of AtlasGraph
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit f096f93fdb23a9d1a3b4525307ba539f41a6e1c6
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Wed Apr 22 23:07:46 2020 -0700
ATLAS-3737: Refactoring: Support multiple instances of AtlasGraph
(cherry picked from commit 2b38550b88ea201a3bd9169c44c0ede900ec1dd8)
---
.../repository/graphdb/janus/AtlasJanusGraph.java | 4 +-
.../graphdb/janus/query/NativeJanusGraphQuery.java | 2 +-
.../atlas/discovery/EntityDiscoveryService.java | 2 +-
.../atlas/discovery/EntityLineageService.java | 6 +-
.../converters/AtlasInstanceConverter.java | 7 +-
.../atlas/repository/graph/FullTextMapperV2.java | 5 +-
.../repository/graph/GraphBackedSearchIndexer.java | 69 ++++++++++++
.../apache/atlas/repository/graph/GraphHelper.java | 84 +++------------
.../atlas/repository/impexp/ExportService.java | 8 +-
.../store/graph/v1/DeleteHandlerDelegate.java | 11 +-
.../repository/store/graph/v1/DeleteHandlerV1.java | 15 +--
.../store/graph/v1/HardDeleteHandlerV1.java | 5 +-
.../store/graph/v1/SoftDeleteHandlerV1.java | 5 +-
.../graph/v2/AtlasEntityGraphDiscoveryV2.java | 9 +-
.../store/graph/v2/AtlasEntityStoreV2.java | 67 ++++++------
.../store/graph/v2/AtlasGraphUtilsV2.java | 108 +++++++++++--------
.../store/graph/v2/AtlasRelationshipStoreV2.java | 24 +++--
.../store/graph/v2/BulkImporterImpl.java | 15 +--
.../store/graph/v2/ClassificationAssociator.java | 25 +++--
.../store/graph/v2/EntityGraphMapper.java | 88 +++++++++-------
.../store/graph/v2/EntityGraphRetriever.java | 30 +++---
.../store/graph/v2/EntityStateChecker.java | 10 +-
.../store/graph/v2/IDBasedEntityResolver.java | 11 +-
.../graph/v2/UniqAttrBasedEntityResolver.java | 9 +-
.../store/graph/v2/bulkimport/MigrationImport.java | 48 +++++----
.../store/graph/v2/bulkimport/RegularImport.java | 12 ++-
.../graph/v2/bulkimport/pc/EntityConsumer.java | 116 ++++++++++++++++-----
.../v2/bulkimport/pc/EntityConsumerBuilder.java | 25 +++--
.../repository/impexp/ExportSkipLineageTest.java | 6 +-
.../repository/migration/HiveParititionTest.java | 6 +-
.../atlas/repository/migration/HiveStocksTest.java | 6 +-
.../repository/migration/MigrationBaseAsserts.java | 22 ++--
.../store/graph/v2/AtlasEntityStoreV2Test.java | 2 +-
.../store/graph/v2/AtlasEntityTestBase.java | 2 +-
.../graph/v2/AtlasRelationshipStoreV2Test.java | 8 +-
.../graph/v2/ClassificationAssociatorTest.java | 25 ++---
36 files changed, 544 insertions(+), 353 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index 0176ba7..eb02062 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -319,8 +319,8 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
}
- private JanusGraph getGraph() {
- return getGraphInstance();
+ public JanusGraph getGraph() {
+ return this.janusGraph;
}
@Override
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
index 2485938..16fc445 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/query/NativeJanusGraphQuery.java
@@ -56,7 +56,7 @@ public class NativeJanusGraphQuery implements NativeTinkerpopGraphQuery<AtlasJan
private JanusGraphQuery<?> query;
public NativeJanusGraphQuery(AtlasJanusGraph graph) {
- this.query = AtlasJanusGraphDatabase.getGraphInstance().query();
+ this.query = graph.getGraph().query();
this.graph = graph;
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 25042c1..dd4d1b4 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -99,7 +99,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
SearchTracker searchTracker,
UserProfileService userProfileService) throws AtlasException {
this.graph = graph;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.entityRetriever = new EntityGraphRetriever(this.graph, typeRegistry);
this.indexer = indexer;
this.searchTracker = searchTracker;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index 5548740..d18ace8 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -94,7 +94,7 @@ public class EntityLineageService implements AtlasLineageService {
EntityLineageService(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) {
this.graph = atlasGraph;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.entityRetriever = new EntityGraphRetriever(atlasGraph, typeRegistry);
this.atlasTypeRegistry = typeRegistry;
}
@@ -259,7 +259,7 @@ public class EntityLineageService implements AtlasLineageService {
}
if (isDataSet) {
- AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
if (direction == INPUT || direction == BOTH) {
traverseEdges(datasetVertex, true, depth, ret);
@@ -269,7 +269,7 @@ public class EntityLineageService implements AtlasLineageService {
traverseEdges(datasetVertex, false, depth, ret);
}
} else {
- AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
// make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1'
if (direction == INPUT || direction == BOTH) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
index 172e187..6fc0c65 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
@@ -34,6 +34,7 @@ import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
import org.apache.atlas.model.instance.GuidMapping;
import org.apache.atlas.model.legacy.EntityResult;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.v1.model.instance.Referenceable;
import org.apache.atlas.v1.model.instance.Struct;
@@ -68,11 +69,11 @@ public class AtlasInstanceConverter {
private final EntityGraphRetriever entityGraphRetrieverIgnoreRelationshipAttrs;
@Inject
- public AtlasInstanceConverter(AtlasTypeRegistry typeRegistry, AtlasFormatConverters instanceFormatters) {
+ public AtlasInstanceConverter(AtlasGraph graph, AtlasTypeRegistry typeRegistry, AtlasFormatConverters instanceFormatters) {
this.typeRegistry = typeRegistry;
this.instanceFormatters = instanceFormatters;
- this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
- this.entityGraphRetrieverIgnoreRelationshipAttrs = new EntityGraphRetriever(typeRegistry, true);
+ this.entityGraphRetriever = new EntityGraphRetriever(graph, typeRegistry);
+ this.entityGraphRetrieverIgnoreRelationshipAttrs = new EntityGraphRetriever(graph, typeRegistry, true);
}
public Referenceable[] getReferenceables(Collection<AtlasEntity> entities) throws AtlasBaseException {
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index 417c96c..497a877 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -25,6 +25,7 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes;
@@ -68,13 +69,13 @@ public class FullTextMapperV2 implements IFullTextMapper {
@Inject
- public FullTextMapperV2(AtlasTypeRegistry typeRegistry, Configuration configuration) {
+ public FullTextMapperV2(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, Configuration configuration) {
this.typeRegistry = typeRegistry;
this.configuration = configuration;
followReferences = this.configuration != null && this.configuration.getBoolean(FULL_TEXT_FOLLOW_REFERENCES, false);
// If followReferences = false then ignore relationship attr loading
- entityGraphRetriever = new EntityGraphRetriever(typeRegistry, !followReferences);
+ entityGraphRetriever = new EntityGraphRetriever(atlasGraph, typeRegistry, !followReferences);
}
/**
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 21d467f..379a912 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -30,8 +30,11 @@ import org.apache.atlas.listener.ActiveStateChangeHandler;
import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
@@ -41,8 +44,12 @@ import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
+import org.apache.solr.common.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -192,6 +199,10 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
//resolve index fields names for the new entity attributes.
resolveIndexFieldNames(management, changedTypeDefs);
+
+ createEdgeLabels(management, changedTypeDefs.getCreatedTypeDefs());
+ createEdgeLabels(management, changedTypeDefs.getUpdatedTypeDefs());
+
//Commit indexes
commit(management);
} catch (RepositoryException | IndexException e) {
@@ -349,6 +360,13 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// create fulltext indexes
createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY, String.class, SINGLE);
+ createPropertyKey(management, IS_PROXY_KEY, Boolean.class, SINGLE);
+ createPropertyKey(management, PROVENANCE_TYPE_KEY, Integer.class, SINGLE);
+ createPropertyKey(management, HOME_ID_KEY, String.class, SINGLE);
+
+ createEdgeLabel(management, Constants.TERM_ASSIGNMENT_LABEL);
+ createEdgeLabel(management, Constants.CLASSIFICATION_LABEL);
+
commit(management);
LOG.info("Index creation for global keys complete.");
@@ -689,6 +707,14 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
String label = Constants.INTERNAL_PROPERTY_KEY_PREFIX + propertyName;
+ createEdgeLabelUsingLabelName(management, label);
+ }
+
+ private void createEdgeLabelUsingLabelName(final AtlasGraphManagement management, final String label) {
+ if (StringUtils.isEmpty(label)) {
+ return;
+ }
+
org.apache.atlas.repository.graphdb.AtlasEdgeLabel edgeLabel = management.getEdgeLabel(label);
if (edgeLabel == null) {
@@ -999,4 +1025,47 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+
+ private void createEdgeLabels(AtlasGraphManagement management, List<? extends AtlasBaseTypeDef> typeDefs) {
+ if (CollectionUtils.isEmpty(typeDefs)) {
+ return;
+ }
+
+ for (AtlasBaseTypeDef typeDef : typeDefs) {
+ if (typeDef instanceof AtlasEntityDef) {
+ AtlasEntityDef entityDef = (AtlasEntityDef) typeDef;
+ createEdgeLabelsForStruct(management, entityDef);
+ } else if (typeDef instanceof AtlasRelationshipDef) {
+ createEdgeLabels(management, (AtlasRelationshipDef) typeDef);
+ }
+ }
+ }
+
+ private void createEdgeLabelsForStruct(AtlasGraphManagement management, AtlasEntityDef entityDef) {
+ try {
+ AtlasType type = typeRegistry.getType(entityDef.getName());
+ if (!(type instanceof AtlasEntityType)) {
+ return;
+ }
+
+ AtlasEntityType entityType = (AtlasEntityType) type;
+ for (AtlasAttributeDef attributeDef : entityDef.getAttributeDefs()) {
+ AtlasAttribute attribute = entityType.getAttribute(attributeDef.getName());
+ if (attribute.getAttributeType().getTypeCategory() == TypeCategory.STRUCT) {
+ String relationshipLabel = attribute.getRelationshipEdgeLabel();
+ createEdgeLabelUsingLabelName(management, relationshipLabel);
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.error("Error fetching type: {}", entityDef.getName(), e);
+ }
+ }
+
+ private void createEdgeLabels(AtlasGraphManagement management, AtlasRelationshipDef relationshipDef) {
+ String relationshipTypeName = relationshipDef.getName();
+ AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationshipTypeName);
+ String relationshipLabel = relationshipType.getRelationshipLabel();
+
+ createEdgeLabelUsingLabelName(management, relationshipLabel);
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index dfaa739..2b8227a 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -95,18 +95,13 @@ public final class GraphHelper {
public static final String RETRY_DELAY = "atlas.graph.storage.retry.sleeptime.ms";
public static final String DEFAULT_REMOVE_PROPAGATIONS_ON_ENTITY_DELETE = "atlas.graph.remove.propagations.default";
- private final AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
-
- private static volatile GraphHelper INSTANCE;
-
private AtlasGraph graph;
- private static int maxRetries;
- private static long retrySleepTimeMillis;
- private static boolean removePropagations;
+ private int maxRetries = 3;
+ private long retrySleepTimeMillis = 1000;
+ private boolean removePropagations = false;
- @VisibleForTesting
- GraphHelper(AtlasGraph graph) {
+ public GraphHelper(AtlasGraph graph) {
this.graph = graph;
try {
maxRetries = ApplicationProperties.get().getInt(RETRY_COUNT, 3);
@@ -117,69 +112,10 @@ public final class GraphHelper {
}
}
- public static GraphHelper getInstance() {
- if ( INSTANCE == null) {
- synchronized (GraphHelper.class) {
- if (INSTANCE == null) {
- INSTANCE = new GraphHelper(AtlasGraphProvider.getGraphInstance());
- }
- }
- }
- return INSTANCE;
- }
-
- @VisibleForTesting
- static GraphHelper getInstance(AtlasGraph graph) {
- if ( INSTANCE == null) {
- synchronized (GraphHelper.class) {
- if (INSTANCE == null) {
- INSTANCE = new GraphHelper(graph);
- }
- }
- }
- return INSTANCE;
- }
-
public static boolean isTermEntityEdge(AtlasEdge edge) {
return StringUtils.equals(edge.getLabel(), TERM_ASSIGNMENT_LABEL);
}
- public AtlasVertex createVertexWithIdentity(Referenceable typedInstance, Set<String> superTypeNames) {
- final String guid = UUID.randomUUID().toString();
-
- final AtlasVertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
- new Id(guid, 0, typedInstance.getTypeName()), superTypeNames);
-
- // add identity
- AtlasGraphUtilsV2.setEncodedProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid);
-
- // add version information
- AtlasGraphUtilsV2.setEncodedProperty(vertexWithIdentity, Constants.VERSION_PROPERTY_KEY, Long.valueOf(typedInstance.getId().getVersion()));
-
- return vertexWithIdentity;
- }
-
- public AtlasVertex createVertexWithoutIdentity(String typeName, Id typedInstanceId, Set<String> superTypeNames) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating AtlasVertex for type {} id {}", typeName, typedInstanceId != null ? typedInstanceId._getId() : null);
- }
-
- final AtlasVertex ret = graph.addVertex();
-
- AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, typeName);
- AtlasGraphUtilsV2.setEncodedProperty(ret, STATE_PROPERTY_KEY, ACTIVE.name());
- AtlasGraphUtilsV2.setEncodedProperty(ret, TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
- AtlasGraphUtilsV2.setEncodedProperty(ret, MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
- AtlasGraphUtilsV2.setEncodedProperty(ret, CREATED_BY_KEY, RequestContext.get().getUser());
- AtlasGraphUtilsV2.setEncodedProperty(ret, MODIFIED_BY_KEY, RequestContext.get().getUser());
-
- for (String superTypeName : superTypeNames) {
- AtlasGraphUtilsV2.addEncodedProperty(ret, SUPER_TYPES_PROPERTY_KEY, superTypeName);
- }
-
- return ret;
- }
-
public AtlasEdge addClassificationEdge(AtlasVertex entityVertex, AtlasVertex classificationVertex, boolean isPropagated) {
AtlasEdge ret = addEdge(entityVertex, classificationVertex, CLASSIFICATION_LABEL);
@@ -1193,6 +1129,14 @@ public final class GraphHelper {
return ret;
}
+ public AtlasGraph getGraph() {
+ return this.graph;
+ }
+
+ public Boolean getDefaultRemovePropagations() {
+ return this.removePropagations;
+ }
+
/**
* Guid and AtlasVertex combo
*/
@@ -1663,10 +1607,6 @@ public final class GraphHelper {
return StringUtils.equals(getGuid(vertexB), getGuid(vertexA));
}
- public static boolean getDefaultRemovePropagations() {
- return removePropagations;
- }
-
public static String getDelimitedClassificationNames(Collection<String> classificationNames) {
String ret = null;
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 9c57d54..0491a85 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -66,14 +66,14 @@ public class ExportService {
private final HdfsPathEntityCreator hdfsPathEntityCreator;
@Inject
- public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
+ public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph graph,
AuditsWriter auditsWriter, HdfsPathEntityCreator hdfsPathEntityCreator) {
this.typeRegistry = typeRegistry;
- this.entityGraphRetriever = new EntityGraphRetriever(this.typeRegistry);
+ this.entityGraphRetriever = new EntityGraphRetriever(graph, this.typeRegistry);
this.auditsWriter = auditsWriter;
this.hdfsPathEntityCreator = hdfsPathEntityCreator;
- this.startEntityFetchByExportRequest = new StartEntityFetchByExportRequest(atlasGraph, typeRegistry, AtlasGremlinQueryProvider.INSTANCE);
- this.entitiesExtractor = new EntitiesExtractor(atlasGraph, typeRegistry);
+ this.startEntityFetchByExportRequest = new StartEntityFetchByExportRequest(graph, typeRegistry, AtlasGremlinQueryProvider.INSTANCE);
+ this.entitiesExtractor = new EntitiesExtractor(graph, typeRegistry);
}
public AtlasExportResult run(ZipSink exportSink, AtlasExportRequest request, String userName, String hostName,
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java
index 512eb46..e414a8c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerDelegate.java
@@ -19,6 +19,7 @@
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.RequestContext;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.store.DeleteType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
@@ -37,11 +38,13 @@ public class DeleteHandlerDelegate {
private final SoftDeleteHandlerV1 softDeleteHandler;
private final HardDeleteHandlerV1 hardDeleteHandler;
private final DeleteHandlerV1 defaultHandler;
+ private final AtlasGraph atlasGraph;
@Inject
- public DeleteHandlerDelegate(AtlasTypeRegistry typeRegistry) {
- this.softDeleteHandler = new SoftDeleteHandlerV1(typeRegistry);
- this.hardDeleteHandler = new HardDeleteHandlerV1(typeRegistry);
+ public DeleteHandlerDelegate(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry) {
+ this.atlasGraph = atlasGraph;
+ this.softDeleteHandler = new SoftDeleteHandlerV1(atlasGraph, typeRegistry);
+ this.hardDeleteHandler = new HardDeleteHandlerV1(atlasGraph, typeRegistry);
this.defaultHandler = getDefaultConfiguredHandler(typeRegistry);
}
@@ -74,7 +77,7 @@ public class DeleteHandlerDelegate {
LOG.info("Default delete handler set to: {}", handlerFromProperties.getName());
- ret = (DeleteHandlerV1) handlerFromProperties.getConstructor(AtlasTypeRegistry.class).newInstance(typeRegistry);
+ ret = (DeleteHandlerV1) handlerFromProperties.getConstructor(AtlasGraph.class, AtlasTypeRegistry.class).newInstance(this.atlasGraph, typeRegistry);
} catch (Exception ex) {
LOG.error("Error instantiating default delete handler. Defaulting to: {}", softDeleteHandler.getClass().getName(), ex);
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index f120c45..3f8503a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -32,6 +32,7 @@ import org.apache.atlas.repository.graph.AtlasEdgeLabel;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
@@ -62,15 +63,17 @@ import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelation
public abstract class DeleteHandlerV1 {
public static final Logger LOG = LoggerFactory.getLogger(DeleteHandlerV1.class);
- protected static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+ protected final GraphHelper graphHelper;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
private final boolean shouldUpdateInverseReferences;
private final boolean softDelete;
- public DeleteHandlerV1(AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) {
+ public DeleteHandlerV1(AtlasGraph graph, AtlasTypeRegistry typeRegistry, boolean shouldUpdateInverseReference, boolean softDelete) {
this.typeRegistry = typeRegistry;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.graphHelper = new GraphHelper(graph);
+ this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
this.shouldUpdateInverseReferences = shouldUpdateInverseReference;
this.softDelete = softDelete;
}
@@ -210,7 +213,7 @@ public abstract class DeleteHandlerV1 {
if (attributeInfo.getAttributeDef().isSoftReferenced()) {
String softRefVal = vertex.getProperty(attributeInfo.getVertexPropertyName(), String.class);
AtlasObjectId refObjId = AtlasEntityUtil.parseSoftRefValue(softRefVal);
- AtlasVertex refVertex = refObjId != null ? AtlasGraphUtilsV2.findByGuid(refObjId.getGuid()) : null;
+ AtlasVertex refVertex = refObjId != null ? AtlasGraphUtilsV2.findByGuid(this.graphHelper.getGraph(), refObjId.getGuid()) : null;
if (refVertex != null) {
vertices.push(refVertex);
@@ -244,7 +247,7 @@ public abstract class DeleteHandlerV1 {
if (CollectionUtils.isNotEmpty(refObjIds)) {
for (AtlasObjectId refObjId : refObjIds) {
- AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
+ AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(this.graphHelper.getGraph(), refObjId.getGuid());
if (refVertex != null) {
vertices.push(refVertex);
@@ -257,7 +260,7 @@ public abstract class DeleteHandlerV1 {
if (MapUtils.isNotEmpty(refObjIds)) {
for (AtlasObjectId refObjId : refObjIds.values()) {
- AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(refObjId.getGuid());
+ AtlasVertex refVertex = AtlasGraphUtilsV2.findByGuid(this.graphHelper.getGraph(), refObjId.getGuid());
if (refVertex != null) {
vertices.push(refVertex);
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java
index a32b671..72dd632 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/HardDeleteHandlerV1.java
@@ -22,6 +22,7 @@ import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.springframework.stereotype.Component;
@@ -33,8 +34,8 @@ import javax.inject.Inject;
public class HardDeleteHandlerV1 extends DeleteHandlerV1 {
@Inject
- public HardDeleteHandlerV1(AtlasTypeRegistry typeRegistry) {
- super(typeRegistry, true, false);
+ public HardDeleteHandlerV1(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
+ super(graph, typeRegistry, true, false);
}
@Override
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java
index e614a4e..59e7cf8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/SoftDeleteHandlerV1.java
@@ -22,6 +22,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -36,8 +37,8 @@ import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
public class SoftDeleteHandlerV1 extends DeleteHandlerV1 {
@Inject
- public SoftDeleteHandlerV1(AtlasTypeRegistry typeRegistry) {
- super(typeRegistry, false, true);
+ public SoftDeleteHandlerV1(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
+ super(graph, typeRegistry, false, true);
}
@Override
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
index d3d9367..b51951f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java
@@ -24,6 +24,7 @@ import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
@@ -56,11 +57,13 @@ import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.valid
public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityGraphDiscoveryV2.class);
+ private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphDiscoveryContext discoveryContext;
private final EntityGraphMapper entityGraphMapper;
- public AtlasEntityGraphDiscoveryV2(AtlasTypeRegistry typeRegistry, EntityStream entityStream, EntityGraphMapper entityGraphMapper) {
+ public AtlasEntityGraphDiscoveryV2(AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityStream entityStream, EntityGraphMapper entityGraphMapper) {
+ this.graph = graph;
this.typeRegistry = typeRegistry;
this.discoveryContext = new EntityGraphDiscoveryContext(typeRegistry, entityStream);
this.entityGraphMapper = entityGraphMapper;
@@ -189,8 +192,8 @@ public class AtlasEntityGraphDiscoveryV2 implements EntityGraphDiscovery {
protected void resolveReferences() throws AtlasBaseException {
MetricRecorder metric = RequestContext.get().startMetricRecord("resolveReferences");
- EntityResolver[] entityResolvers = new EntityResolver[] { new IDBasedEntityResolver(typeRegistry, entityGraphMapper),
- new UniqAttrBasedEntityResolver(typeRegistry, entityGraphMapper)
+ EntityResolver[] entityResolvers = new EntityResolver[] { new IDBasedEntityResolver(this.graph, typeRegistry),
+ new UniqAttrBasedEntityResolver(this.graph, typeRegistry, entityGraphMapper)
};
for (EntityResolver resolver : entityResolvers) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index 379150b..89076c1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -43,6 +43,7 @@ import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.EntityGraphDiscovery;
@@ -100,22 +101,22 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV2.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("store.EntityStore");
-
+ private final AtlasGraph graph;
private final DeleteHandlerDelegate deleteDelegate;
private final AtlasTypeRegistry typeRegistry;
private final IAtlasEntityChangeNotifier entityChangeNotifier;
private final EntityGraphMapper entityGraphMapper;
private final EntityGraphRetriever entityRetriever;
-
@Inject
- public AtlasEntityStoreV2(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry,
+ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry,
IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper) {
+ this.graph = graph;
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
this.entityChangeNotifier = entityChangeNotifier;
this.entityGraphMapper = entityGraphMapper;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
}
@Override
@@ -129,7 +130,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME);
}
- List<String> ret = AtlasGraphUtilsV2.findEntityGUIDsByType(typename);
+ List<String> ret = AtlasGraphUtilsV2.findEntityGUIDsByType(graph, typename);
if (LOG.isDebugEnabled()) {
LOG.debug("<== getEntityGUIDS({})", typename);
@@ -151,7 +152,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> getById({}, {})", guid, isMinExtInfo);
}
- EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry, ignoreRelationships);
+ EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, ignoreRelationships);
AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(guid, isMinExtInfo);
@@ -175,7 +176,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> getHeaderById({})", guid);
}
- EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry);
+ EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
AtlasEntityHeader ret = entityRetriever.toAtlasEntityHeader(guid);
@@ -205,7 +206,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> getByIds({}, {})", guids, isMinExtInfo);
}
- EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry, ignoreRelationships);
+ EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, ignoreRelationships);
AtlasEntitiesWithExtInfo ret = entityRetriever.toAtlasEntitiesWithExtInfo(guids, isMinExtInfo);
@@ -231,7 +232,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> getEntitiesByUniqueAttributes({}, {})", entityType.getTypeName(), uniqueAttributes);
}
- EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry, ignoreRelationships);
+ EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, ignoreRelationships);
AtlasEntitiesWithExtInfo ret = entityRetriever.getEntitiesByUniqueAttributes(entityType.getTypeName(), uniqueAttributes, isMinExtInfo);
@@ -262,9 +263,9 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> getByUniqueAttribute({}, {})", entityType.getTypeName(), uniqAttributes);
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes);
- EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry, ignoreRelationships);
+ EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, ignoreRelationships);
AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex, isMinExtInfo);
@@ -289,9 +290,9 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> getEntityHeaderByUniqueAttributes({}, {})", entityType.getTypeName(), uniqAttributes);
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes);
- EntityGraphRetriever entityRetriever = new EntityGraphRetriever(typeRegistry);
+ EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
AtlasEntityHeader ret = entityRetriever.toAtlasEntityHeader(entityVertex);
@@ -322,7 +323,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
LOG.debug("==> checkState({})", request);
}
- EntityStateChecker entityStateChecker = new EntityStateChecker(typeRegistry);
+ EntityStateChecker entityStateChecker = new EntityStateChecker(graph, typeRegistry);
AtlasCheckStateResult ret = entityStateChecker.checkState(request);
@@ -372,7 +373,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, objectId.getTypeName());
}
- guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(typeRegistry.getEntityTypeByName(objectId.getTypeName()), objectId.getUniqueAttributes());
+ guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(graph, typeRegistry.getEntityTypeByName(objectId.getTypeName()), objectId.getUniqueAttributes());
}
AtlasEntity entity = updatedEntityInfo.getEntity();
@@ -394,7 +395,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "no entity to update.");
}
- String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(entityType, uniqAttributes);
+ String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(graph, entityType, uniqAttributes);
AtlasEntity entity = updatedEntityInfo.getEntity();
entity.setGuid(guid);
@@ -463,7 +464,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
Collection<AtlasVertex> deletionCandidates = new ArrayList<>();
- AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (vertex != null) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex);
@@ -497,7 +498,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
Collection<AtlasVertex> deletionCandidates = new ArrayList<>();
for (String guid : guids) {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (vertex == null) {
if (LOG.isDebugEnabled()) {
@@ -539,7 +540,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
Collection<AtlasVertex> purgeCandidates = new ArrayList<>();
for (String guid : guids) {
- AtlasVertex vertex = AtlasGraphUtilsV2.findDeletedByGuid(guid);
+ AtlasVertex vertex = AtlasGraphUtilsV2.findDeletedByGuid(graph, guid);
if (vertex == null) {
// Entity does not exist - treat as non-error, since the caller
@@ -572,7 +573,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
Collection<AtlasVertex> deletionCandidates = new ArrayList<>();
- AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(entityType, uniqAttributes);
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(graph, entityType, uniqAttributes);
if (vertex != null) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(vertex);
@@ -599,7 +600,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
@Override
@GraphTransaction
public String getGuidByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> uniqAttributes) throws AtlasBaseException{
- return AtlasGraphUtilsV2.getGuidByUniqueAttributes(entityType, uniqAttributes);
+ return AtlasGraphUtilsV2.getGuidByUniqueAttributes(graph, entityType, uniqAttributes);
}
@Override
@@ -619,7 +620,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid);
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -670,7 +671,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid);
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -715,7 +716,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guids);
for (String guid : guids) {
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -840,7 +841,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
@Override
@GraphTransaction
public String setClassifications(AtlasEntityHeaders entityHeaders) {
- ClassificationAssociator.Updater associator = new ClassificationAssociator.Updater(typeRegistry, this);
+ ClassificationAssociator.Updater associator = new ClassificationAssociator.Updater(graph, typeRegistry, this);
return associator.setClassifications(entityHeaders.getGuidHeaderMap());
}
@@ -860,7 +861,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "businessAttributes is null/empty");
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -923,7 +924,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "businessAttributes is null/empty");
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -958,7 +959,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "guid is null/empty");
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -1021,7 +1022,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "labels is null/empty");
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -1060,7 +1061,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "labels is null/empty");
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -1247,7 +1248,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, EntityGraphMapper entityGraphMapper, boolean isPartialUpdate) throws AtlasBaseException {
MetricRecorder metric = RequestContext.get().startMetricRecord("preCreateOrUpdate");
- EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV2(typeRegistry, entityStream, entityGraphMapper);
+ EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV2(graph, typeRegistry, entityStream, entityGraphMapper);
EntityGraphDiscoveryContext discoveryContext = graphDiscoverer.discoverEntities();
EntityMutationContext context = new EntityMutationContext(discoveryContext);
RequestContext requestContext = RequestContext.get();
@@ -1428,7 +1429,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
*/
private void validateEntityAssociations(String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
List<String> entityClassifications = getClassificationNames(guid);
- String entityTypeName = AtlasGraphUtilsV2.getTypeNameFromGuid(guid);
+ String entityTypeName = AtlasGraphUtilsV2.getTypeNameFromGuid(graph, guid);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName);
for (AtlasClassification classification : classifications) {
@@ -1607,7 +1608,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
String vertexKey = typeName + "_" + uniqueAttrName + "_" + uniqueAttrValue;
AtlasVertex atlasVertex = vertexCache.get(vertexKey);
if (atlasVertex == null) {
- atlasVertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(typeName, uniqueAttrName, uniqueAttrValue);
+ atlasVertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(graph, typeName, uniqueAttrName, uniqueAttrValue);
}
if (atlasVertex == null) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index f393e51..fc5d48d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
@@ -45,6 +45,7 @@ import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.util.FileUtils;
+import org.apache.atlas.utils.AtlasPerfMetrics;
import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.configuration.Configuration;
@@ -268,9 +269,12 @@ public class AtlasGraphUtilsV2 {
return returnType.cast(property);
}
-
public static AtlasVertex getVertexByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> attrValues) throws AtlasBaseException {
- AtlasVertex vertex = findByUniqueAttributes(entityType, attrValues);
+ return getVertexByUniqueAttributes(getGraphInstance(), entityType, attrValues);
+ }
+
+ public static AtlasVertex getVertexByUniqueAttributes(AtlasGraph graph, AtlasEntityType entityType, Map<String, Object> attrValues) throws AtlasBaseException {
+ AtlasVertex vertex = findByUniqueAttributes(graph, entityType, attrValues);
if (vertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(),
@@ -281,11 +285,20 @@ public class AtlasGraphUtilsV2 {
}
public static String getGuidByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> attrValues) throws AtlasBaseException {
- AtlasVertex vertexByUniqueAttributes = getVertexByUniqueAttributes(entityType, attrValues);
+ AtlasVertex vertexByUniqueAttributes = getVertexByUniqueAttributes(getGraphInstance(), entityType, attrValues);
+ return getIdFromVertex(vertexByUniqueAttributes);
+ }
+
+ public static String getGuidByUniqueAttributes(AtlasGraph graph, AtlasEntityType entityType, Map<String, Object> attrValues) throws AtlasBaseException {
+ AtlasVertex vertexByUniqueAttributes = getVertexByUniqueAttributes(graph, entityType, attrValues);
return getIdFromVertex(vertexByUniqueAttributes);
}
public static AtlasVertex findByUniqueAttributes(AtlasEntityType entityType, Map<String, Object> attrValues) {
+ return findByUniqueAttributes(getGraphInstance(), entityType, attrValues);
+ }
+
+ public static AtlasVertex findByUniqueAttributes(AtlasGraph graph, AtlasEntityType entityType, Map<String, Object> attrValues) {
MetricRecorder metric = RequestContext.get().startMetricRecord("findByUniqueAttributes");
AtlasVertex vertex = null;
@@ -300,22 +313,22 @@ public class AtlasGraphUtilsV2 {
continue;
}
- if (canUseIndexQuery(entityType, attribute.getName())) {
- vertex = AtlasGraphUtilsV2.getAtlasVertexFromIndexQuery(entityType, attribute, attrValue);
+ if (canUseIndexQuery(graph, entityType, attribute.getName())) {
+ vertex = AtlasGraphUtilsV2.getAtlasVertexFromIndexQuery(graph, entityType, attribute, attrValue);
} else {
if (USE_UNIQUE_INDEX_PROPERTY_TO_FIND_ENTITY && attribute.getVertexUniquePropertyName() != null) {
- vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(entityType.getTypeName(), attribute.getVertexUniquePropertyName(), attrValue);
+ vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(graph, entityType.getTypeName(), attribute.getVertexUniquePropertyName(), attrValue);
// if no instance of given typeName is found, try to find an instance of type's sub-type
if (vertex == null && !entityType.getAllSubTypes().isEmpty()) {
- vertex = AtlasGraphUtilsV2.findBySuperTypeAndUniquePropertyName(entityType.getTypeName(), attribute.getVertexUniquePropertyName(), attrValue);
+ vertex = AtlasGraphUtilsV2.findBySuperTypeAndUniquePropertyName(graph, entityType.getTypeName(), attribute.getVertexUniquePropertyName(), attrValue);
}
} else {
- vertex = AtlasGraphUtilsV2.findByTypeAndPropertyName(entityType.getTypeName(), attribute.getVertexPropertyName(), attrValue);
+ vertex = AtlasGraphUtilsV2.findByTypeAndPropertyName(graph, entityType.getTypeName(), attribute.getVertexPropertyName(), attrValue);
// if no instance of given typeName is found, try to find an instance of type's sub-type
if (vertex == null && !entityType.getAllSubTypes().isEmpty()) {
- vertex = AtlasGraphUtilsV2.findBySuperTypeAndPropertyName(entityType.getTypeName(), attribute.getVertexPropertyName(), attrValue);
+ vertex = AtlasGraphUtilsV2.findBySuperTypeAndPropertyName(graph, entityType.getTypeName(), attribute.getVertexPropertyName(), attrValue);
}
}
@@ -338,11 +351,16 @@ public class AtlasGraphUtilsV2 {
}
public static AtlasVertex findByGuid(String guid) {
+ return findByGuid(getGraphInstance(), guid);
+ }
+
+ public static AtlasVertex findByGuid(AtlasGraph graph, String guid) {
+ AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("findByGuid");
+
AtlasVertex ret = GraphTransactionInterceptor.getVertexFromCache(guid);
if (ret == null) {
- AtlasGraphQuery query = getGraphInstance().query()
- .has(Constants.GUID_PROPERTY_KEY, guid);
+ AtlasGraphQuery query = graph.query().has(Constants.GUID_PROPERTY_KEY, guid);
Iterator<AtlasVertex> results = query.vertices().iterator();
@@ -353,14 +371,15 @@ public class AtlasGraphUtilsV2 {
}
}
+ RequestContext.get().endMetricRecord(metric);
return ret;
}
- public static AtlasVertex findDeletedByGuid(String guid) {
+ public static AtlasVertex findDeletedByGuid(AtlasGraph graph, String guid) {
AtlasVertex ret = GraphTransactionInterceptor.getVertexFromCache(guid);
if (ret == null) {
- AtlasGraphQuery query = getGraphInstance().query()
+ AtlasGraphQuery query = graph.query()
.has(Constants.GUID_PROPERTY_KEY, guid)
.has(STATE_PROPERTY_KEY, Status.DELETED.name());
@@ -376,20 +395,23 @@ public class AtlasGraphUtilsV2 {
return ret;
}
- public static String getTypeNameFromGuid(String guid) {
+ public static String getTypeNameFromGuid(AtlasGraph graph, String guid) {
String ret = null;
if (StringUtils.isNotEmpty(guid)) {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(graph, guid);
ret = (vertex != null) ? AtlasGraphUtilsV2.getTypeName(vertex) : null;
}
return ret;
}
-
public static boolean typeHasInstanceVertex(String typeName) throws AtlasBaseException {
- AtlasGraphQuery query = getGraphInstance()
+ return typeHasInstanceVertex(getGraphInstance(), typeName);
+ }
+
+ public static boolean typeHasInstanceVertex(AtlasGraph graph, String typeName) throws AtlasBaseException {
+ AtlasGraphQuery query = graph
.query()
.has(TYPE_NAME_PROPERTY_KEY, AtlasGraphQuery.ComparisionOperator.EQUAL, typeName);
@@ -405,9 +427,13 @@ public class AtlasGraphUtilsV2 {
}
public static AtlasVertex findByTypeAndUniquePropertyName(String typeName, String propertyName, Object attrVal) {
+ return findByTypeAndUniquePropertyName(getGraphInstance(), typeName, propertyName, attrVal);
+ }
+
+ public static AtlasVertex findByTypeAndUniquePropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
MetricRecorder metric = RequestContext.get().startMetricRecord("findByTypeAndUniquePropertyName");
- AtlasGraphQuery query = getGraphInstance().query()
+ AtlasGraphQuery query = graph.query()
.has(ENTITY_TYPE_PROPERTY_KEY, typeName)
.has(propertyName, attrVal);
@@ -420,10 +446,10 @@ public class AtlasGraphUtilsV2 {
return vertex;
}
- public static AtlasVertex findBySuperTypeAndUniquePropertyName(String typeName, String propertyName, Object attrVal) {
+ public static AtlasVertex findBySuperTypeAndUniquePropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
MetricRecorder metric = RequestContext.get().startMetricRecord("findBySuperTypeAndUniquePropertyName");
- AtlasGraphQuery query = getGraphInstance().query()
+ AtlasGraphQuery query = graph.query()
.has(Constants.SUPER_TYPES_PROPERTY_KEY, typeName)
.has(propertyName, attrVal);
@@ -436,10 +462,10 @@ public class AtlasGraphUtilsV2 {
return vertex;
}
- public static AtlasVertex findByTypeAndPropertyName(String typeName, String propertyName, Object attrVal) {
+ public static AtlasVertex findByTypeAndPropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
MetricRecorder metric = RequestContext.get().startMetricRecord("findByTypeAndPropertyName");
- AtlasGraphQuery query = getGraphInstance().query()
+ AtlasGraphQuery query = graph.query()
.has(ENTITY_TYPE_PROPERTY_KEY, typeName)
.has(propertyName, attrVal)
.has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
@@ -453,10 +479,10 @@ public class AtlasGraphUtilsV2 {
return vertex;
}
- public static AtlasVertex findBySuperTypeAndPropertyName(String typeName, String propertyName, Object attrVal) {
+ public static AtlasVertex findBySuperTypeAndPropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
MetricRecorder metric = RequestContext.get().startMetricRecord("findBySuperTypeAndPropertyName");
- AtlasGraphQuery query = getGraphInstance().query()
+ AtlasGraphQuery query = graph.query()
.has(Constants.SUPER_TYPES_PROPERTY_KEY, typeName)
.has(propertyName, attrVal)
.has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
@@ -470,14 +496,12 @@ public class AtlasGraphUtilsV2 {
return vertex;
}
- public int getOpenTransactions() {
- Set openTransactions = getGraphInstance().getOpenTransactions();
-
- return (openTransactions != null) ? openTransactions.size() : 0;
+ public static List<String> findEntityGUIDsByType(String typename, SortOrder sortOrder) {
+ return findEntityGUIDsByType(getGraphInstance(), typename, sortOrder);
}
- public static List<String> findEntityGUIDsByType(String typename, SortOrder sortOrder) {
- AtlasGraphQuery query = getGraphInstance().query()
+ public static List<String> findEntityGUIDsByType(AtlasGraph graph, String typename, SortOrder sortOrder) {
+ AtlasGraphQuery query = graph.query()
.has(ENTITY_TYPE_PROPERTY_KEY, typename);
if (sortOrder != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
@@ -498,8 +522,8 @@ public class AtlasGraphUtilsV2 {
return ret;
}
- public static Iterator<AtlasVertex> findActiveEntityVerticesByType(String typename) {
- return findActiveEntityVerticesByType(getGraphInstance(), typename);
+ public static List<String> findEntityGUIDsByType(AtlasGraph graph, String typename) {
+ return findEntityGUIDsByType(graph, typename, null);
}
public static Iterator<AtlasVertex> findActiveEntityVerticesByType(AtlasGraph graph, String typename) {
@@ -510,12 +534,12 @@ public class AtlasGraphUtilsV2 {
return query.vertices().iterator();
}
- public static List<String> findEntityGUIDsByType(String typename) {
- return findEntityGUIDsByType(typename, null);
+ public static boolean relationshipTypeHasInstanceEdges(String typeName) throws AtlasBaseException {
+ return relationshipTypeHasInstanceEdges(getGraphInstance(), typeName);
}
- public static boolean relationshipTypeHasInstanceEdges(String typeName) throws AtlasBaseException {
- AtlasGraphQuery query = getGraphInstance()
+ public static boolean relationshipTypeHasInstanceEdges(AtlasGraph graph, String typeName) throws AtlasBaseException {
+ AtlasGraphQuery query = graph
.query()
.has(TYPE_NAME_PROPERTY_KEY, AtlasGraphQuery.ComparisionOperator.EQUAL, typeName);
@@ -584,7 +608,7 @@ public class AtlasGraphUtilsV2 {
return element.getProperty(STATE_PROPERTY_KEY, String.class);
}
- private static boolean canUseIndexQuery(AtlasEntityType entityType, String attributeName) {
+ private static boolean canUseIndexQuery(AtlasGraph graph, AtlasEntityType entityType, String attributeName) {
boolean ret = false;
if (USE_INDEX_QUERY_TO_FIND_ENTITY_BY_UNIQUE_ATTRIBUTES) {
@@ -593,7 +617,7 @@ public class AtlasGraphUtilsV2 {
ret = typeAndSubTypesQryStr.length() <= SearchProcessor.MAX_QUERY_STR_LENGTH_TYPES;
if (ret) {
- Set<String> indexSet = getGraphInstance().getVertexIndexKeys();
+ Set<String> indexSet = graph.getVertexIndexKeys();
try {
ret = indexSet.contains(entityType.getQualifiedAttributeName(attributeName));
}
@@ -606,9 +630,9 @@ public class AtlasGraphUtilsV2 {
return ret;
}
- private static AtlasVertex getAtlasVertexFromIndexQuery(AtlasEntityType entityType, AtlasAttribute attribute, Object attrVal) {
+ private static AtlasVertex getAtlasVertexFromIndexQuery(AtlasGraph graph, AtlasEntityType entityType, AtlasAttribute attribute, Object attrVal) {
String propertyName = attribute.getVertexPropertyName();
- AtlasIndexQuery query = getIndexQuery(entityType, propertyName, attrVal.toString());
+ AtlasIndexQuery query = getIndexQuery(graph, entityType, propertyName, attrVal.toString());
for (Iterator<Result> iter = query.vertices(); iter.hasNext(); ) {
Result result = iter.next();
@@ -645,7 +669,7 @@ public class AtlasGraphUtilsV2 {
return null;
}
- private static AtlasIndexQuery getIndexQuery(AtlasEntityType entityType, String propertyName, String value) {
+ private static AtlasIndexQuery getIndexQuery(AtlasGraph graph, AtlasEntityType entityType, String propertyName, String value) {
StringBuilder sb = new StringBuilder();
sb.append(INDEX_SEARCH_PREFIX + "\"").append(TYPE_NAME_PROPERTY_KEY).append("\":").append(entityType.getTypeAndAllSubTypesQryStr())
@@ -654,7 +678,7 @@ public class AtlasGraphUtilsV2 {
.append(" AND ")
.append(INDEX_SEARCH_PREFIX + "\"").append(STATE_PROPERTY_KEY).append("\":ACTIVE");
- return getGraphInstance().indexQuery(Constants.VERTEX_INDEX, sb.toString());
+ return graph.indexQuery(Constants.VERTEX_INDEX, sb.toString());
}
public static String getIndexSearchPrefix() {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 7425ac6..d1c1f12 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -40,6 +40,7 @@ import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
@@ -91,20 +92,23 @@ import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTy
@Component
public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
private static final Logger LOG = LoggerFactory.getLogger(AtlasRelationshipStoreV2.class);
-
private static final Long DEFAULT_RELATIONSHIP_VERSION = 0L;
+
+ private final AtlasGraph graph;
private boolean notificationsEnabled = NOTIFICATION_RELATIONSHIPS_ENABLED.getBoolean();
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
private final DeleteHandlerDelegate deleteDelegate;
- private final GraphHelper graphHelper = GraphHelper.getInstance();
+ private final GraphHelper graphHelper;
private final IAtlasEntityChangeNotifier entityChangeNotifier;
@Inject
- public AtlasRelationshipStoreV2(AtlasTypeRegistry typeRegistry, DeleteHandlerDelegate deleteDelegate, IAtlasEntityChangeNotifier entityChangeNotifier) {
+ public AtlasRelationshipStoreV2(AtlasGraph graph, AtlasTypeRegistry typeRegistry, DeleteHandlerDelegate deleteDelegate, IAtlasEntityChangeNotifier entityChangeNotifier) {
+ this.graph = graph;
this.typeRegistry = typeRegistry;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.graphHelper = new GraphHelper(graph);
+ this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
this.deleteDelegate = deleteDelegate;
this.entityChangeNotifier = entityChangeNotifier;
}
@@ -274,7 +278,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_DELETED, guid);
}
- String relationShipType = GraphHelper.getTypeName(edge);
+ String relationShipType = graphHelper.getTypeName(edge);
AtlasEntityHeader end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getOutVertex());
AtlasEntityHeader end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(edge.getInVertex());
@@ -721,7 +725,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
String guid = end.getGuid();
String typeName = end.getTypeName();
Map<String, Object> uniqueAttributes = end.getUniqueAttributes();
- AtlasVertex endVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex endVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
if (!AtlasTypeUtil.isValidGuid(guid) || endVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -729,7 +733,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
} else if (MapUtils.isNotEmpty(uniqueAttributes)) {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
- if (AtlasGraphUtilsV2.findByUniqueAttributes(entityType, uniqueAttributes) == null) {
+ if (AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, uniqueAttributes) == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, typeName, uniqueAttributes.toString());
}
} else {
@@ -801,11 +805,11 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
AtlasVertex ret = null;
if (StringUtils.isNotEmpty(endPoint.getGuid())) {
- ret = AtlasGraphUtilsV2.findByGuid(endPoint.getGuid());
+ ret = AtlasGraphUtilsV2.findByGuid(this.graph, endPoint.getGuid());
} else if (StringUtils.isNotEmpty(endPoint.getTypeName()) && MapUtils.isNotEmpty(endPoint.getUniqueAttributes())) {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(endPoint.getTypeName());
- ret = AtlasGraphUtilsV2.findByUniqueAttributes(entityType, endPoint.getUniqueAttributes());
+ ret = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, endPoint.getUniqueAttributes());
}
return ret;
@@ -888,7 +892,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
String typeName = objectId.getTypeName();
if (StringUtils.isBlank(typeName)) {
- typeName = AtlasGraphUtilsV2.getTypeNameFromGuid(objectId.getGuid());
+ typeName = AtlasGraphUtilsV2.getTypeNameFromGuid(this.graph, objectId.getGuid());
}
return typeName;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
index a4d732a..9898fe8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
@@ -26,6 +26,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.BulkImporter;
@@ -51,10 +52,12 @@ public class BulkImporterImpl implements BulkImporter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV2.class);
private final AtlasEntityStore entityStore;
+ private final AtlasGraph atlasGraph;
private AtlasTypeRegistry typeRegistry;
@Inject
- public BulkImporterImpl(AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
+ public BulkImporterImpl(AtlasGraph atlasGraph, AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
+ this.atlasGraph = atlasGraph;
this.entityStore = entityStore;
this.typeRegistry = typeRegistry;
}
@@ -65,9 +68,9 @@ public class BulkImporterImpl implements BulkImporter {
if (importResult.getRequest().getOptions() != null &&
importResult.getRequest().getOptions().containsKey(AtlasImportRequest.OPTION_KEY_MIGRATION)) {
- importStrategy = new MigrationImport(new AtlasGraphProvider(), this.typeRegistry);
+ importStrategy = new MigrationImport(this.atlasGraph, new AtlasGraphProvider(), this.typeRegistry);
} else {
- importStrategy = new RegularImport(this.entityStore, this.typeRegistry);
+ importStrategy = new RegularImport(this.atlasGraph, this.entityStore, this.typeRegistry);
}
LOG.info("BulkImportImpl: {}", importStrategy.getClass().getSimpleName());
@@ -121,14 +124,14 @@ public class BulkImporterImpl implements BulkImporter {
}
}
- public static void updateVertexGuid(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityGraphRetriever, AtlasEntity entity) {
+ public static void updateVertexGuid(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityGraphRetriever, AtlasEntity entity) {
String entityGuid = entity.getGuid();
AtlasObjectId objectId = entityGraphRetriever.toAtlasObjectIdWithoutGuid(entity);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
String vertexGuid = null;
try {
- vertexGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(entityType, objectId.getUniqueAttributes());
+ vertexGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(atlasGraph, entityType, objectId.getUniqueAttributes());
} catch (AtlasBaseException e) {
LOG.warn("Entity: {}: Does not exist!", objectId);
return;
@@ -138,7 +141,7 @@ public class BulkImporterImpl implements BulkImporter {
return;
}
- AtlasVertex v = AtlasGraphUtilsV2.findByGuid(vertexGuid);
+ AtlasVertex v = AtlasGraphUtilsV2.findByGuid(atlasGraph, vertexGuid);
if (v == null) {
return;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java
index 05878f4..f12903f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java
@@ -23,6 +23,8 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.repository.audit.EntityAuditRepository;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.type.AtlasEntityType;
@@ -49,11 +51,15 @@ public class ClassificationAssociator {
private final EntityAuditRepository auditRepository;
private final EntityGraphRetriever entityRetriever;
- public Retriever(AtlasTypeRegistry typeRegistry, EntityAuditRepository auditRepository) {
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ public Retriever(AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityAuditRepository auditRepository) {
+ this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
this.auditRepository = auditRepository;
}
+ public Retriever(AtlasTypeRegistry typeRegistry, EntityAuditRepository auditRepository) {
+ this(AtlasGraphProvider.getGraphInstance(), typeRegistry, auditRepository);
+ }
+
Retriever(EntityGraphRetriever entityGraphRetriever, EntityAuditRepository auditRepository) {
this.entityRetriever = entityGraphRetriever;
this.auditRepository = auditRepository;
@@ -104,16 +110,21 @@ public class ClassificationAssociator {
static final String PROCESS_DELETE = "Delete";
static final String JSONIFY_STRING_FORMAT = "\"%s\",";
+ private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
private final AtlasEntityStore entitiesStore;
private final EntityGraphRetriever entityRetriever;
- private StringBuilder actionSummary = new StringBuilder();
- private Map<String, String> typeNameUniqueAttributeNameMap = new HashMap<>();
+ private final StringBuilder actionSummary = new StringBuilder();
- public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
+ public Updater(AtlasGraph graph, AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
+ this.graph = graph;
this.typeRegistry = typeRegistry;
this.entitiesStore = entitiesStore;
- entityRetriever = new EntityGraphRetriever(typeRegistry);
+ entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
+ }
+
+ public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
+ this(AtlasGraphProvider.getGraphInstance(), typeRegistry, entitiesStore);
}
public String setClassifications(Map<String, AtlasEntityHeader> map) {
@@ -246,7 +257,7 @@ public class ClassificationAssociator {
AtlasEntityHeader getByUniqueAttributes(AtlasEntityType entityType, String qualifiedName, Map<String, Object> attrValues) {
try {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(entityType, attrValues);
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, attrValues);
if (vertex == null) {
return null;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 8c43807..14876b1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -92,7 +92,6 @@ import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
-import static org.apache.atlas.repository.graph.GraphHelper.getDefaultRemovePropagations;
import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassificationNames;
import static org.apache.atlas.repository.graph.GraphHelper.getLabels;
import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
@@ -130,7 +129,7 @@ public class EntityGraphMapper {
private static final boolean ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES = AtlasConfiguration.ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES.getBoolean();
private static final boolean CLASSIFICATION_PROPAGATION_DEFAULT = AtlasConfiguration.CLASSIFICATION_PROPAGATION_DEFAULT.getBoolean();
- private final GraphHelper graphHelper = GraphHelper.getInstance();
+ private final GraphHelper graphHelper;
private final AtlasGraph graph;
private final DeleteHandlerDelegate deleteDelegate;
private final AtlasTypeRegistry typeRegistry;
@@ -141,16 +140,17 @@ public class EntityGraphMapper {
private final IFullTextMapper fullTextMapperV2;
@Inject
- public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
+ public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph graph,
AtlasRelationshipStore relationshipStore, IAtlasEntityChangeNotifier entityChangeNotifier,
AtlasInstanceConverter instanceConverter, IFullTextMapper fullTextMapperV2) {
+ this.graphHelper = new GraphHelper(graph);
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
- this.graph = atlasGraph;
+ this.graph = graph;
this.relationshipStore = relationshipStore;
this.entityChangeNotifier = entityChangeNotifier;
this.instanceConverter = instanceConverter;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
this.fullTextMapperV2 = fullTextMapperV2;
}
@@ -361,12 +361,12 @@ public class EntityGraphMapper {
updateLabels(vertex, labels);
- entityChangeNotifier.onLabelsUpdatedFromEntity(GraphHelper.getGuid(vertex), addedLabels, removedLabels);
+ entityChangeNotifier.onLabelsUpdatedFromEntity(graphHelper.getGuid(vertex), addedLabels, removedLabels);
}
public void addLabels(AtlasVertex vertex, Set<String> labels) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(labels)) {
- final Set<String> existingLabels = GraphHelper.getLabels(vertex);
+ final Set<String> existingLabels = graphHelper.getLabels(vertex);
final Set<String> updatedLabels;
if (CollectionUtils.isEmpty(existingLabels)) {
@@ -378,14 +378,14 @@ public class EntityGraphMapper {
if (!updatedLabels.equals(existingLabels)) {
updateLabels(vertex, updatedLabels);
updatedLabels.removeAll(existingLabels);
- entityChangeNotifier.onLabelsUpdatedFromEntity(GraphHelper.getGuid(vertex), updatedLabels, null);
+ entityChangeNotifier.onLabelsUpdatedFromEntity(graphHelper.getGuid(vertex), updatedLabels, null);
}
}
}
public void removeLabels(AtlasVertex vertex, Set<String> labels) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(labels)) {
- final Set<String> existingLabels = GraphHelper.getLabels(vertex);
+ final Set<String> existingLabels = graphHelper.getLabels(vertex);
Set<String> updatedLabels;
if (CollectionUtils.isNotEmpty(existingLabels)) {
@@ -395,7 +395,7 @@ public class EntityGraphMapper {
if (!updatedLabels.equals(existingLabels)) {
updateLabels(vertex, updatedLabels);
existingLabels.removeAll(updatedLabels);
- entityChangeNotifier.onLabelsUpdatedFromEntity(GraphHelper.getGuid(vertex), null, existingLabels);
+ entityChangeNotifier.onLabelsUpdatedFromEntity(graphHelper.getGuid(vertex), null, existingLabels);
}
}
}
@@ -695,9 +695,10 @@ public class EntityGraphMapper {
}
private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException {
+ boolean isDeletedEntity = context.isDeletedEntity(vertex);
+ AtlasType attrType = attribute.getAttributeType();
if (attrValue == null) {
AtlasAttributeDef attributeDef = attribute.getAttributeDef();
- AtlasType attrType = attribute.getAttributeType();
if (attrType.getTypeCategory() == TypeCategory.PRIMITIVE) {
if (attributeDef.getDefaultValue() != null) {
@@ -712,9 +713,12 @@ public class EntityGraphMapper {
}
}
- AttributeMutationContext ctx = new AttributeMutationContext(op, vertex, attribute, attrValue);
-
- mapToVertexByTypeCategory(ctx, context);
+ if (attrType.getTypeCategory() == TypeCategory.PRIMITIVE || attrType.getTypeCategory() == TypeCategory.ENUM) {
+ mapPrimitiveValue(vertex, attribute, attrValue, isDeletedEntity);
+ } else {
+ AttributeMutationContext ctx = new AttributeMutationContext(op, vertex, attribute, attrValue);
+ mapToVertexByTypeCategory(ctx, context);
+ }
}
private Object mapToVertexByTypeCategory(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException {
@@ -906,7 +910,7 @@ public class EntityGraphMapper {
if (inverseUpdated) {
RequestContext requestContext = RequestContext.get();
- if (!requestContext.isDeletedEntity(GraphHelper.getGuid(inverseVertex))) {
+ if (!requestContext.isDeletedEntity(graphHelper.getGuid(inverseVertex))) {
updateModificationMetadata(inverseVertex);
requestContext.recordEntityUpdate(entityRetriever.toAtlasEntityHeader(inverseVertex));
@@ -960,7 +964,7 @@ public class EntityGraphMapper {
return;
}
- String parentGuid = GraphHelper.getGuid(inverseVertex);
+ String parentGuid = graphHelper.getGuid(inverseVertex);
if(StringUtils.isEmpty(parentGuid)) {
return;
}
@@ -997,9 +1001,13 @@ public class EntityGraphMapper {
}
private Object mapPrimitiveValue(AttributeMutationContext ctx, EntityMutationContext context) {
- boolean isIndexableStrAttr = ctx.getAttributeDef().getIsIndexable() && ctx.getAttrType() instanceof AtlasBuiltInTypes.AtlasStringType;
+ return mapPrimitiveValue(ctx.getReferringVertex(), ctx.getAttribute(), ctx.getValue(), context.isDeletedEntity(ctx.referringVertex));
+ }
+
+ private Object mapPrimitiveValue(AtlasVertex vertex, AtlasAttribute attribute, Object valueFromEntity, boolean isDeletedEntity) {
+ boolean isIndexableStrAttr = attribute.getAttributeDef().getIsIndexable() && attribute.getAttributeType() instanceof AtlasBuiltInTypes.AtlasStringType;
- Object ret = ctx.getValue();
+ Object ret = valueFromEntity;
// Janus bug, when an indexed string attribute has a value longer than a certain length then the reverse indexed key generated by JanusGraph
// exceeds the HBase row length's hard limit (Short.MAX). This trimming and hashing procedure is to circumvent that limitation
@@ -1024,26 +1032,26 @@ public class EntityGraphMapper {
}
if (trimmedLength < value.length()) {
- LOG.warn("Length of indexed attribute {} is {} characters, longer than safe-limit {}; trimming to {} - attempt #{}", ctx.getAttribute().getQualifiedName(), value.length(), INDEXED_STR_SAFE_LEN, trimmedLength, requestContext.getAttemptCount());
+ LOG.warn("Length of indexed attribute {} is {} characters, longer than safe-limit {}; trimming to {} - attempt #{}", attribute.getQualifiedName(), value.length(), INDEXED_STR_SAFE_LEN, trimmedLength, requestContext.getAttemptCount());
String checksumSuffix = ":" + DigestUtils.shaHex(value); // Storing SHA checksum in case verification is needed after retrieval
ret = value.substring(0, trimmedLength - checksumSuffix.length()) + checksumSuffix;
} else {
- LOG.warn("Length of indexed attribute {} is {} characters, longer than safe-limit {}", ctx.getAttribute().getQualifiedName(), value.length(), INDEXED_STR_SAFE_LEN);
+ LOG.warn("Length of indexed attribute {} is {} characters, longer than safe-limit {}", attribute.getQualifiedName(), value.length(), INDEXED_STR_SAFE_LEN);
}
}
}
- AtlasGraphUtilsV2.setEncodedProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), ret);
+ AtlasGraphUtilsV2.setEncodedProperty(vertex, attribute.getVertexPropertyName(), ret);
- String uniqPropName = ctx.getAttribute() != null ? ctx.getAttribute().getVertexUniquePropertyName() : null;
+ String uniqPropName = attribute != null ? attribute.getVertexUniquePropertyName() : null;
if (uniqPropName != null) {
- if (context.isDeletedEntity(ctx.getReferringVertex()) || AtlasGraphUtilsV2.getState(ctx.getReferringVertex()) == DELETED) {
- ctx.getReferringVertex().removeProperty(uniqPropName);
+ if (isDeletedEntity || AtlasGraphUtilsV2.getState(vertex) == DELETED) {
+ vertex.removeProperty(uniqPropName);
} else {
- AtlasGraphUtilsV2.setEncodedProperty(ctx.getReferringVertex(), uniqPropName, ret);
+ AtlasGraphUtilsV2.setEncodedProperty(vertex, uniqPropName, ret);
}
}
@@ -1203,7 +1211,7 @@ public class EntityGraphMapper {
ret = getOrCreateRelationship(fromVertex, toVertex, relationshipName, relationshipAttributes);
- boolean isCreated = GraphHelper.getCreatedTime(ret) == RequestContext.get().getRequestTime();
+ boolean isCreated = graphHelper.getCreatedTime(ret) == RequestContext.get().getRequestTime();
if (isCreated) {
// if relationship did not exist before and new relationship was created
@@ -1549,7 +1557,7 @@ public class EntityGraphMapper {
return null;
}
- private static void setAssignedGuid(Object val, EntityMutationContext context) {
+ private void setAssignedGuid(Object val, EntityMutationContext context) {
if (val != null) {
Map<String, String> guidAssignements = context.getGuidAssignments();
@@ -1566,7 +1574,7 @@ public class EntityGraphMapper {
AtlasVertex vertex = context.getDiscoveryContext().getResolvedEntityVertex(objId);
if (vertex != null) {
- assignedGuid = GraphHelper.getGuid(vertex);
+ assignedGuid = graphHelper.getGuid(vertex);
}
}
@@ -1589,7 +1597,7 @@ public class EntityGraphMapper {
AtlasVertex vertex = context.getDiscoveryContext().getResolvedEntityVertex(new AtlasObjectId(mapObjId));
if (vertex != null) {
- assignedGuid = GraphHelper.getGuid(vertex);
+ assignedGuid = graphHelper.getGuid(vertex);
}
}
@@ -1902,7 +1910,7 @@ public class EntityGraphMapper {
}
if (removePropagations == null) {
- removePropagations = getDefaultRemovePropagations();
+ removePropagations = graphHelper.getDefaultRemovePropagations();
classification.setRemovePropagationsOnEntityDelete(removePropagations);
}
@@ -2002,7 +2010,7 @@ public class EntityGraphMapper {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "delete", entityGuid);
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(entityGuid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, entityGuid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, entityGuid);
@@ -2016,7 +2024,7 @@ public class EntityGraphMapper {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "delete", entityGuid);
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(entityGuid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, entityGuid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, entityGuid);
@@ -2093,7 +2101,7 @@ public class EntityGraphMapper {
}
private AtlasEntity updateClassificationText(AtlasVertex vertex) throws AtlasBaseException {
- String guid = GraphHelper.getGuid(vertex);
+ String guid = graphHelper.getGuid(vertex);
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES);
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
@@ -2106,7 +2114,7 @@ public class EntityGraphMapper {
return;
}
- String guid = GraphHelper.getGuid(vertex);
+ String guid = graphHelper.getGuid(vertex);
AtlasEntity entity = instanceConverter.getAndCacheEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES);
List<String> classificationNames = new ArrayList<>();
List<String> propagatedClassificationNames = new ArrayList<>();
@@ -2155,7 +2163,7 @@ public class EntityGraphMapper {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid);
}
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -2307,7 +2315,7 @@ public class EntityGraphMapper {
}
for (AtlasVertex vertex : notificationVertices) {
- String entityGuid = GraphHelper.getGuid(vertex);
+ String entityGuid = graphHelper.getGuid(vertex);
AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES);
if (isActive(entity)) {
@@ -2361,7 +2369,7 @@ public class EntityGraphMapper {
}
public void deleteClassifications(String guid) throws AtlasBaseException {
- AtlasVertex instanceVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex instanceVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
if (instanceVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -2400,7 +2408,7 @@ public class EntityGraphMapper {
if (vertex != null) {
RequestContext req = RequestContext.get();
- if (!req.isUpdatedEntity(GraphHelper.getGuid(vertex))) {
+ if (!req.isUpdatedEntity(graphHelper.getGuid(vertex))) {
updateModificationMetadata(vertex);
req.recordEntityUpdate(entityRetriever.toAtlasEntityHeader(vertex));
@@ -2527,7 +2535,7 @@ public class EntityGraphMapper {
continue;
}
- if (value.length() > CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH) {
+ if (!key.startsWith(CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX) && value.length() > CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_VALUE, value, String.valueOf(CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH));
}
}
@@ -2555,7 +2563,7 @@ public class EntityGraphMapper {
if(CollectionUtils.isNotEmpty(propagatedVertices)) {
for(AtlasVertex vertex : propagatedVertices) {
- AtlasEntity entity = instanceConverter.getAndCacheEntity(GraphHelper.getGuid(vertex), ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES);
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(graphHelper.getGuid(vertex), ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES);
if (isActive(entity)) {
vertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity));
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index 36bee30..757fcb1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -43,6 +43,7 @@ import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
@@ -131,19 +132,22 @@ public class EntityGraphRetriever {
public static final String QUALIFIED_NAME = "qualifiedName";
private static final TypeReference<List<TimeBoundary>> TIME_BOUNDARIES_LIST_TYPE = new TypeReference<List<TimeBoundary>>() {};
- private static final GraphHelper graphHelper = GraphHelper.getInstance();
+ private final GraphHelper graphHelper;
private final AtlasTypeRegistry typeRegistry;
private final boolean ignoreRelationshipAttr;
+ private final AtlasGraph graph;
@Inject
- public EntityGraphRetriever(AtlasTypeRegistry typeRegistry) {
- this(typeRegistry, false);
+ public EntityGraphRetriever(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
+ this(graph, typeRegistry, false);
}
- public EntityGraphRetriever(AtlasTypeRegistry typeRegistry, boolean ignoreRelationshipAttr) {
- this.typeRegistry = typeRegistry;
+ public EntityGraphRetriever(AtlasGraph graph, AtlasTypeRegistry typeRegistry, boolean ignoreRelationshipAttr) {
+ this.graph = graph;
+ this.graphHelper = new GraphHelper(graph);
+ this.typeRegistry = typeRegistry;
this.ignoreRelationshipAttr = ignoreRelationshipAttr;
}
@@ -365,7 +369,7 @@ public class EntityGraphRetriever {
}
public AtlasVertex getEntityVertex(String guid) throws AtlasBaseException {
- AtlasVertex ret = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex ret = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
if (ret == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
@@ -419,7 +423,7 @@ public class EntityGraphRetriever {
if (entityType != null) {
for (Map<String, Object> uniqAttributes : uniqueAttributesList) {
try {
- AtlasVertex vertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);
+ AtlasVertex vertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(this.graph, entityType, uniqAttributes);
if (vertex != null) {
AtlasEntity entity = mapVertexToAtlasEntity(vertex, ret, isMinExtInfo);
@@ -442,7 +446,7 @@ public class EntityGraphRetriever {
public void evaluateClassificationPropagation(AtlasVertex classificationVertex, List<AtlasVertex> entitiesToAddPropagation, List<AtlasVertex> entitiesToRemovePropagation) {
if (classificationVertex != null) {
String entityGuid = getClassificationEntityGuid(classificationVertex);
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(entityGuid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, entityGuid);
String classificationId = classificationVertex.getIdForDisplay();
List<AtlasVertex> propagatedEntities = getAllPropagatedEntityVertices(classificationVertex);
List<AtlasVertex> impactedEntities = getImpactedVerticesV2(entityVertex, null, classificationId);
@@ -472,7 +476,7 @@ public class EntityGraphRetriever {
for (AtlasVertex classificationVertex : classificationVertices) {
String classificationId = classificationVertex.getIdForDisplay();
String sourceEntityId = getClassificationEntityGuid(classificationVertex);
- AtlasVertex sourceEntityVertex = AtlasGraphUtilsV2.findByGuid(sourceEntityId);
+ AtlasVertex sourceEntityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, sourceEntityId);
List<AtlasVertex> entitiesPropagatingTo = getImpactedVerticesV2(sourceEntityVertex, relationshipGuidToExclude, classificationId);
ret.put(classificationVertex, entitiesPropagatingTo);
@@ -582,12 +586,12 @@ public class EntityGraphRetriever {
}
if (AtlasTypeUtil.isAssignedGuid(objId)) {
- ret = AtlasGraphUtilsV2.findByGuid(objId.getGuid());
+ ret = AtlasGraphUtilsV2.findByGuid(this.graph, objId.getGuid());
} else {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(objId.getTypeName());
Map<String, Object> uniqAttributes = objId.getUniqueAttributes();
- ret = AtlasGraphUtilsV2.getVertexByUniqueAttributes(entityType, uniqAttributes);
+ ret = AtlasGraphUtilsV2.getVertexByUniqueAttributes(this.graph, entityType, uniqAttributes);
}
if (ret == null) {
@@ -828,10 +832,10 @@ public class EntityGraphRetriever {
return ret;
}
- public List<AtlasTermAssignmentHeader> mapAssignedTerms(AtlasVertex entityVertex) throws AtlasBaseException {
+ public List<AtlasTermAssignmentHeader> mapAssignedTerms(AtlasVertex entityVertex) {
List<AtlasTermAssignmentHeader> ret = new ArrayList<>();
- Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.IN).label(TERM_ASSIGNMENT_LABEL).edges();
+ Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.IN).label(Constants.INTERNAL_PROPERTY_KEY_PREFIX + TERM_ASSIGNMENT_LABEL).edges();
if (edges != null) {
for (final AtlasEdge edge : (Iterable<AtlasEdge>) edges) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
index 2718a06..9d736ce 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityStateChecker.java
@@ -28,13 +28,13 @@ import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
@@ -55,13 +55,15 @@ import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassifi
public final class EntityStateChecker {
private static final Logger LOG = LoggerFactory.getLogger(EntityStateChecker.class);
+ private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityRetriever;
@Inject
- public EntityStateChecker(AtlasTypeRegistry typeRegistry) {
+ public EntityStateChecker(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
+ this.graph = graph;
this.typeRegistry = typeRegistry;
- this.entityRetriever = new EntityGraphRetriever(typeRegistry);
+ this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry);
}
@@ -140,7 +142,7 @@ public final class EntityStateChecker {
* @throws AtlasBaseException
*/
public AtlasEntityState checkEntityState(String guid, boolean fixIssues, AtlasCheckStateResult result) throws AtlasBaseException {
- AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(guid);
+ AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
if (entityVertex == null) {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid);
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
index 3b96948..2f37eac 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IDBasedEntityResolver.java
@@ -22,6 +22,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
@@ -35,12 +36,12 @@ import org.slf4j.LoggerFactory;
public class IDBasedEntityResolver implements EntityResolver {
private static final Logger LOG = LoggerFactory.getLogger(IDBasedEntityResolver.class);
+ private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
- private final EntityGraphMapper entityGraphMapper;
- public IDBasedEntityResolver(AtlasTypeRegistry typeRegistry, EntityGraphMapper entityGraphMapper) {
+ public IDBasedEntityResolver(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
+ this.graph = graph;
this.typeRegistry = typeRegistry;
- this.entityGraphMapper = entityGraphMapper;
}
public EntityGraphDiscoveryContext resolveEntityReferences(EntityGraphDiscoveryContext context) throws AtlasBaseException {
@@ -52,7 +53,7 @@ public class IDBasedEntityResolver implements EntityResolver {
for (String guid : context.getReferencedGuids()) {
boolean isAssignedGuid = AtlasTypeUtil.isAssignedGuid(guid);
- AtlasVertex vertex = isAssignedGuid ? AtlasGraphUtilsV2.findByGuid(guid) : null;
+ AtlasVertex vertex = isAssignedGuid ? AtlasGraphUtilsV2.findByGuid(this.graph, guid) : null;
if (vertex == null && !RequestContext.get().isImportInProgress()) { // if not found in the store, look if the entity is present in the stream
AtlasEntity entity = entityStream.getByGuid(guid);
@@ -64,7 +65,7 @@ public class IDBasedEntityResolver implements EntityResolver {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
}
- vertex = AtlasGraphUtilsV2.findByUniqueAttributes(entityType, entity.getAttributes());
+ vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, entity.getAttributes());
} else if (!isAssignedGuid) { // for local-guids, entity must be in the stream
throw new AtlasBaseException(AtlasErrorCode.REFERENCED_ENTITY_NOT_FOUND, guid);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqAttrBasedEntityResolver.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqAttrBasedEntityResolver.java
index 14ddc5d..d1c3bde 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqAttrBasedEntityResolver.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqAttrBasedEntityResolver.java
@@ -22,6 +22,7 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext;
import org.apache.atlas.repository.store.graph.EntityResolver;
@@ -36,10 +37,12 @@ import java.util.List;
public class UniqAttrBasedEntityResolver implements EntityResolver {
private static final Logger LOG = LoggerFactory.getLogger(UniqAttrBasedEntityResolver.class);
+ private final AtlasGraph graph;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphMapper entityGraphMapper;
- public UniqAttrBasedEntityResolver(AtlasTypeRegistry typeRegistry, EntityGraphMapper entityGraphMapper) {
+ public UniqAttrBasedEntityResolver(AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityGraphMapper entityGraphMapper) {
+ this.graph = graph;
this.typeRegistry = typeRegistry;
this.entityGraphMapper = entityGraphMapper;
}
@@ -61,7 +64,7 @@ public class UniqAttrBasedEntityResolver implements EntityResolver {
throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), objId.getTypeName());
}
- AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(entityType, objId.getUniqueAttributes());
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(this.graph, entityType, objId.getUniqueAttributes());
if (vertex == null && RequestContext.get().isCreateShellEntityForNonExistingReference()) {
vertex = entityGraphMapper.createShellEntityVertex(objId, context);
@@ -77,4 +80,4 @@ public class UniqAttrBasedEntityResolver implements EntityResolver {
return context;
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
index ff55e40..fe8699d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
@@ -28,7 +28,6 @@ import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.migration.DataMigrationStatusService;
-import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
@@ -46,15 +45,14 @@ import org.slf4j.LoggerFactory;
public class MigrationImport extends ImportStrategy {
private static final Logger LOG = LoggerFactory.getLogger(MigrationImport.class);
+ private final AtlasGraph graph;
+ private final AtlasGraphProvider graphProvider;
private final AtlasTypeRegistry typeRegistry;
- private AtlasGraph atlasGraph;
- private EntityGraphRetriever entityGraphRetriever;
- private EntityGraphMapper entityGraphMapper;
- private AtlasEntityStore entityStore;
- public MigrationImport(AtlasGraphProvider atlasGraphProvider, AtlasTypeRegistry typeRegistry) {
+ public MigrationImport(AtlasGraph graph, AtlasGraphProvider graphProvider, AtlasTypeRegistry typeRegistry) {
+ this.graph = graph;
+ this.graphProvider = graphProvider;
this.typeRegistry = typeRegistry;
- setupEntityStore(atlasGraphProvider, typeRegistry);
LOG.info("MigrationImport: Using bulkLoading...");
}
@@ -72,7 +70,7 @@ public class MigrationImport extends ImportStrategy {
long index = 0;
int streamSize = entityStream.size();
EntityMutationResponse ret = new EntityMutationResponse();
- EntityCreationManager creationManager = createEntityCreationManager(atlasGraph, importResult, dataMigrationStatusService);
+ EntityCreationManager creationManager = createEntityCreationManager(importResult, dataMigrationStatusService);
try {
LOG.info("Migration Import: Size: {}: Starting...", streamSize);
@@ -95,16 +93,24 @@ public class MigrationImport extends ImportStrategy {
return dataMigrationStatusService;
}
- private EntityCreationManager createEntityCreationManager(AtlasGraph threadedAtlasGraph,
- AtlasImportResult importResult,
+ private EntityCreationManager createEntityCreationManager(AtlasImportResult importResult,
DataMigrationStatusService dataMigrationStatusService) {
- atlasGraph = threadedAtlasGraph;
+ AtlasGraph graphBulk = graphProvider.getBulkLoading();
+
+ EntityGraphRetriever entityGraphRetriever = new EntityGraphRetriever(this.graph, typeRegistry);
+ EntityGraphRetriever entityGraphRetrieverBulk = new EntityGraphRetriever(graphBulk, typeRegistry);
+
+ AtlasEntityStoreV2 entityStore = createEntityStore(this.graph, typeRegistry);
+ AtlasEntityStoreV2 entityStoreBulk = createEntityStore(graphBulk, typeRegistry);
+
int batchSize = importResult.getRequest().getOptionKeyBatchSize();
int numWorkers = getNumWorkers(importResult.getRequest().getOptionKeyNumWorkers());
EntityConsumerBuilder consumerBuilder =
- new EntityConsumerBuilder(threadedAtlasGraph, entityStore, entityGraphRetriever, typeRegistry, batchSize);
+ new EntityConsumerBuilder(typeRegistry, this.graph, entityStore, entityGraphRetriever, graphBulk,
+ entityStoreBulk, entityGraphRetrieverBulk, batchSize);
+ LOG.info("MigrationImport: EntityCreationManager: Created!");
return new EntityCreationManager(consumerBuilder, batchSize, numWorkers, importResult, dataMigrationStatusService);
}
@@ -114,17 +120,17 @@ public class MigrationImport extends ImportStrategy {
return ret;
}
- private void setupEntityStore(AtlasGraphProvider atlasGraphProvider, AtlasTypeRegistry typeRegistry) {
- this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
- this.atlasGraph = atlasGraphProvider.getBulkLoading();
- DeleteHandlerDelegate deleteDelegate = new DeleteHandlerDelegate(typeRegistry);
-
+ private AtlasEntityStoreV2 createEntityStore(AtlasGraph graph, AtlasTypeRegistry typeRegistry) {
+ FullTextMapperV2Nop fullTextMapperV2 = new FullTextMapperV2Nop();
IAtlasEntityChangeNotifier entityChangeNotifier = new EntityChangeNotifierNop();
- AtlasRelationshipStore relationshipStore = new AtlasRelationshipStoreV2(typeRegistry, deleteDelegate, entityChangeNotifier);
+ DeleteHandlerDelegate deleteDelegate = new DeleteHandlerDelegate(graph, typeRegistry);
AtlasFormatConverters formatConverters = new AtlasFormatConverters(typeRegistry);
- AtlasInstanceConverter instanceConverter = new AtlasInstanceConverter(typeRegistry, formatConverters);
- this.entityGraphMapper = new EntityGraphMapper(deleteDelegate, typeRegistry, atlasGraph, relationshipStore, entityChangeNotifier, instanceConverter, new FullTextMapperV2Nop());
- this.entityStore = new AtlasEntityStoreV2(deleteDelegate, typeRegistry, entityChangeNotifier, entityGraphMapper);
+
+ AtlasInstanceConverter instanceConverter = new AtlasInstanceConverter(graph, typeRegistry, formatConverters);
+ AtlasRelationshipStore relationshipStore = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier);
+ EntityGraphMapper entityGraphMapper = new EntityGraphMapper(deleteDelegate, typeRegistry, graph, relationshipStore, entityChangeNotifier, instanceConverter, fullTextMapperV2);
+
+ return new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, entityChangeNotifier, entityGraphMapper);
}
private void shutdownEntityCreationManager(EntityCreationManager creationManager) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java
index ecce1b0..3f7e861 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java
@@ -31,6 +31,7 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
@@ -57,15 +58,18 @@ import static org.apache.atlas.repository.store.graph.v2.BulkImporterImpl.update
public class RegularImport extends ImportStrategy {
private static final Logger LOG = LoggerFactory.getLogger(RegularImport.class);
private static final int MAX_ATTEMPTS = 3;
+
+ private final AtlasGraph graph;
private final AtlasEntityStore entityStore;
private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityGraphRetriever;
private boolean directoryBasedImportConfigured;
- public RegularImport(AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
+ public RegularImport(AtlasGraph graph, AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
+ this.graph = graph;
this.entityStore = entityStore;
this.typeRegistry = typeRegistry;
- this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
+ this.entityGraphRetriever = new EntityGraphRetriever(graph, typeRegistry);
this.directoryBasedImportConfigured = StringUtils.isNotEmpty(AtlasConfiguration.IMPORT_TEMP_DIRECTORY.getString());
}
@@ -156,7 +160,7 @@ public class RegularImport extends ImportStrategy {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
String vertexGuid = null;
try {
- vertexGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(entityType, objectId.getUniqueAttributes());
+ vertexGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(this.graph, entityType, objectId.getUniqueAttributes());
} catch (AtlasBaseException e) {
LOG.warn("Entity: {}: Does not exist!", objectId);
return;
@@ -166,7 +170,7 @@ public class RegularImport extends ImportStrategy {
return;
}
- AtlasVertex v = AtlasGraphUtilsV2.findByGuid(vertexGuid);
+ AtlasVertex v = AtlasGraphUtilsV2.findByGuid(this.graph, vertexGuid);
if (v == null) {
return;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
index fc7d392..6b5b523 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
@@ -30,7 +30,9 @@ import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStreamForImport;
import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.repository.store.graph.v2.EntityStream;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.AtlasPerfMetrics;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
@@ -49,23 +51,30 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
private AtomicLong counter = new AtomicLong(1);
private AtomicLong currentBatch = new AtomicLong(1);
- private final AtlasGraph atlasGraph;
- private final AtlasEntityStore entityStoreV2;
+ private AtlasGraph atlasGraph;
+ private final AtlasEntityStore entityStore;
+ private final AtlasGraph atlasGraphBulk;
+ private final AtlasEntityStore entityStoreBulk;
private final AtlasTypeRegistry typeRegistry;
- private final EntityGraphRetriever entityGraphRetriever;
+ private final EntityGraphRetriever entityRetrieverBulk;
private List<AtlasEntity.AtlasEntityWithExtInfo> entityBuffer = new ArrayList<>();
private List<EntityMutationResponse> localResults = new ArrayList<>();
- public EntityConsumer(AtlasGraph atlasGraph, AtlasEntityStore entityStore,
- EntityGraphRetriever entityGraphRetriever, AtlasTypeRegistry typeRegistry,
+ public EntityConsumer(AtlasTypeRegistry typeRegistry,
+ AtlasGraph atlasGraph, AtlasEntityStore entityStore,
+ AtlasGraph atlasGraphBulk, AtlasEntityStore entityStoreBulk, EntityGraphRetriever entityRetrieverBulk,
BlockingQueue queue, int batchSize) {
super(queue);
+ this.typeRegistry = typeRegistry;
this.atlasGraph = atlasGraph;
- this.entityStoreV2 = entityStore;
- this.entityGraphRetriever = entityGraphRetriever;
- this.typeRegistry = typeRegistry;
+ this.entityStore = entityStore;
+
+ this.atlasGraphBulk = atlasGraphBulk;
+ this.entityStoreBulk = entityStoreBulk;
+ this.entityRetrieverBulk = entityRetrieverBulk;
+
this.batchSize = batchSize;
}
@@ -77,7 +86,6 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
long currentCount = counter.addAndGet(delta);
currentBatch.addAndGet(delta);
- entityBuffer.add(entityWithExtInfo);
try {
processEntity(entityWithExtInfo, currentCount);
@@ -88,24 +96,68 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
}
private void processEntity(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, long currentCount) {
- try {
- RequestContext.get().setImportInProgress(true);
- RequestContext.get().setCreateShellEntityForNonExistingReference(true);
-
- AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
+ RequestContext.get().setImportInProgress(true);
+ RequestContext.get().setCreateShellEntityForNonExistingReference(true);
+ try {
LOG.debug("Processing: {}", currentCount);
- EntityMutationResponse result = entityStoreV2.createOrUpdateForImportNoCommit(oneEntityStream);
- localResults.add(result);
+ importUsingBulkEntityStore(entityWithExtInfo);
+ } catch (IllegalStateException | IllegalArgumentException e) {
+ LOG.warn("{}: {} - {}", e.getClass().getSimpleName(), entityWithExtInfo.getEntity().getTypeName(), entityWithExtInfo.getEntity().getGuid(), e);
+ importUsingRegularEntityStore(entityWithExtInfo, e);
} catch (AtlasBaseException e) {
- addResult(entityWithExtInfo.getEntity().getGuid());
- LOG.warn("Exception: {}", entityWithExtInfo.getEntity().getGuid(), e);
+ LOG.warn("AtlasBaseException: {} - {}", entityWithExtInfo.getEntity().getTypeName(), entityWithExtInfo.getEntity().getGuid(), e);
} catch (AtlasSchemaViolationException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Entity: {}", entityWithExtInfo.getEntity().getGuid(), e);
}
- BulkImporterImpl.updateVertexGuid(typeRegistry, entityGraphRetriever, entityWithExtInfo.getEntity());
+ BulkImporterImpl.updateVertexGuid(this.atlasGraphBulk, typeRegistry, entityRetrieverBulk, entityWithExtInfo.getEntity());
+ }
+ }
+
+ private void importUsingBulkEntityStore(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException {
+ EntityStream oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
+ EntityMutationResponse result = entityStoreBulk.createOrUpdateForImportNoCommit(oneEntityStream);
+ localResults.add(result);
+ entityBuffer.add(entityWithExtInfo);
+ }
+
+ private void importUsingRegularEntityStore(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, Exception ex) {
+ commitValidatedEntities(ex);
+ performRegularImport(entityWithExtInfo);
+ }
+
+ private void performRegularImport(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ synchronized (atlasGraph) {
+ try {
+ LOG.info("Regular: EntityStore: {}: Starting...", this.counter.get());
+ AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
+ EntityMutationResponse result = this.entityStore.createOrUpdateForImportNoCommit(oneEntityStream);
+ atlasGraph.commit();
+ localResults.add(result);
+ dispatchResults();
+ } catch (Exception e) {
+ atlasGraph.rollback();
+ LOG.error("Regular: EntityStore: Rollback!: Entity creation using regular (non-bulk) failed! Please correct entity and re-submit!", e);
+ } finally {
+ LOG.info("Regular: EntityStore: {}: Commit: Done!", this.counter.get());
+ atlasGraph.commit();
+ addResult(entityWithExtInfo.getEntity().getGuid());
+ clear();
+ LOG.info("Regular: EntityStore: {}: Done!", this.counter.get());
+ }
+ }
+ }
+
+ private void commitValidatedEntities(Exception ex) {
+ try {
+ LOG.info("Validated Entities: Commit: Starting...");
+ rollbackPauseRetry(1, ex);
+ doCommit();
+ }
+ finally {
+ LOG.info("Validated Entities: Commit: Done!");
}
}
@@ -137,8 +189,10 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
}
private boolean commitWithRetry(int retryCount) {
+ AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("commitWithRetry");
+
try {
- atlasGraph.commit();
+ atlasGraphBulk.commit();
if (LOG.isDebugEnabled()) {
LOG.debug("Commit: Done!: Buffer: {}: Batch: {}: Counter: {}", entityBuffer.size(), currentBatch.get(), counter.get());
}
@@ -148,14 +202,15 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
} catch (Exception ex) {
rollbackPauseRetry(retryCount, ex);
return false;
+ } finally {
+ RequestContext.get().endMetricRecord(metric);
}
}
private void rollbackPauseRetry(int retryCount, Exception ex) {
- atlasGraph.rollback();
- clearCache();
+ bulkGraphRollback(retryCount);
- LOG.error("Rollback: Done! Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount);
+ LOG.warn("Rollback: Done! Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount);
pause(retryCount);
String exceptionClass = ex.getClass().getSimpleName();
if (!exceptionClass.equals("JanusGraphException") && !exceptionClass.equals("PermanentLockingException")) {
@@ -164,14 +219,27 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
retryProcessEntity(retryCount);
}
+ private void bulkGraphRollback(int retryCount) {
+ try {
+ atlasGraphBulk.rollback();
+ clearCache();
+ } catch (Exception e) {
+ LOG.error("Rollback: Exception! Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount);
+ }
+ }
+
private void retryProcessEntity(int retryCount) {
if (LOG.isDebugEnabled() || retryCount > 1) {
LOG.info("Replaying: Starting!: Buffer: {}: Retry count: {}", entityBuffer.size(), retryCount);
}
- for (AtlasEntity.AtlasEntityWithExtInfo e : entityBuffer) {
+ List<AtlasEntity.AtlasEntityWithExtInfo> localBuffer = new ArrayList<>(entityBuffer);
+ entityBuffer.clear();
+
+ for (AtlasEntity.AtlasEntityWithExtInfo e : localBuffer) {
processEntity(e, counter.get());
}
+
LOG.info("Replaying: Done!: Buffer: {}: Retry count: {}", entityBuffer.size(), retryCount);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java
index 69d33b2..7eac8df 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java
@@ -22,6 +22,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.pc.WorkItemBuilder;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -30,21 +31,33 @@ import java.util.concurrent.BlockingQueue;
public class EntityConsumerBuilder implements WorkItemBuilder<EntityConsumer, AtlasEntity.AtlasEntityWithExtInfo> {
private AtlasGraph atlasGraph;
private AtlasEntityStore entityStore;
- private final EntityGraphRetriever entityGraphRetriever;
+ private AtlasGraph atlasGraphBulk;
+ private AtlasEntityStore entityStoreBulk;
+ private final EntityGraphRetriever entityRetriever;
private final AtlasTypeRegistry typeRegistry;
+ private EntityGraphRetriever entityRetrieverBulk;
private int batchSize;
- public EntityConsumerBuilder(AtlasGraph atlasGraph, AtlasEntityStore entityStore,
- EntityGraphRetriever entityGraphRetriever, AtlasTypeRegistry typeRegistry, int batchSize) {
+ public EntityConsumerBuilder(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, AtlasEntityStoreV2 entityStore, EntityGraphRetriever entityRetriever,
+ AtlasGraph atlasGraphBulk, AtlasEntityStoreV2 entityStoreBulk, EntityGraphRetriever entityRetrieverBulk,
+ int batchSize) {
+ this.typeRegistry = typeRegistry;
+
this.atlasGraph = atlasGraph;
this.entityStore = entityStore;
- this.entityGraphRetriever = entityGraphRetriever;
- this.typeRegistry = typeRegistry;
+ this.entityRetriever = entityRetriever;
+
+ this.atlasGraphBulk = atlasGraphBulk;
+ this.entityStoreBulk = entityStoreBulk;
+ this.entityRetrieverBulk = entityRetrieverBulk;
+
this.batchSize = batchSize;
}
@Override
public EntityConsumer build(BlockingQueue<AtlasEntity.AtlasEntityWithExtInfo> queue) {
- return new EntityConsumer(atlasGraph, entityStore, entityGraphRetriever, typeRegistry, queue, this.batchSize);
+ return new EntityConsumer(typeRegistry, atlasGraph, entityStore,
+ atlasGraphBulk, entityStoreBulk, entityRetrieverBulk,
+ queue, this.batchSize);
}
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
index f8b7eda7..93dbce5 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportSkipLineageTest.java
@@ -26,6 +26,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.AtlasTestBase;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
@@ -66,6 +67,9 @@ public class ExportSkipLineageTest extends AtlasTestBase {
@Inject
ExportService exportService;
+ @Inject
+ AtlasGraph atlasGraph;
+
private DeleteHandlerDelegate deleteDelegate = mock(DeleteHandlerDelegate.class);
private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
private AtlasEntityStoreV2 entityStore;
@@ -76,7 +80,7 @@ public class ExportSkipLineageTest extends AtlasTestBase {
loadHiveModel(typeDefStore, typeRegistry);
RequestContext.get().setImportInProgress(true);
- entityStore = new AtlasEntityStoreV2(deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
+ entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
createEntities(entityStore, ENTITIES_SUB_DIR, new String[]{"db", "table-columns", "table-view", "table-table-lineage"});
final String[] entityGuids = {DB_GUID, TABLE_GUID, TABLE_TABLE_GUID, TABLE_VIEW_GUID};
verifyCreatedEntities(entityStore, entityGuids, 4);
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
index 7567daa..252255f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
@@ -51,9 +51,9 @@ public class HiveParititionTest extends MigrationBaseAsserts {
assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95");
assertTypeCountNameGuid("hive_process", 1, "", "");
- assertEdges("hive_db", "parts_db", AtlasEdgeDirection.IN,1, 1, "");
- assertEdges("hive_table", "t1", AtlasEdgeDirection.OUT, 1, 1, "hive_table_db");
- assertEdges("hive_table", "tv1", AtlasEdgeDirection.OUT, 1, 1, "hive_table_db");
+ assertEdges("hive_db", "parts_db", AtlasEdgeDirection.IN, 1, "");
+ assertEdges("hive_table", "t1", AtlasEdgeDirection.OUT, 1, "hive_table_db");
+ assertEdges("hive_table", "tv1", AtlasEdgeDirection.OUT, 1, "hive_table_db");
assertMigrationStatus(EXPECTED_TOTAL_COUNT);
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
index 90e714c..c73d368 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
@@ -59,9 +59,9 @@ public class HiveStocksTest extends MigrationBaseAsserts {
assertTypeCountNameGuid("hive_storagedesc", 1, "", "294290d8-4498-4677-973c-c266d594b039");
assertTypeCountNameGuid("Tag1", 1, "", "");
- assertEdges(getVertex("hive_db", "stocks").getEdges(AtlasEdgeDirection.IN).iterator(),1, 1, "");
- assertEdges(getVertex("hive_table", "stocks_daily").getEdges(AtlasEdgeDirection.OUT).iterator(), 1, 1, "hive_table_db");
- assertEdges(getVertex("hive_column", "high").getEdges(AtlasEdgeDirection.OUT).iterator(), 1,1, "hive_table_columns");
+ assertEdges(getVertex("hive_db", "stocks").getEdges(AtlasEdgeDirection.IN).iterator(), 1, "");
+ assertEdges(getVertex("hive_table", "stocks_daily").getEdges(AtlasEdgeDirection.OUT).iterator(), 1, "hive_table_db");
+ assertEdges(getVertex("hive_column", "high").getEdges(AtlasEdgeDirection.OUT).iterator(), 1, "hive_table_columns");
assertMigrationStatus(EXPECTED_TOTAL_COUNT);
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
index ce9056b..bd8359c 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
@@ -133,25 +133,33 @@ public class MigrationBaseAsserts {
return iterator.hasNext() ? iterator.next() : null;
}
- protected void assertEdges(String typeName, String assetName, AtlasEdgeDirection edgeDirection, int startIdx, int expectedItems, String edgeTypeName) {
- assertEdges(getVertex(typeName, assetName).getEdges(edgeDirection).iterator(),startIdx, expectedItems, edgeTypeName);
+ protected void assertEdges(String typeName, String assetName, AtlasEdgeDirection edgeDirection, int expectedItems, String edgeTypeName) {
+ Iterator edgeIterator = getVertex(typeName, assetName).getEdges(edgeDirection).iterator();
+ assertEdges(edgeIterator, expectedItems, edgeTypeName);
}
- protected void assertEdges(Iterator<AtlasEdge> results, int startIdx, int expectedItems, String edgeTypeName) {
+ protected void assertEdges(Iterator<AtlasEdge> results, int expectedItems, String edgeTypeNameExpected) {
int count = 0;
AtlasEdge e = null;
- for (Iterator<AtlasEdge> it = results; it.hasNext() && count < startIdx; count++) {
+ boolean searchedEdgeFound = false;
+ for (Iterator<AtlasEdge> it = results; it.hasNext();) {
e = it.next();
+ String typeName = AtlasGraphUtilsV2.getEncodedProperty(e, TYPE_NAME_PROPERTY, String.class);
+ searchedEdgeFound = StringUtils.isEmpty(edgeTypeNameExpected) || typeName.equals(edgeTypeNameExpected);
+ if (searchedEdgeFound) {
+ count++;
+ break;
+ }
}
assertNotNull(AtlasGraphUtilsV2.getEncodedProperty(e, R_GUID_PROPERTY_NAME, Object.class));
assertNotNull(AtlasGraphUtilsV2.getEncodedProperty(e, "tagPropagation", Object.class));
- if(StringUtils.isNotEmpty(edgeTypeName)) {
- assertEquals(AtlasGraphUtilsV2.getEncodedProperty(e, TYPE_NAME_PROPERTY, Object.class), edgeTypeName, edgeTypeName);
+ if(StringUtils.isNotEmpty(edgeTypeNameExpected)) {
+ assertTrue(searchedEdgeFound, edgeTypeNameExpected);
}
- assertEquals(count, expectedItems, String.format("%s", edgeTypeName));
+ assertEquals(count, expectedItems, String.format("%s", edgeTypeNameExpected));
}
protected void assertEdgesWithLabel(Iterator<AtlasEdge> results, int startIdx, String edgeTypeName) {
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
index 38228a8..b9cbef1 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
@@ -125,7 +125,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
}
@BeforeTest
public void init() throws Exception {
- entityStore = new AtlasEntityStoreV2(deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
+ entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
RequestContext.clear();
RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java
index 368951d..752f1ac 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityTestBase.java
@@ -107,7 +107,7 @@ public class AtlasEntityTestBase {
@BeforeTest
public void init() throws Exception {
- entityStore = new AtlasEntityStoreV2(deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
+ entityStore = new AtlasEntityStoreV2(graph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
RequestContext.clear();
RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java
index a40cf85..24683a5 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2Test.java
@@ -32,6 +32,7 @@ import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
@@ -86,6 +87,9 @@ public abstract class AtlasRelationshipStoreV2Test {
@Inject
AtlasEntityChangeNotifier entityNotifier;
+ @Inject
+ AtlasGraph atlasGraph;
+
AtlasEntityStore entityStore;
AtlasRelationshipStore relationshipStore;
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@@ -124,8 +128,8 @@ public abstract class AtlasRelationshipStoreV2Test {
@BeforeTest
public void init() throws Exception {
- entityStore = new AtlasEntityStoreV2(deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
- relationshipStore = new AtlasRelationshipStoreV2(typeRegistry, deleteDelegate, entityNotifier);
+ entityStore = new AtlasEntityStoreV2(atlasGraph, deleteDelegate, typeRegistry, mockChangeNotifier, graphMapper);
+ relationshipStore = new AtlasRelationshipStoreV2(atlasGraph, typeRegistry, deleteDelegate, entityNotifier);
RequestContext.clear();
RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java
index 37acbe5..d31b464 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociatorTest.java
@@ -20,15 +20,13 @@ package org.apache.atlas.repository.store.graph.v2;
import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasEntityHeaders;
import org.apache.atlas.model.typedef.AtlasEntityDef;
-import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.audit.EntityAuditRepository;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.AtlasJson;
import org.apache.atlas.utils.TestResourceFileUtils;
@@ -37,7 +35,6 @@ import org.elasticsearch.common.util.CollectionUtils;
import org.testng.annotations.Test;
import java.io.IOException;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -65,13 +62,13 @@ public class ClassificationAssociatorTest {
private class ClassificationAssociatorUpdaterForSpy extends ClassificationAssociator.Updater {
private final String entityFileName;
- public ClassificationAssociatorUpdaterForSpy(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
- super(typeRegistry, entitiesStore);
+ public ClassificationAssociatorUpdaterForSpy(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore) {
+ super(atlasGraph, typeRegistry, entitiesStore);
this.entityFileName = StringUtils.EMPTY;
}
- public ClassificationAssociatorUpdaterForSpy(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, String entityFileName) {
- super(typeRegistry, entitiesStore);
+ public ClassificationAssociatorUpdaterForSpy(AtlasGraph atlasGraph, AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, String entityFileName) {
+ super(atlasGraph, typeRegistry, entitiesStore);
this.entityFileName = entityFileName;
}
@@ -142,7 +139,8 @@ public class ClassificationAssociatorTest {
AtlasTypeRegistry typeRegistry = mock(AtlasTypeRegistry.class);
when(typeRegistry.getEntityTypeByName(anyString())).thenReturn(null);
- ClassificationAssociator.Updater updater = new ClassificationAssociator.Updater(typeRegistry, entitiesStore);
+ AtlasGraph atlasGraph = mock(AtlasGraph.class);
+ ClassificationAssociator.Updater updater = new ClassificationAssociator.Updater(atlasGraph, typeRegistry, entitiesStore);
String summary = updater.setClassifications(entityHeaderMap.getGuidHeaderMap());
assertTrue(summary.contains("hive_"));
@@ -155,11 +153,12 @@ public class ClassificationAssociatorTest {
AtlasEntityType hiveTable = mock(AtlasEntityType.class);
AtlasEntityStore entitiesStore = mock(AtlasEntityStore.class);
AtlasTypeRegistry typeRegistry = mock(AtlasTypeRegistry.class);
+ AtlasGraph atlasGraph = mock(AtlasGraph.class);
when(typeRegistry.getEntityTypeByName(anyString())).thenReturn(hiveTable);
when(hiveTable.getTypeName()).thenReturn("hive_column");
- ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(typeRegistry, entitiesStore);
+ ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(atlasGraph, typeRegistry, entitiesStore);
String summary = updater.setClassifications(entityHeaderMap.getGuidHeaderMap());
TypeReference<String[]> typeReference = new TypeReference<String[]>() {};
@@ -178,8 +177,9 @@ public class ClassificationAssociatorTest {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = typeRegistry.lockTypeRegistryForUpdate();
ttr.addTypes(CollectionUtils.newSingletonArrayList(ed));
+ AtlasGraph atlasGraph = mock(AtlasGraph.class);
- ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(ttr, entitiesStore, "col-entity-PII");
+ ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(atlasGraph, ttr, entitiesStore, "col-entity-PII");
String summary = updater.setClassifications(entityHeaderMap.getGuidHeaderMap());
TypeReference<String[]> typeReference = new TypeReference<String[]>() {};
@@ -233,11 +233,12 @@ public class ClassificationAssociatorTest {
AtlasEntityType hiveTable = mock(AtlasEntityType.class);
AtlasEntityStore entitiesStore = mock(AtlasEntityStore.class);
AtlasTypeRegistry typeRegistry = mock(AtlasTypeRegistry.class);
+ AtlasGraph atlasGraph = mock(AtlasGraph.class);
when(typeRegistry.getEntityTypeByName(anyString())).thenReturn(hiveTable);
when(hiveTable.getTypeName()).thenReturn("hive_column");
- ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(typeRegistry, entitiesStore, entityFileName);
+ ClassificationAssociatorUpdaterForSpy updater = new ClassificationAssociatorUpdaterForSpy(atlasGraph, typeRegistry, entitiesStore, entityFileName);
String summary = updater.setClassifications(entityHeaderMap.getGuidHeaderMap());
TypeReference<String[]> typeReference = new TypeReference<String[]>() {};
[atlas] 11/20: ATLAS-3735: CustomAttribute length validation bypass
for keys with special prefix.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 0030bc23acd1bb36462f77ba795fe42f8bda423d
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon Apr 20 17:49:29 2020 -0700
ATLAS-3735: CustomAttribute length validation bypass for keys with special prefix.
(cherry picked from commit 318c645fb555b17b9fdd8c2f1b642a3065e337fc)
---
intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 ++
.../apache/atlas/repository/store/graph/v2/EntityGraphMapper.java | 6 ++++++
2 files changed, 8 insertions(+)
diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
index 5d8776b..2c007ca 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -64,6 +64,8 @@ public enum AtlasConfiguration {
CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH("atlas.custom.attribute.key.max.length", 50),
CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH("atlas.custom.attribute.value.max.length", 500),
+ CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX("atlas.custom.attribute.special.prefix", ""),
+
LABEL_MAX_LENGTH("atlas.entity.label.max.length", 50),
IMPORT_TEMP_DIRECTORY("atlas.import.temp.directory", ""),
MIGRATION_IMPORT_START_POSITION("atlas.migration.import.start.position", 0),
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index d949ed0..8c43807 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -119,6 +119,8 @@ public class EntityGraphMapper {
private static final String SOFT_REF_FORMAT = "%s:%s";
private static final int INDEXED_STR_SAFE_LEN = AtlasConfiguration.GRAPHSTORE_INDEXED_STRING_SAFE_LENGTH.getInt();
private static final boolean WARN_ON_NO_RELATIONSHIP = AtlasConfiguration.RELATIONSHIP_WARN_NO_RELATIONSHIPS.getBoolean();
+ private static final String CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX = AtlasConfiguration.CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX.getString();
+
private static final String CLASSIFICATION_NAME_DELIMITER = "|";
private static final Pattern CUSTOM_ATTRIBUTE_KEY_REGEX = Pattern.compile("^[a-zA-Z0-9_-]*$");
private static final Pattern LABEL_REGEX = Pattern.compile("^[a-zA-Z0-9_-]*$");
@@ -2521,6 +2523,10 @@ public class EntityGraphMapper {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_KEY_CHARACTERS, key);
}
+ if (StringUtils.isNotEmpty(CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX) && key.startsWith(CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX)) {
+ continue;
+ }
+
if (value.length() > CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CUSTOM_ATTRIBUTE_VALUE, value, String.valueOf(CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH));
}
[atlas] 09/20: ATLAS-3663: ZipFileMigrator: Automatic Resume During
Migration.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 206c3ed4d6ac686ae03adedd16764f09024658bd
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon Mar 16 09:58:27 2020 -0700
ATLAS-3663: ZipFileMigrator: Automatic Resume During Migration.
(cherry picked from commit 3d0c98779f16c7f0c6faa2b441877458ea152675)
---
.../atlas/model/impexp/AtlasImportRequest.java | 3 +-
.../model/migration/MigrationImportStatus.java | 62 ++++++
.../java/org/apache/atlas/pc/StatusReporter.java | 41 +++-
.../org/apache/atlas/pc/StatusReporterTest.java | 18 ++
.../atlas/repository/impexp/ImportService.java | 10 +-
.../impexp/MigrationProgressService.java | 42 +++-
.../migration/DataMigrationStatusService.java | 245 +++++++++++++++++++++
.../migration/ZipFileMigrationImporter.java | 43 +++-
.../store/graph/v2/bulkimport/MigrationImport.java | 21 +-
.../graph/v2/bulkimport/pc/EntityConsumer.java | 5 +-
.../v2/bulkimport/pc/EntityCreationManager.java | 19 +-
.../impexp/DataMigrationStatusServiceTest.java | 66 ++++++
12 files changed, 548 insertions(+), 27 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 09dafdf..2c18704 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -44,12 +44,13 @@ public class AtlasImportRequest implements Serializable {
public static final String TRANSFORMS_KEY = "transforms";
public static final String TRANSFORMERS_KEY = "transformers";
public static final String OPTION_KEY_REPLICATED_FROM = "replicatedFrom";
+ public static final String OPTION_KEY_MIGRATION_FILE_NAME = "migrationFileName";
public static final String OPTION_KEY_MIGRATION = "migration";
public static final String OPTION_KEY_NUM_WORKERS = "numWorkers";
public static final String OPTION_KEY_BATCH_SIZE = "batchSize";
public static final String OPTION_KEY_FORMAT = "format";
public static final String OPTION_KEY_FORMAT_ZIP_DIRECT = "zipDirect";
- private static final String START_POSITION_KEY = "startPosition";
+ public static final String START_POSITION_KEY = "startPosition";
private static final String START_GUID_KEY = "startGuid";
private static final String FILE_NAME_KEY = "fileName";
private static final String UPDATE_TYPE_DEFINITION_KEY = "updateTypeDefinition";
diff --git a/intg/src/main/java/org/apache/atlas/model/migration/MigrationImportStatus.java b/intg/src/main/java/org/apache/atlas/model/migration/MigrationImportStatus.java
new file mode 100644
index 0000000..3430fda
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/migration/MigrationImportStatus.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.model.migration;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import org.apache.atlas.model.AtlasBaseModelObject;
+import org.apache.atlas.model.impexp.MigrationStatus;
+import org.apache.commons.lang.StringUtils;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MigrationImportStatus extends MigrationStatus {
+ private String name;
+
+ public MigrationImportStatus() {
+ }
+
+ public MigrationImportStatus(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(", name=").append(name);
+ sb.append(super.toString());
+ return sb.toString();
+ }
+}
diff --git a/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java b/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
index f84e8d0..7baf973 100644
--- a/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
+++ b/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
@@ -18,6 +18,9 @@
package org.apache.atlas.pc;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
@@ -25,8 +28,20 @@ import java.util.Map;
import java.util.Set;
public class StatusReporter<T, U> {
+ private static final Logger LOG = LoggerFactory.getLogger(StatusReporter.class);
+
private Map<T,U> producedItems = new LinkedHashMap<>();
private Set<T> processedSet = new HashSet<>();
+ private long timeoutDuration;
+ private long lastAck;
+
+ public StatusReporter() {
+ this.timeoutDuration = -1;
+ }
+
+ public StatusReporter(long timeoutDurationInMs) {
+ this.timeoutDuration = timeoutDurationInMs;
+ }
public void produced(T item, U index) {
this.producedItems.put(item, index);
@@ -44,7 +59,8 @@ public class StatusReporter<T, U> {
U ack = null;
U ret;
do {
- ret = completionIndex(getFirstElement(this.producedItems));
+ Map.Entry<T, U> firstElement = getFirstElement(this.producedItems);
+ ret = completionIndex(firstElement);
if (ret != null) {
ack = ret;
}
@@ -63,13 +79,32 @@ public class StatusReporter<T, U> {
private U completionIndex(Map.Entry<T, U> lookFor) {
U ack = null;
- if (lookFor == null || !processedSet.contains(lookFor.getKey())) {
+ if (lookFor == null) {
return ack;
}
- ack = lookFor.getValue();
+ if (hasTimeoutDurationReached(System.currentTimeMillis())) {
+ LOG.warn("Ack: Timeout: {} - {}", lookFor.getKey(), lookFor.getValue());
+ return acknowledged(lookFor);
+ }
+
+ if (!processedSet.contains(lookFor.getKey())) {
+ return ack;
+ }
+
+ return acknowledged(lookFor);
+ }
+
+ private U acknowledged(Map.Entry<T, U> lookFor) {
+ U ack = lookFor.getValue();
producedItems.remove(lookFor.getKey());
processedSet.remove(lookFor);
return ack;
}
+
+ private boolean hasTimeoutDurationReached(long now) {
+ boolean b = (this.timeoutDuration > -1) && (this.lastAck != 0) && ((now - this.lastAck) >= timeoutDuration);
+ lastAck = System.currentTimeMillis();
+ return b;
+ }
}
diff --git a/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java b/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
index 3e50562..45bdbb0 100644
--- a/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
+++ b/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
@@ -23,6 +23,8 @@ import org.testng.annotations.Test;
import java.util.concurrent.BlockingQueue;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
public class StatusReporterTest {
private static class IntegerConsumer extends WorkItemConsumer<Integer> {
@@ -91,4 +93,20 @@ public class StatusReporterTest {
statusReporter.processed((Integer) result);
}
}
+
+ @Test
+ public void reportWithTimeout() throws InterruptedException {
+ StatusReporter<Integer, Integer> statusReporter = new StatusReporter<>(2000);
+ statusReporter.produced(1, 100);
+ statusReporter.produced(2, 200);
+
+ statusReporter.processed(2);
+ Integer ack = statusReporter.ack();
+ assertNull(ack);
+
+ Thread.sleep(3000);
+ ack = statusReporter.ack();
+ assertNotNull(ack);
+ assertEquals(ack, Integer.valueOf(200));
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index c18c4ab..1d29bf8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -235,6 +235,10 @@ public class ImportService {
result.incrementMeticsCounter("duration", getDuration(this.endTimestamp, this.startTimestamp));
result.setOperationStatus(AtlasImportResult.OperationStatus.SUCCESS);
+ if (isMigrationMode(result.getRequest())) {
+ return;
+ }
+
auditsWriter.write(userName, result, startTimestamp, endTimestamp, importSource.getCreationOrder());
}
@@ -250,7 +254,7 @@ public class ImportService {
private EntityImportStream createZipSource(AtlasImportRequest request, InputStream inputStream, String configuredTemporaryDirectory) throws AtlasBaseException {
try {
- if (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_MIGRATION) || (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_FORMAT) &&
+ if (isMigrationMode(request) || (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_FORMAT) &&
request.getOptions().get(AtlasImportRequest.OPTION_KEY_FORMAT).equals(AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT))) {
LOG.info("ZipSource Format: ZipDirect: Size: {}", request.getOptions().get("size"));
return getZipDirectEntityImportStream(request, inputStream);
@@ -288,4 +292,8 @@ public class ImportService {
exportRequest.getFetchTypeOptionValue().equalsIgnoreCase(AtlasExportRequest.FETCH_TYPE_INCREMENTAL) &&
exportRequest.getSkipLineageOptionValue();
}
+
+ private boolean isMigrationMode(AtlasImportRequest request) {
+ return request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_MIGRATION);
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java
index 54ae32a..6bb5f1e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java
@@ -19,35 +19,61 @@
package org.apache.atlas.repository.impexp;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.annotation.AtlasService;
import org.apache.atlas.model.impexp.MigrationStatus;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.GraphDBMigrator;
+import org.apache.atlas.repository.migration.DataMigrationStatusService;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
+import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME;
+
@AtlasService
@Singleton
public class MigrationProgressService {
private static final Logger LOG = LoggerFactory.getLogger(MigrationProgressService.class);
+ private static final String FILE_EXTENSION_ZIP = ".zip";
public static final String MIGRATION_QUERY_CACHE_TTL = "atlas.migration.query.cache.ttlInSecs";
@VisibleForTesting
- static long DEFAULT_CACHE_TTL_IN_SECS = 30 * 1000; // 30 secs
+ static long DEFAULT_CACHE_TTL_IN_SECS = 120 * 1000; // 30 secs
private final long cacheValidity;
private final GraphDBMigrator migrator;
private MigrationStatus cachedStatus;
private long cacheExpirationTime = 0;
+ private DataMigrationStatusService dataMigrationStatusService;
+ private boolean zipFileBasedMigrationImport;
@Inject
public MigrationProgressService(Configuration configuration, GraphDBMigrator migrator) {
this.migrator = migrator;
this.cacheValidity = (configuration != null) ? configuration.getLong(MIGRATION_QUERY_CACHE_TTL, DEFAULT_CACHE_TTL_IN_SECS) : DEFAULT_CACHE_TTL_IN_SECS;
+
+ this.zipFileBasedMigrationImport = isZipFileBasedMigrationEnabled();
+ initConditionallyZipFileBasedMigrator();
+ }
+
+ private void initConditionallyZipFileBasedMigrator() {
+ if (!zipFileBasedMigrationImport) {
+ return;
+ }
+
+ dataMigrationStatusService = new DataMigrationStatusService(AtlasGraphProvider.getGraphInstance());
+ dataMigrationStatusService.init(getFileNameFromMigrationProperty());
+ }
+
+ private boolean isZipFileBasedMigrationEnabled() {
+ return StringUtils.endsWithIgnoreCase(getFileNameFromMigrationProperty(), FILE_EXTENSION_ZIP);
}
public MigrationStatus getStatus() {
@@ -58,7 +84,11 @@ public class MigrationProgressService {
long currentTime = System.currentTimeMillis();
if(resetCache(currentTime)) {
- cachedStatus = migrator.getMigrationStatus();
+ if (this.zipFileBasedMigrationImport) {
+ cachedStatus = dataMigrationStatusService.getStatus();
+ } else {
+ cachedStatus = migrator.getMigrationStatus();
+ }
}
return cachedStatus;
@@ -73,4 +103,12 @@ public class MigrationProgressService {
return ret;
}
+
+ public String getFileNameFromMigrationProperty() {
+ try {
+ return ApplicationProperties.get().getString(ATLAS_MIGRATION_MODE_FILENAME, StringUtils.EMPTY);
+ } catch (AtlasException e) {
+ return StringUtils.EMPTY;
+ }
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
new file mode 100644
index 0000000..8c7a3a8
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import org.apache.atlas.model.migration.MigrationImportStatus;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Date;
+import java.util.Iterator;
+
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getEncodedProperty;
+import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.setEncodedProperty;
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.encodePropertyKey;
+import static org.apache.atlas.type.Constants.INTERNAL_PROPERTY_KEY_PREFIX;
+
+public class DataMigrationStatusService {
+ private static final Logger LOG = LoggerFactory.getLogger(DataMigrationStatusService.class);
+ private final MigrationStatusVertexManagement migrationStatusVertexManagement;
+
+ private MigrationImportStatus status;
+
+ public DataMigrationStatusService() {
+ this.migrationStatusVertexManagement = new MigrationStatusVertexManagement(AtlasGraphProvider.getGraphInstance());
+ }
+
+ public DataMigrationStatusService(AtlasGraph atlasGraph) {
+ this.migrationStatusVertexManagement = new MigrationStatusVertexManagement(atlasGraph);
+ }
+
+
+ public void init(String fileToImport) {
+ this.status = new MigrationImportStatus(fileToImport);
+ if (!this.migrationStatusVertexManagement.exists(fileToImport)) {
+ return;
+ }
+
+ getCreate(fileToImport);
+ }
+
+ public MigrationImportStatus getCreate(String fileName) {
+ return getCreate(new MigrationImportStatus(fileName));
+ }
+
+ public MigrationImportStatus getCreate(MigrationImportStatus status) {
+ try {
+ this.status = this.migrationStatusVertexManagement.createOrUpdate(status);
+ } catch (Exception ex) {
+ LOG.error("DataMigrationStatusService: Setting status: {}: Resulted in error!", status.getName(), ex);
+ }
+
+ return this.status;
+ }
+
+ public MigrationImportStatus getStatus() {
+ if (this.status != null &&
+ StringUtils.isEmpty(this.status.getOperationStatus()) &&
+ this.migrationStatusVertexManagement.exists(this.status.getName())) {
+ return getCreate(this.status);
+ } else {
+ return this.status;
+ }
+ }
+
+ public MigrationImportStatus getByName(String name) {
+ return this.migrationStatusVertexManagement.findByName(name);
+ }
+
+ public void delete() {
+ if (this.status == null) {
+ return;
+ }
+
+ MigrationImportStatus status = getByName(this.status.getName());
+ this.migrationStatusVertexManagement.delete(status.getName());
+ this.status = null;
+ }
+
+ public void savePosition(String position) {
+ this.status.setCurrentIndex(Long.valueOf(position));
+ this.migrationStatusVertexManagement.updateVertexPartial(this.status);
+ }
+
+ public void setStatus(String status) {
+ this.status.setOperationStatus(status);
+ this.migrationStatusVertexManagement.updateVertexPartial(this.status);
+ }
+
+ private static class MigrationStatusVertexManagement {
+ public static final String PROPERTY_KEY_START_TIME = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "migration.startTime");
+ public static final String PROPERTY_KEY_SIZE = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "migration.size");
+ public static final String PROPERTY_KEY_POSITION = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "migration.position");
+ public static final String PROPERTY_KEY_STATUS = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "migration.status");
+
+ private AtlasGraph atlasGraph;
+ private AtlasVertex vertex;
+
+ public MigrationStatusVertexManagement(AtlasGraph atlasGraph) {
+ this.atlasGraph = atlasGraph;
+ }
+
+ public MigrationImportStatus createOrUpdate(MigrationImportStatus status) {
+ this.vertex = findByNameInternal(status.getName());
+
+ if (this.vertex == null) {
+ this.vertex = atlasGraph.addVertex();
+ LOG.info("MigrationStatusVertexManagement: Vertex created!");
+ updateVertex(this.vertex, status);
+ }
+
+ return to(this.vertex);
+ }
+
+ public boolean exists(String name) {
+ return findByNameInternal(name) != null;
+ }
+
+ public MigrationImportStatus findByName(String name) {
+ if (this.vertex != null) {
+ return to(this.vertex);
+ }
+
+ AtlasVertex v = findByNameInternal(name);
+ if (v == null) {
+ return null;
+ }
+
+ this.vertex = v;
+ LOG.info("MigrationImportStatus: Vertex found!");
+ return to(v);
+ }
+
+ public void delete(String name) {
+ try {
+ AtlasVertex vertex = findByNameInternal(name);
+ atlasGraph.removeVertex(vertex);
+ this.vertex = null;
+ } finally {
+ atlasGraph.commit();
+ }
+ }
+
+ private AtlasVertex findByNameInternal(String name) {
+ try {
+ String idxQueryString = String.format("%s\"%s\":\"%s\"", AtlasGraphUtilsV2.getIndexSearchPrefix(), Constants.GUID_PROPERTY_KEY, name);
+ AtlasIndexQuery idxQuery = atlasGraph.indexQuery(Constants.VERTEX_INDEX, idxQueryString);
+ Iterator<AtlasIndexQuery.Result<Object, Object>> results = idxQuery.vertices();
+
+ AtlasIndexQuery.Result<?, ?> qryResult = results.hasNext() ? results.next() : null;
+ if (qryResult != null) {
+ return qryResult.getVertex();
+ } else {
+ return null;
+ }
+ } catch (Exception e) {
+ LOG.error("MigrationStatusVertexManagement.findByNameInternal: Failed!", e);
+ } finally {
+ atlasGraph.commit();
+ }
+
+ return null;
+ }
+
+ public void updateVertexPartial(MigrationImportStatus status) {
+ try {
+ setEncodedProperty(vertex, PROPERTY_KEY_POSITION, status.getCurrentIndex());
+ } catch (Exception e) {
+ LOG.warn("Error updating status. Please rely on log messages.", e);
+ } finally {
+ atlasGraph.commit();
+ }
+ }
+
+ private void updateVertex(AtlasVertex vertex, MigrationImportStatus status) {
+ try {
+ setEncodedProperty(vertex, Constants.GUID_PROPERTY_KEY, status.getName());
+
+ setEncodedProperty(vertex, PROPERTY_KEY_START_TIME,
+ (status.getStartTime() != null)
+ ? status.getStartTime().getTime()
+ : System.currentTimeMillis());
+
+ setEncodedProperty(vertex, PROPERTY_KEY_SIZE, status.getTotalCount());
+ setEncodedProperty(vertex, PROPERTY_KEY_POSITION, status.getCurrentIndex());
+ setEncodedProperty(vertex, PROPERTY_KEY_STATUS, status.getOperationStatus());
+ } catch (Exception ex) {
+ LOG.error("Error updating MigrationImportStatus vertex. Status may not be persisted correctly.", ex);
+ } finally {
+ atlasGraph.commit();
+ }
+ }
+
+ private static MigrationImportStatus to(AtlasVertex vertex) {
+ MigrationImportStatus ret = new MigrationImportStatus();
+
+ try {
+ ret.setName(getEncodedProperty(vertex, Constants.GUID_PROPERTY_KEY, String.class));
+
+ Long dateValue = getEncodedProperty(vertex, PROPERTY_KEY_START_TIME, Long.class);
+ if (dateValue != null) {
+ ret.setStartTime(new Date(dateValue));
+ }
+
+ Long size = getEncodedProperty(vertex, PROPERTY_KEY_SIZE, Long.class);
+ if (size != null) {
+ ret.setTotalCount(size);
+ }
+
+ Long position = getEncodedProperty(vertex, PROPERTY_KEY_POSITION, Long.class);
+ if (position != null) {
+ ret.setCurrentIndex(position);
+ }
+
+ ret.setOperationStatus(getEncodedProperty(vertex, PROPERTY_KEY_STATUS, String.class));
+ } catch (Exception ex) {
+ LOG.error("Error converting to MigrationImportStatus. Will proceed with default values.", ex);
+ }
+
+ return ret;
+ }
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
index 35a76ea..f44f2a8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
@@ -23,6 +23,8 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.migration.MigrationImportStatus;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.impexp.ImportService;
import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
@@ -41,7 +43,7 @@ public class ZipFileMigrationImporter implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ZipFileMigrationImporter.class);
private static final String APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS = "atlas.migration.mode.workers";
- private static final String APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE = "atlas.migration.mode.batch.size";
+ private static final String APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE = "atlas.migration.mode.batch.size";
private static final String DEFAULT_NUMBER_OF_WORKERS = "4";
private static final String DEFAULT_BATCH_SIZE = "100";
private static final String ZIP_FILE_COMMENT_ENTITIES_COUNT = "entitiesCount";
@@ -51,20 +53,24 @@ public class ZipFileMigrationImporter implements Runnable {
private final ImportService importService;
private final String fileToImport;
+ private DataMigrationStatusService dataMigrationStatusService;
+ private MigrationImportStatus migrationImportStatus;
public ZipFileMigrationImporter(ImportService importService, String fileName) {
this.importService = importService;
this.fileToImport = fileName;
+ this.dataMigrationStatusService = new DataMigrationStatusService(AtlasGraphProvider.getGraphInstance());
}
@Override
public void run() {
try {
- FileWatcher fileWatcher = new FileWatcher(fileToImport);
- fileWatcher.start();
+ detectFileToImport();
int streamSize = getStreamSizeFromComment(fileToImport);
- performImport(new FileInputStream(new File(fileToImport)), streamSize);
+ migrationImportStatus = getCreateMigrationStatus(fileToImport, streamSize);
+ performImport(fileToImport, streamSize, Long.toString(migrationImportStatus.getCurrentIndex()));
+ dataMigrationStatusService.setStatus("DONE");
} catch (IOException e) {
LOG.error("Migration Import: IO Error!", e);
} catch (AtlasBaseException e) {
@@ -72,6 +78,22 @@ public class ZipFileMigrationImporter implements Runnable {
}
}
+ private MigrationImportStatus getCreateMigrationStatus(String fileName, int streamSize) {
+ MigrationImportStatus status = new MigrationImportStatus(fileName);
+ status.setTotalCount(streamSize);
+
+ MigrationImportStatus statusRetrieved = dataMigrationStatusService.getCreate(status);
+
+ LOG.info("DataMigrationStatusService: Position: {}", statusRetrieved.getCurrentIndex());
+ dataMigrationStatusService.setStatus("STARTED");
+ return statusRetrieved;
+ }
+
+ private void detectFileToImport() throws IOException {
+ FileWatcher fileWatcher = new FileWatcher(fileToImport);
+ fileWatcher.start();
+ }
+
private int getStreamSizeFromComment(String fileToImport) {
int ret = 1;
try {
@@ -99,13 +121,13 @@ public class ZipFileMigrationImporter implements Runnable {
return entitiesCount;
}
- private void performImport(InputStream fs, int streamSize) throws AtlasBaseException {
+ private void performImport(String fileToImport, int streamSize, String startPosition) throws AtlasBaseException {
try {
- LOG.info("Migration Import: {}: Starting...", fileToImport);
-
+ LOG.info("Migration Import: {}: Starting at: {}...", fileToImport, startPosition);
+ InputStream fs = new FileInputStream(new File(fileToImport));
RequestContext.get().setUser(getUserNameFromEnvironment(), null);
- importService.run(fs, getImportRequest(streamSize),
+ importService.run(fs, getImportRequest(fileToImport, streamSize, startPosition),
getUserNameFromEnvironment(),
InetAddress.getLocalHost().getHostName(),
InetAddress.getLocalHost().getHostAddress());
@@ -122,16 +144,19 @@ public class ZipFileMigrationImporter implements Runnable {
return System.getProperty(ENV_USER_NAME);
}
- private AtlasImportRequest getImportRequest(int streamSize) throws AtlasException {
+ private AtlasImportRequest getImportRequest(String fileToImport, int streamSize, String position) throws AtlasException {
AtlasImportRequest request = new AtlasImportRequest();
+ request.setOption(AtlasImportRequest.OPTION_KEY_MIGRATION_FILE_NAME, fileToImport);
request.setSizeOption(streamSize);
request.setOption(AtlasImportRequest.OPTION_KEY_MIGRATION, "true");
request.setOption(AtlasImportRequest.OPTION_KEY_NUM_WORKERS, getPropertyValue(APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS, DEFAULT_NUMBER_OF_WORKERS));
request.setOption(AtlasImportRequest.OPTION_KEY_BATCH_SIZE, getPropertyValue(APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE, DEFAULT_BATCH_SIZE));
+ request.setOption(AtlasImportRequest.START_POSITION_KEY, (StringUtils.isEmpty(position) ? "0" : position));
return request;
}
+
private String getPropertyValue(String property, String defaultValue) throws AtlasException {
return ApplicationProperties.get().getString(property, defaultValue);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
index 4c912fd..ff55e40 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
@@ -20,12 +20,14 @@ package org.apache.atlas.repository.store.graph.v2.bulkimport;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.converters.AtlasFormatConverters;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.migration.DataMigrationStatusService;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
@@ -65,10 +67,12 @@ public class MigrationImport extends ImportStrategy {
throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "importResult should contain request");
}
- int index = 0;
+ DataMigrationStatusService dataMigrationStatusService = createMigrationStatusService(importResult);
+
+ long index = 0;
int streamSize = entityStream.size();
EntityMutationResponse ret = new EntityMutationResponse();
- EntityCreationManager creationManager = createEntityCreationManager(atlasGraph, importResult);
+ EntityCreationManager creationManager = createEntityCreationManager(atlasGraph, importResult, dataMigrationStatusService);
try {
LOG.info("Migration Import: Size: {}: Starting...", streamSize);
@@ -85,14 +89,23 @@ public class MigrationImport extends ImportStrategy {
return ret;
}
- private EntityCreationManager createEntityCreationManager(AtlasGraph threadedAtlasGraph, AtlasImportResult importResult) {
+ private DataMigrationStatusService createMigrationStatusService(AtlasImportResult importResult) {
+ DataMigrationStatusService dataMigrationStatusService = new DataMigrationStatusService();
+ dataMigrationStatusService.init(importResult.getRequest().getOptions().get(AtlasImportRequest.OPTION_KEY_MIGRATION_FILE_NAME));
+ return dataMigrationStatusService;
+ }
+
+ private EntityCreationManager createEntityCreationManager(AtlasGraph threadedAtlasGraph,
+ AtlasImportResult importResult,
+ DataMigrationStatusService dataMigrationStatusService) {
+ atlasGraph = threadedAtlasGraph;
int batchSize = importResult.getRequest().getOptionKeyBatchSize();
int numWorkers = getNumWorkers(importResult.getRequest().getOptionKeyNumWorkers());
EntityConsumerBuilder consumerBuilder =
new EntityConsumerBuilder(threadedAtlasGraph, entityStore, entityGraphRetriever, typeRegistry, batchSize);
- return new EntityCreationManager(consumerBuilder, batchSize, numWorkers, importResult);
+ return new EntityCreationManager(consumerBuilder, batchSize, numWorkers, importResult, dataMigrationStatusService);
}
private static int getNumWorkers(int numWorkersFromOptions) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
index e8f4b02..d0fac10 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
@@ -155,7 +155,10 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
LOG.error("Rollback: Done! Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount);
pause(retryCount);
- LOG.warn("Commit error! Will pause and retry: Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount, ex);
+ String exceptionClass = ex.getClass().getSimpleName();
+ if (!exceptionClass.equals("JanusGraphException") && !exceptionClass.equals("PermanentLockingException")) {
+ LOG.warn("Commit error! Will pause and retry: Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount, ex);
+ }
retryProcessEntity(retryCount);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
index 16bb49e..177b563 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
@@ -23,6 +23,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.pc.StatusReporter;
import org.apache.atlas.pc.WorkItemBuilder;
import org.apache.atlas.pc.WorkItemManager;
+import org.apache.atlas.repository.migration.DataMigrationStatusService;
import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
import org.apache.commons.lang.StringUtils;
@@ -32,24 +33,28 @@ import org.slf4j.LoggerFactory;
public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManager {
private static final Logger LOG = LoggerFactory.getLogger(EntityCreationManager.class);
private static final String WORKER_PREFIX = "migration-import";
+ private static final long STATUS_REPORT_TIMEOUT_DURATION = 1 * 60 * 1000; // 5 min
private final StatusReporter<String, String> statusReporter;
private final AtlasImportResult importResult;
+ private final DataMigrationStatusService dataMigrationStatusService;
private String currentTypeName;
private float currentPercent;
private EntityImportStream entityImportStream;
- public EntityCreationManager(WorkItemBuilder builder, int batchSize, int numWorkers, AtlasImportResult importResult) {
+ public EntityCreationManager(WorkItemBuilder builder, int batchSize, int numWorkers, AtlasImportResult importResult, DataMigrationStatusService dataMigrationStatusService) {
super(builder, WORKER_PREFIX, batchSize, numWorkers, true);
this.importResult = importResult;
+ this.dataMigrationStatusService = dataMigrationStatusService;
- this.statusReporter = new StatusReporter<>();
+ this.statusReporter = new StatusReporter<>(STATUS_REPORT_TIMEOUT_DURATION);
}
- public int read(EntityImportStream entityStream) {
- int currentIndex = 0;
+ public long read(EntityImportStream entityStream) {
+ long currentIndex = this.dataMigrationStatusService.getStatus().getCurrentIndex();
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo;
this.entityImportStream = entityStream;
+ this.dataMigrationStatusService.setStatus("IN_PROGRESS");
while ((entityWithExtInfo = entityStream.getNextEntityWithExtInfo()) != null) {
AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
if (entity == null) {
@@ -66,7 +71,7 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
return currentIndex;
}
- private void produce(int currentIndex, String typeName, AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ private void produce(long currentIndex, String typeName, AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
String previousTypeName = getCurrentTypeName();
if (StringUtils.isNotEmpty(typeName)
@@ -104,7 +109,9 @@ public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManag
}
importResult.incrementMeticsCounter(split[0]);
- this.currentPercent = updateImportMetrics(split[0], Integer.parseInt(split[1]), this.entityImportStream.size(), getCurrentPercent());
+ String currentPosition = split[1];
+ dataMigrationStatusService.savePosition(currentPosition);
+ this.currentPercent = updateImportMetrics(split[0], Integer.parseInt(currentPosition), this.entityImportStream.size(), getCurrentPercent());
}
private static float updateImportMetrics(String typeNameGuid, int currentIndex, int streamSize, float currentPercent) {
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
new file mode 100644
index 0000000..bf1d9a0
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.migration.MigrationImportStatus;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.migration.DataMigrationStatusService;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.util.Date;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class DataMigrationStatusServiceTest {
+ @Inject
+ AtlasGraph atlasGraph;
+
+ @Test
+ public void createUpdateDelete() throws AtlasBaseException {
+ DataMigrationStatusService dataMigrationStatusService = new DataMigrationStatusService(atlasGraph);
+
+ MigrationImportStatus expected = new MigrationImportStatus("/tmp/defg.zip");
+ expected.setTotalCount(3333);
+ expected.setCurrentIndex(20);
+ expected.setStartTime(new Date());
+
+ MigrationImportStatus ret = dataMigrationStatusService.getCreate(expected);
+
+ assertNotNull(ret);
+ assertEquals(ret.getName(), expected.getName());
+ assertEquals(ret.getStartTime(), expected.getStartTime());
+ assertEquals(ret.getTotalCount(), expected.getTotalCount());
+ assertEquals(ret.getCurrentIndex(), expected.getCurrentIndex());
+
+ dataMigrationStatusService.savePosition("100");
+ assertNotNull(dataMigrationStatusService.getStatus());
+ assertNotNull(dataMigrationStatusService.getStatus().getCurrentIndex(), "100");
+ assertNotNull(dataMigrationStatusService.getCreate(expected).getCurrentIndex(), "100");
+
+ dataMigrationStatusService.delete();
+ assertNull(dataMigrationStatusService.getStatus());
+ assertNull(dataMigrationStatusService.getByName(ret.getName()));
+ }
+}
[atlas] 02/20: ATLAS-3773: Shadow Attributes Update Post-Migration
from 0.8.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 599e7c45a8a5412c8c5b2c24ffaba878011e9558
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Fri May 1 10:54:41 2020 -0700
ATLAS-3773: Shadow Attributes Update Post-Migration from 0.8.
(cherry picked from commit a43b5488fbd10c874d181bf27e076fd502693a93)
---
.../repository/patches/UniqueAttributePatch.java | 43 +++++++++++++++++++---
1 file changed, 37 insertions(+), 6 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
index 2b58119..d3111f1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/UniqueAttributePatch.java
@@ -23,7 +23,11 @@ import org.apache.atlas.pc.WorkItemManager;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer.UniqueKind;
-import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.graphdb.AtlasCardinality;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasEntityType;
@@ -159,10 +163,37 @@ public class UniqueAttributePatch extends AtlasPatchHandler {
protected void processItem(Long vertexId, AtlasVertex vertex, String typeName, AtlasEntityType entityType) {
LOG.debug("processItem(typeName={}, vertexId={})", typeName, vertexId);
+ processIndexStringAttribute(vertexId, vertex, typeName, entityType);
+ processUniqueAttribute(vertexId, vertex, typeName, entityType);
+
+ LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);
+ }
+
+ private void processIndexStringAttribute(Long vertexId, AtlasVertex vertex, String typeName, AtlasEntityType entityType) {
+ for (AtlasAttribute attribute : entityType.getAllAttributes().values()) {
+ if (attribute.getAttributeDef().getIndexType() != null &&
+ attribute.getAttributeDef().getIndexType() == AtlasAttributeDef.IndexType.STRING) {
+
+ String vertexPropertyName = attribute.getVertexPropertyName();
+ if (vertex.getProperty(vertexPropertyName, String.class) != null) {
+ continue;
+ }
+
+ Object attrVal = AtlasGraphUtilsV2.getEncodedProperty(vertex, attribute.getQualifiedName(), String.class);
+ if (attrVal != null) {
+ AtlasGraphUtilsV2.setEncodedProperty(vertex, vertexPropertyName, attrVal);
+ }
+ }
+ }
+
+ LOG.debug("processIndexStringAttribute(typeName={}, vertexId={}): Done!", typeName, vertexId);
+ }
+
+ private void processUniqueAttribute(Long vertexId, AtlasVertex vertex, String typeName, AtlasEntityType entityType) {
for (AtlasAttribute attribute : entityType.getUniqAttributes().values()) {
- String uniquePropertyKey = attribute.getVertexUniquePropertyName();
- Collection<? extends String> propertyKeys = vertex.getPropertyKeys();
- Object uniqAttrValue = null;
+ String uniquePropertyKey = attribute.getVertexUniquePropertyName();
+ Collection<? extends String> propertyKeys = vertex.getPropertyKeys();
+ Object uniqAttrValue = null;
if (propertyKeys == null || !propertyKeys.contains(uniquePropertyKey)) {
try {
@@ -171,14 +202,14 @@ public class UniqueAttributePatch extends AtlasPatchHandler {
uniqAttrValue = EntityGraphRetriever.mapVertexToPrimitive(vertex, propertyKey, attribute.getAttributeDef());
AtlasGraphUtilsV2.setEncodedProperty(vertex, uniquePropertyKey, uniqAttrValue);
- } catch(AtlasSchemaViolationException ex) {
+ } catch (AtlasSchemaViolationException ex) {
LOG.error("Duplicates detected: {}:{}:{}", typeName, uniqAttrValue, getIdFromVertex(vertex));
vertex.removeProperty(uniquePropertyKey);
}
}
}
- LOG.debug("processItem(typeName={}, vertexId={}): Done!", typeName, vertexId);
+ LOG.debug("processUniqueAttribute(typeName={}, vertexId={}): Done!", typeName, vertexId);
}
}
}
[atlas] 17/20: ATLAS-3799: EntityConumer only adds entity GUIDs
that are added when they were produced.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 9a1b2cc28e6c325b2075dcfec307ed99df0fff26
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Fri May 15 11:56:23 2020 -0700
ATLAS-3799: EntityConumer only adds entity GUIDs that are added when they were produced.
(cherry picked from commit 3de30f55d808df6245f60fe01fed4b6cf894a31d)
---
.../graph/v2/bulkimport/pc/EntityConsumer.java | 25 +++++-----------------
1 file changed, 5 insertions(+), 20 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
index 6b5b523..34b0561 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
@@ -59,7 +59,7 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
private final EntityGraphRetriever entityRetrieverBulk;
private List<AtlasEntity.AtlasEntityWithExtInfo> entityBuffer = new ArrayList<>();
- private List<EntityMutationResponse> localResults = new ArrayList<>();
+ private List<String> localResults = new ArrayList<>();
public EntityConsumer(AtlasTypeRegistry typeRegistry,
AtlasGraph atlasGraph, AtlasEntityStore entityStore,
@@ -119,7 +119,7 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
private void importUsingBulkEntityStore(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException {
EntityStream oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
EntityMutationResponse result = entityStoreBulk.createOrUpdateForImportNoCommit(oneEntityStream);
- localResults.add(result);
+ localResults.add(entityWithExtInfo.getEntity().getGuid());
entityBuffer.add(entityWithExtInfo);
}
@@ -133,9 +133,9 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
try {
LOG.info("Regular: EntityStore: {}: Starting...", this.counter.get());
AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
- EntityMutationResponse result = this.entityStore.createOrUpdateForImportNoCommit(oneEntityStream);
+ this.entityStore.createOrUpdateForImportNoCommit(oneEntityStream);
atlasGraph.commit();
- localResults.add(result);
+ localResults.add(entityWithExtInfo.getEntity().getGuid());
dispatchResults();
} catch (Exception e) {
atlasGraph.rollback();
@@ -244,12 +244,7 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
}
private void dispatchResults() {
- localResults.stream().forEach(x -> {
- addResultsFromResponse(x.getCreatedEntities());
- addResultsFromResponse(x.getUpdatedEntities());
- addResultsFromResponse(x.getDeletedEntities());
- });
-
+ localResults.stream().forEach(x -> addResult(x));
clear();
}
@@ -261,16 +256,6 @@ public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWith
}
}
- private void addResultsFromResponse(List<AtlasEntityHeader> entities) {
- if (CollectionUtils.isEmpty(entities)) {
- return;
- }
-
- for (AtlasEntityHeader eh : entities) {
- addResult(eh.getGuid());
- }
- }
-
private void clear() {
localResults.clear();
entityBuffer.clear();
[atlas] 20/20: Revert "ATLAS-3758: support sort params in
FreeTextSearchProcessor - #2"
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 98e3d123f05e37b4b84b8a699f0c09238c701182
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Tue Jun 2 13:04:23 2020 -0700
Revert "ATLAS-3758: support sort params in FreeTextSearchProcessor - #2"
This reverts commit 6b69e49cbc99a776b422ac0f4a2d62670c1a730e.
---
test-tools/src/main/resources/solr/core-template/solrconfig.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test-tools/src/main/resources/solr/core-template/solrconfig.xml b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
index 531458c..ef74a0a 100644
--- a/test-tools/src/main/resources/solr/core-template/solrconfig.xml
+++ b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
@@ -434,7 +434,7 @@
-->
<lst name="defaults">
<str name="defType">edismax</str>
- <str name="qf">3c3p_t^3 3if9_s^3 3nyd_s^3 3ll1_t^3 3k05_s^3 3mdh_s^3 cn9_t^3 x6t_t^3 yrp_t^3 iyt_t^3</str>
+ <str name="qf">7shx_t^3 7qx1_t^3 3pj9_t^3 7cp1_t^3 43r9_t^3 cn9_t^1 79j9_t^3 3vut_t^3 3thh_t^3 3qbp_s^3 7wg5_t^3 7b45_t^3 3nyd_s^3 7vnp_t^3 7rph_t^3 8ydh_t^3 x6t_t^3 7tad_t^3 7jt1_t^3 3mdh_s^3 7e9x_t^3 3rwl_s^3 93wl_t^3 yrp_t^10 7pc5_t^3 7u2t_t^3 3xfp_t^3 426d_t^3 iyt_t^10 3g1x_t^3 40lh_t^3</str>
<str name="hl.fl">*</str>
<bool name="hl.requireFieldMatch">true</bool>
<bool name="lowercaseOperators">true</bool>
[atlas] 01/20: ATLAS-3649: Table-level Imports: Deleting Tables
Differing By Requests Fails
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit dee69accd8ad77cf178fe5112865d00cc386f73f
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Wed Mar 4 12:23:11 2020 -0800
ATLAS-3649: Table-level Imports: Deleting Tables Differing By Requests Fails
(cherry picked from commit 668addac196bafcea7f10f19ef74428f74f778fd)
---
.../impexp/TableReplicationRequestProcessor.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessor.java b/repository/src/main/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessor.java
index d5807a5..1757cb7 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/TableReplicationRequestProcessor.java
@@ -45,7 +45,7 @@ import java.util.HashSet;
public class TableReplicationRequestProcessor {
private static final Logger LOG = LoggerFactory.getLogger(TableReplicationRequestProcessor.class);
- private static final String QUERY_DB_NAME_EQUALS= "where db.name='%s'";
+ private static final String QUERY_DB_NAME_EQUALS = "qualifiedName startsWith '%s'";
private static final String ATTR_NAME_KEY = "name";
private static final String TYPE_HIVE_TABLE = "hive_table";
private static final String ATTR_QUALIFIED_NAME_KEY = "qualifiedName";
@@ -83,7 +83,7 @@ public class TableReplicationRequestProcessor {
deleteTables(sourceCluster, guidsToDelete);
}
- private List<String> getQualifiedNamesFromRequest(AtlasExportRequest exportRequest){
+ private List<String> getQualifiedNamesFromRequest(AtlasExportRequest exportRequest) {
List<String> qualifiedNames = new ArrayList<>();
for (AtlasObjectId objectId : exportRequest.getItemsToExport()) {
@@ -95,7 +95,7 @@ public class TableReplicationRequestProcessor {
private List<String> getEntitiesFromQualifiedNames(List<String> qualifiedNames) throws AtlasBaseException {
List<String> safeGUIDs = new ArrayList<>();
- for(String qualifiedName : qualifiedNames) {
+ for (String qualifiedName : qualifiedNames) {
String guid = getGuidByUniqueAttributes(Collections.singletonMap(ATTR_QUALIFIED_NAME_KEY, qualifiedName));
safeGUIDs.add(guid);
}
@@ -133,9 +133,18 @@ public class TableReplicationRequestProcessor {
if (CollectionUtils.isEmpty(searchResult.getEntities())) {
break;
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("getGuidsToDelete: {}", searchResult.getApproximateCount());
+ }
}
+ String classificationName = String.format(REPLICATED_TAG_NAME, sourceCluster);
for (AtlasEntityHeader entityHeader : searchResult.getEntities()) {
+ if (!entityHeader.getClassificationNames().contains(classificationName)) {
+ continue;
+ }
+
String guid = entityHeader.getGuid();
if (!excludeGUIDs.contains(guid)) {
unsafeGUIDs.add(guid);
@@ -155,8 +164,7 @@ public class TableReplicationRequestProcessor {
parameters.setTypeName(TYPE_HIVE_TABLE);
parameters.setExcludeDeletedEntities(true);
- parameters.setClassification(String.format(REPLICATED_TAG_NAME, sourceCluster));
- parameters.setAttributes(new HashSet<String>(){{ add(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM); }});
+ parameters.setAttributes(new HashSet<String>() {{ add(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM); }});
parameters.setQuery(query);
return parameters;
[atlas] 19/20: ATLAS-3722: ZipFileMigrationImporter: Allow
AtlasPatchService to Run Before Migration Starts
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit bd9e0c91ca3d16afdc092d835c8567553d90c8bc
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Thu May 28 09:40:43 2020 -0700
ATLAS-3722: ZipFileMigrationImporter: Allow AtlasPatchService to Run Before Migration Starts
(cherry picked from commit 00bef883c5561596ccc2b7b1075e8379bf8ac6bb)
---
common/src/main/java/org/apache/atlas/service/Services.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/common/src/main/java/org/apache/atlas/service/Services.java b/common/src/main/java/org/apache/atlas/service/Services.java
index 75eba03..2d54846 100644
--- a/common/src/main/java/org/apache/atlas/service/Services.java
+++ b/common/src/main/java/org/apache/atlas/service/Services.java
@@ -108,7 +108,8 @@ public class Services {
private boolean isNeededForZipFileMigration(Service svc) {
return svc.getClass().getSuperclass().getSimpleName().equals("AbstractStorageBasedAuditRepository") ||
- svc.getClass().getSuperclass().getSimpleName().equals("AbstractNotification");
+ svc.getClass().getSuperclass().getSimpleName().equals("AbstractNotification") ||
+ svc.getClass().getSimpleName().equals("AtlasPatchService");
}
private boolean isDataMigrationService(Service svc) {
[atlas] 15/20: ATLAS-3796: Status Reporter display fix.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit f1297e34f4d0675ab23b531ad3dc9549381ea0bc
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Fri May 15 11:49:51 2020 -0700
ATLAS-3796: Status Reporter display fix.
(cherry picked from commit 935d0ab6465775e16db62896375d96f6a567b884)
---
intg/src/main/java/org/apache/atlas/pc/StatusReporter.java | 10 +++++++++-
intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java | 3 +++
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java b/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
index 7baf973..507e842 100644
--- a/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
+++ b/intg/src/main/java/org/apache/atlas/pc/StatusReporter.java
@@ -98,7 +98,7 @@ public class StatusReporter<T, U> {
private U acknowledged(Map.Entry<T, U> lookFor) {
U ack = lookFor.getValue();
producedItems.remove(lookFor.getKey());
- processedSet.remove(lookFor);
+ processedSet.remove(lookFor.getKey());
return ack;
}
@@ -107,4 +107,12 @@ public class StatusReporter<T, U> {
lastAck = System.currentTimeMillis();
return b;
}
+
+ public int getProducedCount() {
+ return this.producedItems.size();
+ }
+
+ public int getProcessedCount() {
+ return this.processedSet.size();
+ }
}
diff --git a/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java b/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
index 45bdbb0..a0c399b 100644
--- a/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
+++ b/intg/src/test/java/org/apache/atlas/pc/StatusReporterTest.java
@@ -82,6 +82,9 @@ public class StatusReporterTest {
extractResults(wi, statusReporter);
assertEquals(statusReporter.ack().intValue(), (maxItems - 1));
wi.shutdown();
+
+ assertEquals(statusReporter.getProducedCount(), 0);
+ assertEquals(statusReporter.getProcessedCount(), 0);
}
private void extractResults(WorkItemManager<Integer, WorkItemConsumer> wi, StatusReporter<Integer, Integer> statusReporter) {
[atlas] 14/20: ATLAS-3762: Improve edge creation using genuine
iterator. Part 2
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit fe5fa7f56e2d9872a199dbb872b642e74ae33bef
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon May 4 18:46:17 2020 -0700
ATLAS-3762: Improve edge creation using genuine iterator. Part 2
(cherry picked from commit 23aea76fff6c8f530816319649218583f4dfd091)
---
.../atlas/repository/graphdb/AtlasGraph.java | 13 +++++++++--
.../repository/graphdb/janus/AtlasJanusGraph.java | 11 ++++++++++
.../apache/atlas/repository/graph/GraphHelper.java | 20 +++++++++--------
.../store/graph/v2/AtlasRelationshipStoreV2.java | 25 +---------------------
4 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index 7bd5f2d..c016f63 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -48,10 +48,19 @@ public interface AtlasGraph<V, E> {
AtlasEdge<V, E> addEdge(AtlasVertex<V, E> outVertex, AtlasVertex<V, E> inVertex, String label);
/**
- * Adds a vertex to the graph.
- *
+ * Fetch edges between two vertices using relationshipLabel
+ * @param fromVertex
+ * @param toVertex
+ * @param relationshipLabel
* @return
*/
+ AtlasEdge<V, E> getEdgeBetweenVertices(AtlasVertex fromVertex, AtlasVertex toVertex, String relationshipLabel);
+
+ /**
+ * Adds a vertex to the graph.
+ *
+ * @return
+ */
AtlasVertex<V, E> addVertex();
/**
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index a30dbc7..35274c9 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -172,6 +172,17 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
@Override
+ public AtlasEdge getEdgeBetweenVertices(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
+ GraphTraversal gt = V(fromVertex.getId()).outE(edgeLabel).where(__.otherV().hasId(toVertex.getId()));
+ Object o = gt.hasNext() ? gt.next() : null;
+ if (o == null) {
+ return null;
+ }
+
+ return GraphDbObjectFactory.createEdge(this, (Edge) o);
+ }
+
+ @Override
public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> getEdge(String edgeId) {
Iterator<Edge> it = getGraph().edges(edgeId);
Edge e = getSingleElement(it, edgeId);
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index b9e3a5e..b1368e5 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -739,19 +739,21 @@ public final class GraphHelper {
public static List<AtlasVertex> getPropagationEnabledClassificationVertices(AtlasVertex entityVertex) {
List<AtlasVertex> ret = new ArrayList<>();
- Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges();
+ if (entityVertex.hasEdges(AtlasEdgeDirection.OUT, CLASSIFICATION_LABEL)) {
+ Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges();
- if (edges != null) {
- Iterator<AtlasEdge> iterator = edges.iterator();
+ if (edges != null) {
+ Iterator<AtlasEdge> iterator = edges.iterator();
- while (iterator.hasNext()) {
- AtlasEdge edge = iterator.next();
+ while (iterator.hasNext()) {
+ AtlasEdge edge = iterator.next();
- if (edge != null) {
- AtlasVertex classificationVertex = edge.getInVertex();
+ if (edge != null) {
+ AtlasVertex classificationVertex = edge.getInVertex();
- if (isPropagationEnabled(classificationVertex)) {
- ret.add(classificationVertex);
+ if (isPropagationEnabled(classificationVertex)) {
+ ret.add(classificationVertex);
+ }
}
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index ab431bc..8d74489 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -784,36 +784,13 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
AtlasEdge ret = null;
if (toVertex.hasEdges(AtlasEdgeDirection.IN, relationshipLabel) && fromVertex.hasEdges(AtlasEdgeDirection.OUT, relationshipLabel)) {
- long fromVertexOutgoingEdgeCount = graphHelper.getOutGoingEdgesCountByLabel(fromVertex, relationshipLabel);
- long toVertexIncomingEdgeCount = graphHelper.getInComingEdgesCountByLabel(toVertex, relationshipLabel);
- if (toVertexIncomingEdgeCount < fromVertexOutgoingEdgeCount) {
- Iterator<AtlasEdge> edgesIteratorIn = graphHelper.getIncomingEdgesByLabel(toVertex, relationshipLabel);
- ret = getActiveEdgeFromList(edgesIteratorIn, fromVertex.getId(), e -> e.getOutVertex().getId());
- } else {
- Iterator<AtlasEdge> edgesIteratorOut = graphHelper.getOutGoingEdgesByLabel(fromVertex, relationshipLabel);
- ret = getActiveEdgeFromList(edgesIteratorOut, toVertex.getId(), e -> e.getInVertex().getId());
- }
+ ret = graph.getEdgeBetweenVertices(fromVertex, toVertex, relationshipLabel);
}
RequestContext.get().endMetricRecord(metric);
return ret;
}
- private AtlasEdge getActiveEdgeFromList(Iterator<AtlasEdge> edgesIterator, Object vertexIdToCompare, Function<AtlasEdge, Object> edgeIdFn) {
- while (edgesIterator != null && edgesIterator.hasNext()) {
- AtlasEdge edge = edgesIterator.next();
- if (edge != null) {
- Status status = graphHelper.getStatus(edge);
-
- if ((status == null || status == ACTIVE) && edgeIdFn.apply(edge).equals(vertexIdToCompare)) {
- return edge;
- }
- }
- }
-
- return null;
- }
-
private Long getRelationshipVersion(AtlasRelationship relationship) {
Long ret = relationship != null ? relationship.getVersion() : null;
[atlas] 18/20: ATLAS-3797: Refactoring: Improve Edge Creation.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 3294324f240dca9cef7733d026a2288fea88f9d2
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Fri May 15 12:04:34 2020 -0700
ATLAS-3797: Refactoring: Improve Edge Creation.
(cherry picked from commit 0571fa42897e33a29baf4d1bd10ee04ae222dc70)
---
.../repository/graphdb/janus/AtlasJanusGraph.java | 24 +++++++++++++++++-----
.../apache/atlas/repository/graph/GraphHelper.java | 17 ++++++++-------
2 files changed, 27 insertions(+), 14 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index 35274c9..0dd573b 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -26,6 +26,7 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndexClient;
@@ -85,6 +86,7 @@ import java.util.Set;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
import static org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase.getGraphInstance;
+import static org.apache.atlas.type.Constants.STATE_PROPERTY_KEY;
/**
* Janus implementation of AtlasGraph.
@@ -174,12 +176,11 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public AtlasEdge getEdgeBetweenVertices(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
GraphTraversal gt = V(fromVertex.getId()).outE(edgeLabel).where(__.otherV().hasId(toVertex.getId()));
- Object o = gt.hasNext() ? gt.next() : null;
- if (o == null) {
- return null;
- }
- return GraphDbObjectFactory.createEdge(this, (Edge) o);
+ Edge gremlinEdge = getFirstActiveEdge(gt);
+ return (gremlinEdge != null)
+ ? GraphDbObjectFactory.createEdge(this, gremlinEdge)
+ : null;
}
@Override
@@ -567,9 +568,22 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
private final class ConvertGremlinValueFunction implements Function<Object, Object> {
+
@Override
public Object apply(Object input) {
return convertGremlinValue(input);
}
}
+ private Edge getFirstActiveEdge(GraphTraversal gt) {
+ while (gt.hasNext()) {
+ Edge gremlinEdge = (Edge) gt.next();
+ if (gremlinEdge != null && gremlinEdge.property(STATE_PROPERTY_KEY).isPresent() &&
+ gremlinEdge.property(STATE_PROPERTY_KEY).value().equals(AtlasEntity.Status.ACTIVE.toString())
+ ) {
+ return gremlinEdge;
+ }
+ }
+
+ return null;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index b1368e5..7b7ec65 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -150,21 +150,18 @@ public final class GraphHelper {
}
public AtlasEdge getOrCreateEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) throws RepositoryException {
+ AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getOrCreateEdge");
+
for (int numRetries = 0; numRetries < maxRetries; numRetries++) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Running edge creation attempt {}", numRetries);
}
- Iterator<AtlasEdge> edges = getAdjacentEdgesByLabel(inVertex, AtlasEdgeDirection.IN, edgeLabel);
-
- while (edges.hasNext()) {
- AtlasEdge edge = edges.next();
- if (edge.getOutVertex().getId().equals(outVertex.getId())) {
- Id.EntityState edgeState = getState(edge);
- if (edgeState == null || edgeState == Id.EntityState.ACTIVE) {
- return edge;
- }
+ if (inVertex.hasEdges(AtlasEdgeDirection.IN, edgeLabel) && outVertex.hasEdges(AtlasEdgeDirection.OUT, edgeLabel)) {
+ AtlasEdge edge = graph.getEdgeBetweenVertices(outVertex, inVertex, edgeLabel);
+ if (edge != null) {
+ return edge;
}
}
@@ -186,6 +183,8 @@ public final class GraphHelper {
}
}
}
+
+ RequestContext.get().endMetricRecord(metric);
return null;
}
[atlas] 08/20: ATLAS-3320: Migration Import implementation.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 1678b9b5b6bc8bad376776388f2b234e3126bedb
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Thu Mar 5 16:06:56 2020 -0800
ATLAS-3320: Migration Import implementation.
(cherry picked from commit 765ea583b23d25ecc919d520092747c4158466a5)
---
.../repository/graphdb/janus/AtlasJanusGraph.java | 2 +-
.../atlas/model/impexp/AtlasImportRequest.java | 21 ++
.../apache/atlas/GraphTransactionInterceptor.java | 4 +
.../atlas/repository/graph/FullTextMapperV2.java | 9 +-
.../atlas/repository/graph/IFullTextMapper.java | 45 +++++
.../atlas/repository/impexp/ImportService.java | 5 +-
.../atlas/repository/impexp/ZipSourceDirect.java | 8 +
.../migration/ZipFileMigrationImporter.java | 61 +++++-
.../repository/store/graph/AtlasEntityStore.java | 8 +
.../store/graph/v2/AtlasEntityStoreV2.java | 9 +-
.../store/graph/v2/AtlasRelationshipStoreV2.java | 4 +-
.../store/graph/v2/BulkImporterImpl.java | 224 +++++----------------
.../store/graph/v2/EntityGraphMapper.java | 10 +-
.../graph/v2/bulkimport/FullTextMapperV2Nop.java | 57 ++++++
.../store/graph/v2/bulkimport/ImportStrategy.java | 28 +++
.../store/graph/v2/bulkimport/MigrationImport.java | 124 ++++++++++++
.../RegularImport.java} | 77 +++----
.../graph/v2/bulkimport/pc/EntityConsumer.java | 209 +++++++++++++++++++
.../v2/bulkimport/pc/EntityConsumerBuilder.java | 50 +++++
.../v2/bulkimport/pc/EntityCreationManager.java | 126 ++++++++++++
.../test/java/org/apache/atlas/TestModules.java | 5 +
21 files changed, 833 insertions(+), 253 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index 4acb371..0176ba7 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -116,7 +116,7 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
}
- janusGraph = (StandardJanusGraph) AtlasJanusGraphDatabase.getGraphInstance();
+ janusGraph = (StandardJanusGraph) graphInstance;
}
@Override
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 3362bf1..09dafdf 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -44,6 +44,9 @@ public class AtlasImportRequest implements Serializable {
public static final String TRANSFORMS_KEY = "transforms";
public static final String TRANSFORMERS_KEY = "transformers";
public static final String OPTION_KEY_REPLICATED_FROM = "replicatedFrom";
+ public static final String OPTION_KEY_MIGRATION = "migration";
+ public static final String OPTION_KEY_NUM_WORKERS = "numWorkers";
+ public static final String OPTION_KEY_BATCH_SIZE = "batchSize";
public static final String OPTION_KEY_FORMAT = "format";
public static final String OPTION_KEY_FORMAT_ZIP_DIRECT = "zipDirect";
private static final String START_POSITION_KEY = "startPosition";
@@ -124,6 +127,24 @@ public class AtlasImportRequest implements Serializable {
return isReplicationOptionSet() ? options.get(OPTION_KEY_REPLICATED_FROM) : StringUtils.EMPTY;
}
+ @JsonIgnore
+ public int getOptionKeyNumWorkers() {
+ return getOptionsValue(OPTION_KEY_NUM_WORKERS, 1);
+ }
+
+ @JsonIgnore
+ public int getOptionKeyBatchSize() {
+ return getOptionsValue(OPTION_KEY_BATCH_SIZE, 1);
+ }
+
+ private int getOptionsValue(String optionKeyBatchSize, int defaultValue) {
+ String optionsValue = getOptionForKey(optionKeyBatchSize);
+
+ return StringUtils.isEmpty(optionsValue) ?
+ defaultValue :
+ Integer.valueOf(optionsValue);
+ }
+
@JsonAnySetter
public void setOption(String key, String value) {
if (null == options) {
diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
index bbe0dc5..57e454a 100644
--- a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
+++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
@@ -199,6 +199,10 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
return cache.get(guid);
}
+ public static void clearCache() {
+ guidVertexCache.get().clear();
+ }
+
boolean logException(Throwable t) {
if (t instanceof AtlasBaseException) {
Response.Status httpCode = ((AtlasBaseException) t).getAtlasErrorCode().getHttpCode();
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index 0f2b4bf..417c96c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -53,7 +53,7 @@ import java.util.Set;
@Component
-public class FullTextMapperV2 {
+public class FullTextMapperV2 implements IFullTextMapper {
private static final Logger LOG = LoggerFactory.getLogger(FullTextMapperV2.class);
private static final String FULL_TEXT_DELIMITER = " ";
@@ -84,6 +84,8 @@ public class FullTextMapperV2 {
* @return Full text string ONLY for the added classifications
* @throws AtlasBaseException
*/
+
+ @Override
public String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
String ret = null;
final AtlasEntityWithExtInfo entityWithExtInfo;
@@ -120,6 +122,7 @@ public class FullTextMapperV2 {
return ret;
}
+ @Override
public String getIndexTextForEntity(String guid) throws AtlasBaseException {
String ret = null;
final AtlasEntity entity;
@@ -150,6 +153,7 @@ public class FullTextMapperV2 {
return ret;
}
+ @Override
public String getClassificationTextForEntity(AtlasEntity entity) throws AtlasBaseException {
String ret = null;
@@ -271,10 +275,12 @@ public class FullTextMapperV2 {
}
}
+ @Override
public AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
return getAndCacheEntity(guid, true);
}
+ @Override
public AtlasEntity getAndCacheEntity(String guid, boolean includeReferences) throws AtlasBaseException {
RequestContext context = RequestContext.get();
AtlasEntity entity = context.getEntity(guid);
@@ -294,6 +300,7 @@ public class FullTextMapperV2 {
return entity;
}
+ @Override
public AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException {
RequestContext context = RequestContext.get();
AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid);
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/IFullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/IFullTextMapper.java
new file mode 100644
index 0000000..2bbf4d8
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/IFullTextMapper.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+
+import java.util.List;
+
+public interface IFullTextMapper {
+ /**
+ * Map newly associated/defined classifications for the entity with given GUID
+ * @param guid Entity guid
+ * @param classifications new classifications added to the entity
+ * @return Full text string ONLY for the added classifications
+ * @throws AtlasBaseException
+ */
+ String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException;
+
+ String getIndexTextForEntity(String guid) throws AtlasBaseException;
+
+ String getClassificationTextForEntity(AtlasEntity entity) throws AtlasBaseException;
+
+ AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException;
+
+ AtlasEntity getAndCacheEntity(String guid, boolean includeReferences) throws AtlasBaseException;
+
+ AtlasEntity.AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException;
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index 1964ade..c18c4ab 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -250,8 +250,9 @@ public class ImportService {
private EntityImportStream createZipSource(AtlasImportRequest request, InputStream inputStream, String configuredTemporaryDirectory) throws AtlasBaseException {
try {
- if (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_FORMAT) &&
- request.getOptions().get(AtlasImportRequest.OPTION_KEY_FORMAT).equals(AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT) ) {
+ if (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_MIGRATION) || (request.getOptions().containsKey(AtlasImportRequest.OPTION_KEY_FORMAT) &&
+ request.getOptions().get(AtlasImportRequest.OPTION_KEY_FORMAT).equals(AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT))) {
+ LOG.info("ZipSource Format: ZipDirect: Size: {}", request.getOptions().get("size"));
return getZipDirectEntityImportStream(request, inputStream);
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
index cb5a7ac..75b8e9e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSourceDirect.java
@@ -64,6 +64,10 @@ public class ZipSourceDirect implements EntityImportStream {
this.zipInputStream = new ZipInputStream(inputStream);
this.streamSize = streamSize;
prepareStreamForFetch();
+
+ if (this.streamSize == 1) {
+ LOG.info("ZipSourceDirect: Stream Size set to: {}. This will cause inaccurate percentage reporting.", this.streamSize);
+ }
}
@Override
@@ -226,6 +230,10 @@ public class ZipSourceDirect implements EntityImportStream {
}
public int size() {
+ if (this.streamSize == 1) {
+ return currentPosition;
+ }
+
return this.streamSize;
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
index f552525..35a76ea 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
@@ -24,6 +24,8 @@ import org.apache.atlas.RequestContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.repository.impexp.ImportService;
+import org.apache.atlas.type.AtlasType;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,11 +34,20 @@ import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
+import java.util.Map;
+import java.util.zip.ZipFile;
public class ZipFileMigrationImporter implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ZipFileMigrationImporter.class);
- private static String ENV_USER_NAME = "user.name";
+ private static final String APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS = "atlas.migration.mode.workers";
+ private static final String APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE = "atlas.migration.mode.batch.size";
+ private static final String DEFAULT_NUMBER_OF_WORKERS = "4";
+ private static final String DEFAULT_BATCH_SIZE = "100";
+ private static final String ZIP_FILE_COMMENT_ENTITIES_COUNT = "entitiesCount";
+ private static final String ZIP_FILE_COMMENT_TOTAL_COUNT = "total";
+
+ private final static String ENV_USER_NAME = "user.name";
private final ImportService importService;
private final String fileToImport;
@@ -52,7 +63,8 @@ public class ZipFileMigrationImporter implements Runnable {
FileWatcher fileWatcher = new FileWatcher(fileToImport);
fileWatcher.start();
- performImport(new FileInputStream(new File(fileToImport)));
+ int streamSize = getStreamSizeFromComment(fileToImport);
+ performImport(new FileInputStream(new File(fileToImport)), streamSize);
} catch (IOException e) {
LOG.error("Migration Import: IO Error!", e);
} catch (AtlasBaseException e) {
@@ -60,19 +72,46 @@ public class ZipFileMigrationImporter implements Runnable {
}
}
- private void performImport(InputStream fs) throws AtlasBaseException {
+ private int getStreamSizeFromComment(String fileToImport) {
+ int ret = 1;
+ try {
+ ZipFile zipFile = new ZipFile(fileToImport);
+ String comment = zipFile.getComment();
+ ret = processZipFileStreamSizeComment(comment);
+ zipFile.close();
+ } catch (IOException e) {
+ LOG.error("Error opening ZIP file: {}", fileToImport, e);
+ }
+
+ return ret;
+ }
+
+ private int processZipFileStreamSizeComment(String comment) {
+ if (StringUtils.isEmpty(comment)) {
+ return 1;
+ }
+
+ Map map = AtlasType.fromJson(comment, Map.class);
+ int entitiesCount = (int) map.get(ZIP_FILE_COMMENT_ENTITIES_COUNT);
+ int totalCount = (int) map.get(ZIP_FILE_COMMENT_TOTAL_COUNT);
+ LOG.info("ZipFileMigrationImporter: Zip file: Comment: streamSize: {}: total: {}", entitiesCount, totalCount);
+
+ return entitiesCount;
+ }
+
+ private void performImport(InputStream fs, int streamSize) throws AtlasBaseException {
try {
LOG.info("Migration Import: {}: Starting...", fileToImport);
RequestContext.get().setUser(getUserNameFromEnvironment(), null);
- importService.run(fs, getImportRequest(),
+ importService.run(fs, getImportRequest(streamSize),
getUserNameFromEnvironment(),
InetAddress.getLocalHost().getHostName(),
InetAddress.getLocalHost().getHostAddress());
} catch (Exception ex) {
- LOG.error("Error loading zip for migration", ex);
+ LOG.error("Migration Import: Error loading zip for migration!", ex);
throw new AtlasBaseException(ex);
} finally {
LOG.info("Migration Import: {}: Done!", fileToImport);
@@ -83,9 +122,17 @@ public class ZipFileMigrationImporter implements Runnable {
return System.getProperty(ENV_USER_NAME);
}
- private AtlasImportRequest getImportRequest() throws AtlasException {
+ private AtlasImportRequest getImportRequest(int streamSize) throws AtlasException {
AtlasImportRequest request = new AtlasImportRequest();
- request.setOption(AtlasImportRequest.OPTION_KEY_FORMAT, AtlasImportRequest.OPTION_KEY_FORMAT_ZIP_DIRECT);
+
+ request.setSizeOption(streamSize);
+ request.setOption(AtlasImportRequest.OPTION_KEY_MIGRATION, "true");
+ request.setOption(AtlasImportRequest.OPTION_KEY_NUM_WORKERS, getPropertyValue(APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS, DEFAULT_NUMBER_OF_WORKERS));
+ request.setOption(AtlasImportRequest.OPTION_KEY_BATCH_SIZE, getPropertyValue(APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE, DEFAULT_BATCH_SIZE));
+
return request;
}
+ private String getPropertyValue(String property, String defaultValue) throws AtlasException {
+ return ApplicationProperties.get().getString(property, defaultValue);
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
index b4b76f2..7b9455e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java
@@ -152,6 +152,14 @@ public interface AtlasEntityStore {
EntityMutationResponse createOrUpdateForImport(EntityStream entityStream) throws AtlasBaseException;
/**
+ * Create or update entities with parameters necessary for import process without commit. Caller will have to do take care of commit.
+ * @param entityStream AtlasEntityStream
+ * @return EntityMutationResponse Entity mutations operations with the corresponding set of entities on which these operations were performed
+ * @throws AtlasBaseException
+ */
+ EntityMutationResponse createOrUpdateForImportNoCommit(EntityStream entityStream) throws AtlasBaseException;
+
+ /**
* Update a single entity
* @param objectId ID of the entity
* @param updatedEntityInfo updated entity information
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index 118acfa..379150b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -103,14 +103,14 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
private final DeleteHandlerDelegate deleteDelegate;
private final AtlasTypeRegistry typeRegistry;
- private final AtlasEntityChangeNotifier entityChangeNotifier;
+ private final IAtlasEntityChangeNotifier entityChangeNotifier;
private final EntityGraphMapper entityGraphMapper;
private final EntityGraphRetriever entityRetriever;
@Inject
public AtlasEntityStoreV2(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry,
- AtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper) {
+ IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper) {
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
this.entityChangeNotifier = entityChangeNotifier;
@@ -346,6 +346,11 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
@Override
+ public EntityMutationResponse createOrUpdateForImportNoCommit(EntityStream entityStream) throws AtlasBaseException {
+ return createOrUpdate(entityStream, false, true, true);
+ }
+
+ @Override
@GraphTransaction
public EntityMutationResponse updateEntity(AtlasObjectId objectId, AtlasEntityWithExtInfo updatedEntityInfo, boolean isPartialUpdate) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 6b5615f..7425ac6 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -99,10 +99,10 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
private final EntityGraphRetriever entityRetriever;
private final DeleteHandlerDelegate deleteDelegate;
private final GraphHelper graphHelper = GraphHelper.getInstance();
- private final AtlasEntityChangeNotifier entityChangeNotifier;
+ private final IAtlasEntityChangeNotifier entityChangeNotifier;
@Inject
- public AtlasRelationshipStoreV2(AtlasTypeRegistry typeRegistry, DeleteHandlerDelegate deleteDelegate, AtlasEntityChangeNotifier entityChangeNotifier) {
+ public AtlasRelationshipStoreV2(AtlasTypeRegistry typeRegistry, DeleteHandlerDelegate deleteDelegate, IAtlasEntityChangeNotifier entityChangeNotifier) {
this.typeRegistry = typeRegistry;
this.entityRetriever = new EntityGraphRetriever(typeRegistry);
this.deleteDelegate = deleteDelegate;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
index 54c32c5..a4d732a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
@@ -18,33 +18,29 @@
package org.apache.atlas.repository.store.graph.v2;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.atlas.AtlasConfiguration;
-import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.BulkImporter;
+import org.apache.atlas.repository.store.graph.v2.bulkimport.ImportStrategy;
+import org.apache.atlas.repository.store.graph.v2.bulkimport.MigrationImport;
+import org.apache.atlas.repository.store.graph.v2.bulkimport.RegularImport;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.Constants;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -55,131 +51,27 @@ public class BulkImporterImpl implements BulkImporter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV2.class);
private final AtlasEntityStore entityStore;
- private final EntityGraphRetriever entityGraphRetriever;
private AtlasTypeRegistry typeRegistry;
- private final int MAX_ATTEMPTS = 2;
- private boolean directoryBasedImportConfigured;
@Inject
public BulkImporterImpl(AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
this.entityStore = entityStore;
- this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
this.typeRegistry = typeRegistry;
- this.directoryBasedImportConfigured = StringUtils.isNotEmpty(AtlasConfiguration.IMPORT_TEMP_DIRECTORY.getString());
}
@Override
public EntityMutationResponse bulkImport(EntityImportStream entityStream, AtlasImportResult importResult) throws AtlasBaseException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> bulkImport()");
- }
-
- if (entityStream == null || !entityStream.hasNext()) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "no entities to create/update.");
- }
-
- EntityMutationResponse ret = new EntityMutationResponse();
- ret.setGuidAssignments(new HashMap<>());
-
- Set<String> processedGuids = new HashSet<>();
- float currentPercent = 0f;
- List<String> residualList = new ArrayList<>();
-
- EntityImportStreamWithResidualList entityImportStreamWithResidualList = new EntityImportStreamWithResidualList(entityStream, residualList);
-
- while (entityImportStreamWithResidualList.hasNext()) {
- AtlasEntityWithExtInfo entityWithExtInfo = entityImportStreamWithResidualList.getNextEntityWithExtInfo();
- AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
-
- if (entity == null) {
- continue;
- }
-
- for (int attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
- try {
- AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
- EntityMutationResponse resp = entityStore.createOrUpdateForImport(oneEntityStream);
-
- if (resp.getGuidAssignments() != null) {
- ret.getGuidAssignments().putAll(resp.getGuidAssignments());
- }
+ ImportStrategy importStrategy = null;
- currentPercent = updateImportMetrics(entityWithExtInfo, resp, importResult, processedGuids,
- entityStream.getPosition(),
- entityImportStreamWithResidualList.getStreamSize(),
- currentPercent);
-
- entityStream.onImportComplete(entity.getGuid());
- break;
- } catch (AtlasBaseException e) {
- if (!updateResidualList(e, residualList, entityWithExtInfo.getEntity().getGuid())) {
- throw e;
- }
- break;
- } catch (AtlasSchemaViolationException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Entity: {}", entity.getGuid(), e);
- }
-
- if (attempt == 0) {
- updateVertexGuid(entity);
- } else {
- LOG.error("Guid update failed: {}", entityWithExtInfo.getEntity().getGuid());
- throw e;
- }
- } catch (Throwable e) {
- AtlasBaseException abe = new AtlasBaseException(e);
- if (!updateResidualList(abe, residualList, entityWithExtInfo.getEntity().getGuid())) {
- throw abe;
- }
-
- LOG.warn("Exception: {}", entity.getGuid(), e);
- break;
- } finally {
- RequestContext.get().clearCache();
- }
- }
- }
-
- importResult.getProcessedEntities().addAll(processedGuids);
- LOG.info("bulkImport(): done. Total number of entities (including referred entities) imported: {}", processedGuids.size());
-
- return ret;
- }
-
- @GraphTransaction
- public void updateVertexGuid(AtlasEntity entity) {
- String entityGuid = entity.getGuid();
- AtlasObjectId objectId = entityGraphRetriever.toAtlasObjectIdWithoutGuid(entity);
-
- AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
- String vertexGuid = null;
- try {
- vertexGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(entityType, objectId.getUniqueAttributes());
- } catch (AtlasBaseException e) {
- LOG.warn("Entity: {}: Does not exist!", objectId);
- return;
- }
-
- if (StringUtils.isEmpty(vertexGuid) || vertexGuid.equals(entityGuid)) {
- return;
- }
-
- AtlasVertex v = AtlasGraphUtilsV2.findByGuid(vertexGuid);
- if (v == null) {
- return;
+ if (importResult.getRequest().getOptions() != null &&
+ importResult.getRequest().getOptions().containsKey(AtlasImportRequest.OPTION_KEY_MIGRATION)) {
+ importStrategy = new MigrationImport(new AtlasGraphProvider(), this.typeRegistry);
+ } else {
+ importStrategy = new RegularImport(this.entityStore, this.typeRegistry);
}
- addHistoricalGuid(v, vertexGuid);
- AtlasGraphUtilsV2.setProperty(v, Constants.GUID_PROPERTY_KEY, entityGuid);
-
- LOG.warn("GUID Updated: Entity: {}: from: {}: to: {}", objectId, vertexGuid, entity.getGuid());
- }
-
- private void addHistoricalGuid(AtlasVertex v, String vertexGuid) {
- String existingJson = AtlasGraphUtilsV2.getProperty(v, HISTORICAL_GUID_PROPERTY_KEY, String.class);
-
- AtlasGraphUtilsV2.setProperty(v, HISTORICAL_GUID_PROPERTY_KEY, getJsonArray(existingJson, vertexGuid));
+ LOG.info("BulkImportImpl: {}", importStrategy.getClass().getSimpleName());
+ return importStrategy.run(entityStream, importResult);
}
@VisibleForTesting
@@ -193,38 +85,16 @@ public class BulkImporterImpl implements BulkImporter {
return json;
}
- private boolean updateResidualList(AtlasBaseException e, List<String> lineageList, String guid) {
- if (!e.getAtlasErrorCode().getErrorCode().equals(AtlasErrorCode.INVALID_OBJECT_ID.getErrorCode())) {
- return false;
- }
-
- lineageList.add(guid);
-
- return true;
- }
-
- private float updateImportMetrics(AtlasEntity.AtlasEntityWithExtInfo currentEntity,
- EntityMutationResponse resp,
- AtlasImportResult importResult,
- Set<String> processedGuids,
- int currentIndex, int streamSize, float currentPercent) {
- if (!directoryBasedImportConfigured) {
- updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult);
- updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult);
- updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult);
- }
-
- String lastEntityImported = String.format("entity:last-imported:%s:[%s]:(%s)", currentEntity.getEntity().getTypeName(), currentIndex, currentEntity.getEntity().getGuid());
-
- return updateImportProgress(LOG, currentIndex, streamSize, currentPercent, lastEntityImported);
- }
-
@VisibleForTesting
- static float updateImportProgress(Logger log, int currentIndex, int streamSize, float currentPercent, String additionalInfo) {
+ public static float updateImportProgress(Logger log, int currentIndex, int streamSize, float currentPercent, String additionalInfo) {
final double tolerance = 0.000001;
final int MAX_PERCENT = 100;
int maxSize = (currentIndex <= streamSize) ? streamSize : currentIndex;
+ if (maxSize <= 0) {
+ return currentPercent;
+ }
+
float percent = (float) ((currentIndex * MAX_PERCENT) / maxSize);
boolean updateLog = Double.compare(percent, currentPercent) > tolerance;
float updatedPercent = (MAX_PERCENT < maxSize) ? percent : ((updateLog) ? ++currentPercent : currentPercent);
@@ -236,7 +106,7 @@ public class BulkImporterImpl implements BulkImporter {
return updatedPercent;
}
- private static void updateImportMetrics(String prefix, List<AtlasEntityHeader> list, Set<String> processedGuids, AtlasImportResult importResult) {
+ public static void updateImportMetrics(String prefix, List<AtlasEntityHeader> list, Set<String> processedGuids, AtlasImportResult importResult) {
if (list == null) {
return;
}
@@ -251,41 +121,37 @@ public class BulkImporterImpl implements BulkImporter {
}
}
- private static class EntityImportStreamWithResidualList {
- private final EntityImportStream stream;
- private final List<String> residualList;
- private boolean navigateResidualList;
- private int currentResidualListIndex;
-
+ public static void updateVertexGuid(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityGraphRetriever, AtlasEntity entity) {
+ String entityGuid = entity.getGuid();
+ AtlasObjectId objectId = entityGraphRetriever.toAtlasObjectIdWithoutGuid(entity);
- public EntityImportStreamWithResidualList(EntityImportStream stream, List<String> residualList) {
- this.stream = stream;
- this.residualList = residualList;
- this.navigateResidualList = false;
- this.currentResidualListIndex = 0;
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+ String vertexGuid = null;
+ try {
+ vertexGuid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(entityType, objectId.getUniqueAttributes());
+ } catch (AtlasBaseException e) {
+ LOG.warn("Entity: {}: Does not exist!", objectId);
+ return;
}
- public AtlasEntity.AtlasEntityWithExtInfo getNextEntityWithExtInfo() {
- if (navigateResidualList == false) {
- return stream.getNextEntityWithExtInfo();
- } else {
- stream.setPositionUsingEntityGuid(residualList.get(currentResidualListIndex++));
- return stream.getNextEntityWithExtInfo();
- }
+ if (StringUtils.isEmpty(vertexGuid) || vertexGuid.equals(entityGuid)) {
+ return;
}
- public boolean hasNext() {
- if (!navigateResidualList) {
- boolean streamHasNext = stream.hasNext();
- navigateResidualList = (streamHasNext == false);
- return streamHasNext ? streamHasNext : (currentResidualListIndex < residualList.size());
- } else {
- return (currentResidualListIndex < residualList.size());
- }
+ AtlasVertex v = AtlasGraphUtilsV2.findByGuid(vertexGuid);
+ if (v == null) {
+ return;
}
- public int getStreamSize() {
- return stream.size() + residualList.size();
- }
+ addHistoricalGuid(v, vertexGuid);
+ AtlasGraphUtilsV2.setProperty(v, Constants.GUID_PROPERTY_KEY, entityGuid);
+
+ LOG.warn("GUID Updated: Entity: {}: from: {}: to: {}", objectId, vertexGuid, entity.getGuid());
+ }
+
+ public static void addHistoricalGuid(AtlasVertex v, String vertexGuid) {
+ String existingJson = AtlasGraphUtilsV2.getProperty(v, HISTORICAL_GUID_PROPERTY_KEY, String.class);
+
+ AtlasGraphUtilsV2.setProperty(v, HISTORICAL_GUID_PROPERTY_KEY, getJsonArray(existingJson, vertexGuid));
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 779de2a..d949ed0 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -39,7 +39,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinali
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.converters.AtlasInstanceConverter;
-import org.apache.atlas.repository.graph.FullTextMapperV2;
+import org.apache.atlas.repository.graph.IFullTextMapper;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
@@ -133,15 +133,15 @@ public class EntityGraphMapper {
private final DeleteHandlerDelegate deleteDelegate;
private final AtlasTypeRegistry typeRegistry;
private final AtlasRelationshipStore relationshipStore;
- private final AtlasEntityChangeNotifier entityChangeNotifier;
+ private final IAtlasEntityChangeNotifier entityChangeNotifier;
private final AtlasInstanceConverter instanceConverter;
private final EntityGraphRetriever entityRetriever;
- private final FullTextMapperV2 fullTextMapperV2;
+ private final IFullTextMapper fullTextMapperV2;
@Inject
public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph,
- AtlasRelationshipStore relationshipStore, AtlasEntityChangeNotifier entityChangeNotifier,
- AtlasInstanceConverter instanceConverter, FullTextMapperV2 fullTextMapperV2) {
+ AtlasRelationshipStore relationshipStore, IAtlasEntityChangeNotifier entityChangeNotifier,
+ AtlasInstanceConverter instanceConverter, IFullTextMapper fullTextMapperV2) {
this.deleteDelegate = deleteDelegate;
this.typeRegistry = typeRegistry;
this.graph = atlasGraph;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/FullTextMapperV2Nop.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/FullTextMapperV2Nop.java
new file mode 100644
index 0000000..7b7e543
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/FullTextMapperV2Nop.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v2.bulkimport;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.graph.IFullTextMapper;
+
+import java.util.List;
+
+public class FullTextMapperV2Nop implements IFullTextMapper {
+ @Override
+ public String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
+ return null;
+ }
+
+ @Override
+ public String getIndexTextForEntity(String guid) throws AtlasBaseException {
+ return null;
+ }
+
+ @Override
+ public String getClassificationTextForEntity(AtlasEntity entity) throws AtlasBaseException {
+ return null;
+ }
+
+ @Override
+ public AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
+ return null;
+ }
+
+ @Override
+ public AtlasEntity getAndCacheEntity(String guid, boolean includeReferences) throws AtlasBaseException {
+ return null;
+ }
+
+ @Override
+ public AtlasEntity.AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException {
+ return null;
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/ImportStrategy.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/ImportStrategy.java
new file mode 100644
index 0000000..6b70eab
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/ImportStrategy.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.store.graph.v2.bulkimport;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
+
+public abstract class ImportStrategy {
+ public abstract EntityMutationResponse run(EntityImportStream entityStream, AtlasImportResult importResult) throws AtlasBaseException;
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
new file mode 100644
index 0000000..4c912fd
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.store.graph.v2.bulkimport;
+
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.converters.AtlasFormatConverters;
+import org.apache.atlas.repository.converters.AtlasInstanceConverter;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.AtlasRelationshipStore;
+import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
+import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
+import org.apache.atlas.repository.store.graph.v2.IAtlasEntityChangeNotifier;
+import org.apache.atlas.repository.store.graph.v2.bulkimport.pc.EntityConsumerBuilder;
+import org.apache.atlas.repository.store.graph.v2.bulkimport.pc.EntityCreationManager;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MigrationImport extends ImportStrategy {
+ private static final Logger LOG = LoggerFactory.getLogger(MigrationImport.class);
+
+ private final AtlasTypeRegistry typeRegistry;
+ private AtlasGraph atlasGraph;
+ private EntityGraphRetriever entityGraphRetriever;
+ private EntityGraphMapper entityGraphMapper;
+ private AtlasEntityStore entityStore;
+
+ public MigrationImport(AtlasGraphProvider atlasGraphProvider, AtlasTypeRegistry typeRegistry) {
+ this.typeRegistry = typeRegistry;
+ setupEntityStore(atlasGraphProvider, typeRegistry);
+ LOG.info("MigrationImport: Using bulkLoading...");
+ }
+
+ public EntityMutationResponse run(EntityImportStream entityStream, AtlasImportResult importResult) throws AtlasBaseException {
+ if (entityStream == null || !entityStream.hasNext()) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "no entities to create/update.");
+ }
+
+ if (importResult.getRequest() == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "importResult should contain request");
+ }
+
+ int index = 0;
+ int streamSize = entityStream.size();
+ EntityMutationResponse ret = new EntityMutationResponse();
+ EntityCreationManager creationManager = createEntityCreationManager(atlasGraph, importResult);
+
+ try {
+ LOG.info("Migration Import: Size: {}: Starting...", streamSize);
+ index = creationManager.read(entityStream);
+ creationManager.drain();
+ creationManager.extractResults();
+ } catch (Exception ex) {
+ LOG.error("Migration Import: Error: Current position: {}", index, ex);
+ } finally {
+ shutdownEntityCreationManager(creationManager);
+ }
+
+ LOG.info("Migration Import: Size: {}: Done!", streamSize);
+ return ret;
+ }
+
+ private EntityCreationManager createEntityCreationManager(AtlasGraph threadedAtlasGraph, AtlasImportResult importResult) {
+ int batchSize = importResult.getRequest().getOptionKeyBatchSize();
+ int numWorkers = getNumWorkers(importResult.getRequest().getOptionKeyNumWorkers());
+
+ EntityConsumerBuilder consumerBuilder =
+ new EntityConsumerBuilder(threadedAtlasGraph, entityStore, entityGraphRetriever, typeRegistry, batchSize);
+
+ return new EntityCreationManager(consumerBuilder, batchSize, numWorkers, importResult);
+ }
+
+ private static int getNumWorkers(int numWorkersFromOptions) {
+ int ret = (numWorkersFromOptions > 0) ? numWorkersFromOptions : 1;
+ LOG.info("Migration Import: Setting numWorkers: {}", ret);
+ return ret;
+ }
+
+ private void setupEntityStore(AtlasGraphProvider atlasGraphProvider, AtlasTypeRegistry typeRegistry) {
+ this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
+ this.atlasGraph = atlasGraphProvider.getBulkLoading();
+ DeleteHandlerDelegate deleteDelegate = new DeleteHandlerDelegate(typeRegistry);
+
+ IAtlasEntityChangeNotifier entityChangeNotifier = new EntityChangeNotifierNop();
+ AtlasRelationshipStore relationshipStore = new AtlasRelationshipStoreV2(typeRegistry, deleteDelegate, entityChangeNotifier);
+ AtlasFormatConverters formatConverters = new AtlasFormatConverters(typeRegistry);
+ AtlasInstanceConverter instanceConverter = new AtlasInstanceConverter(typeRegistry, formatConverters);
+ this.entityGraphMapper = new EntityGraphMapper(deleteDelegate, typeRegistry, atlasGraph, relationshipStore, entityChangeNotifier, instanceConverter, new FullTextMapperV2Nop());
+ this.entityStore = new AtlasEntityStoreV2(deleteDelegate, typeRegistry, entityChangeNotifier, entityGraphMapper);
+ }
+
+ private void shutdownEntityCreationManager(EntityCreationManager creationManager) {
+ try {
+ creationManager.shutdown();
+ } catch (InterruptedException e) {
+ LOG.error("Migration Import: Shutdown: Interrupted!", e);
+ }
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java
similarity index 80%
copy from repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
copy to repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java
index 54c32c5..ecce1b0 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BulkImporterImpl.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/RegularImport.java
@@ -6,16 +6,18 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.atlas.repository.store.graph.v2;
+
+package org.apache.atlas.repository.store.graph.v2.bulkimport;
+
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasConfiguration;
@@ -26,22 +28,23 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
-import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
-import org.apache.atlas.repository.store.graph.BulkImporter;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStreamForImport;
+import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
+import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -49,27 +52,25 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.repository.Constants.HISTORICAL_GUID_PROPERTY_KEY;
+import static org.apache.atlas.repository.store.graph.v2.BulkImporterImpl.updateImportProgress;
-@Component
-public class BulkImporterImpl implements BulkImporter {
- private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityStoreV2.class);
-
+public class RegularImport extends ImportStrategy {
+ private static final Logger LOG = LoggerFactory.getLogger(RegularImport.class);
+ private static final int MAX_ATTEMPTS = 3;
private final AtlasEntityStore entityStore;
+ private final AtlasTypeRegistry typeRegistry;
private final EntityGraphRetriever entityGraphRetriever;
- private AtlasTypeRegistry typeRegistry;
- private final int MAX_ATTEMPTS = 2;
private boolean directoryBasedImportConfigured;
- @Inject
- public BulkImporterImpl(AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
+ public RegularImport(AtlasEntityStore entityStore, AtlasTypeRegistry typeRegistry) {
this.entityStore = entityStore;
- this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
this.typeRegistry = typeRegistry;
+ this.entityGraphRetriever = new EntityGraphRetriever(typeRegistry);
this.directoryBasedImportConfigured = StringUtils.isNotEmpty(AtlasConfiguration.IMPORT_TEMP_DIRECTORY.getString());
}
@Override
- public EntityMutationResponse bulkImport(EntityImportStream entityStream, AtlasImportResult importResult) throws AtlasBaseException {
+ public EntityMutationResponse run(EntityImportStream entityStream, AtlasImportResult importResult) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> bulkImport()");
}
@@ -81,7 +82,7 @@ public class BulkImporterImpl implements BulkImporter {
EntityMutationResponse ret = new EntityMutationResponse();
ret.setGuidAssignments(new HashMap<>());
- Set<String> processedGuids = new HashSet<>();
+ Set<String> processedGuids = new HashSet<>();
float currentPercent = 0f;
List<String> residualList = new ArrayList<>();
@@ -209,9 +210,9 @@ public class BulkImporterImpl implements BulkImporter {
Set<String> processedGuids,
int currentIndex, int streamSize, float currentPercent) {
if (!directoryBasedImportConfigured) {
- updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult);
- updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult);
- updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult);
+ BulkImporterImpl.updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult);
+ BulkImporterImpl.updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult);
+ BulkImporterImpl.updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult);
}
String lastEntityImported = String.format("entity:last-imported:%s:[%s]:(%s)", currentEntity.getEntity().getTypeName(), currentIndex, currentEntity.getEntity().getGuid());
@@ -219,38 +220,6 @@ public class BulkImporterImpl implements BulkImporter {
return updateImportProgress(LOG, currentIndex, streamSize, currentPercent, lastEntityImported);
}
- @VisibleForTesting
- static float updateImportProgress(Logger log, int currentIndex, int streamSize, float currentPercent, String additionalInfo) {
- final double tolerance = 0.000001;
- final int MAX_PERCENT = 100;
-
- int maxSize = (currentIndex <= streamSize) ? streamSize : currentIndex;
- float percent = (float) ((currentIndex * MAX_PERCENT) / maxSize);
- boolean updateLog = Double.compare(percent, currentPercent) > tolerance;
- float updatedPercent = (MAX_PERCENT < maxSize) ? percent : ((updateLog) ? ++currentPercent : currentPercent);
-
- if (updateLog) {
- log.info("bulkImport(): progress: {}% (of {}) - {}", (int) Math.ceil(percent), maxSize, additionalInfo);
- }
-
- return updatedPercent;
- }
-
- private static void updateImportMetrics(String prefix, List<AtlasEntityHeader> list, Set<String> processedGuids, AtlasImportResult importResult) {
- if (list == null) {
- return;
- }
-
- for (AtlasEntityHeader h : list) {
- if (processedGuids.contains(h.getGuid())) {
- continue;
- }
-
- processedGuids.add(h.getGuid());
- importResult.incrementMeticsCounter(String.format(prefix, h.getTypeName()));
- }
- }
-
private static class EntityImportStreamWithResidualList {
private final EntityImportStream stream;
private final List<String> residualList;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
new file mode 100644
index 0000000..e8f4b02
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumer.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v2.bulkimport.pc;
+
+import org.apache.atlas.GraphTransactionInterceptor;
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.pc.WorkItemConsumer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.v2.AtlasEntityStreamForImport;
+import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class EntityConsumer extends WorkItemConsumer<AtlasEntity.AtlasEntityWithExtInfo> {
+ private static final Logger LOG = LoggerFactory.getLogger(EntityConsumer.class);
+ private static final int MAX_COMMIT_RETRY_COUNT = 3;
+
+ private final int batchSize;
+ private AtomicLong counter = new AtomicLong(1);
+ private AtomicLong currentBatch = new AtomicLong(1);
+
+ private final AtlasGraph atlasGraph;
+ private final AtlasEntityStore entityStoreV2;
+ private final AtlasTypeRegistry typeRegistry;
+ private final EntityGraphRetriever entityGraphRetriever;
+
+ private List<AtlasEntity.AtlasEntityWithExtInfo> entityBuffer = new ArrayList<>();
+ private List<EntityMutationResponse> localResults = new ArrayList<>();
+
+ public EntityConsumer(AtlasGraph atlasGraph, AtlasEntityStore entityStore,
+ EntityGraphRetriever entityGraphRetriever, AtlasTypeRegistry typeRegistry,
+ BlockingQueue queue, int batchSize) {
+ super(queue);
+
+ this.atlasGraph = atlasGraph;
+ this.entityStoreV2 = entityStore;
+ this.entityGraphRetriever = entityGraphRetriever;
+ this.typeRegistry = typeRegistry;
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ protected void processItem(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ int delta = (MapUtils.isEmpty(entityWithExtInfo.getReferredEntities())
+ ? 1
+ : entityWithExtInfo.getReferredEntities().size()) + 1;
+
+ long currentCount = counter.addAndGet(delta);
+ currentBatch.addAndGet(delta);
+ entityBuffer.add(entityWithExtInfo);
+
+ try {
+ processEntity(entityWithExtInfo, currentCount);
+ attemptCommit();
+ } catch (Exception e) {
+ LOG.info("Data loss: Please re-submit!", e);
+ }
+ }
+
+ private void processEntity(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, long currentCount) {
+ try {
+ RequestContext.get().setImportInProgress(true);
+ AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, null);
+
+ LOG.debug("Processing: {}", currentCount);
+ EntityMutationResponse result = entityStoreV2.createOrUpdateForImportNoCommit(oneEntityStream);
+ localResults.add(result);
+ } catch (AtlasBaseException e) {
+ addResult(entityWithExtInfo.getEntity().getGuid());
+ LOG.warn("Exception: {}", entityWithExtInfo.getEntity().getGuid(), e);
+ } catch (AtlasSchemaViolationException e) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Entity: {}", entityWithExtInfo.getEntity().getGuid(), e);
+ }
+
+ BulkImporterImpl.updateVertexGuid(typeRegistry, entityGraphRetriever, entityWithExtInfo.getEntity());
+ }
+ }
+
+ private void attemptCommit() {
+ if (currentBatch.get() < batchSize) {
+ return;
+ }
+
+ doCommit();
+ }
+
+ @Override
+ protected void doCommit() {
+ for (int retryCount = 1; retryCount <= MAX_COMMIT_RETRY_COUNT; retryCount++) {
+ if (commitWithRetry(retryCount)) {
+ return;
+ }
+ }
+
+ LOG.error("Retries exceeded! Potential data loss! Please correct data and re-attempt. Buffer: {}: Counter: {}", entityBuffer.size(), counter.get());
+ clear();
+ }
+
+ @Override
+ protected void commitDirty() {
+ super.commitDirty();
+ LOG.info("Total: Commit: {}", counter.get());
+ counter.set(0);
+ }
+
+ private boolean commitWithRetry(int retryCount) {
+ try {
+ atlasGraph.commit();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Commit: Done!: Buffer: {}: Batch: {}: Counter: {}", entityBuffer.size(), currentBatch.get(), counter.get());
+ }
+
+ dispatchResults();
+ return true;
+ } catch (Exception ex) {
+ rollbackPauseRetry(retryCount, ex);
+ return false;
+ }
+ }
+
+ private void rollbackPauseRetry(int retryCount, Exception ex) {
+ atlasGraph.rollback();
+ clearCache();
+
+ LOG.error("Rollback: Done! Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount);
+ pause(retryCount);
+ LOG.warn("Commit error! Will pause and retry: Buffer: {}: Counter: {}: Retry count: {}", entityBuffer.size(), counter.get(), retryCount, ex);
+ retryProcessEntity(retryCount);
+ }
+
+ private void retryProcessEntity(int retryCount) {
+ LOG.info("Replaying: Starting!: Buffer: {}: Retry count: {}", entityBuffer.size(), retryCount);
+ for (AtlasEntity.AtlasEntityWithExtInfo e : entityBuffer) {
+ processEntity(e, counter.get());
+ }
+ LOG.info("Replaying: Done!: Buffer: {}: Retry count: {}", entityBuffer.size(), retryCount);
+ }
+
+ private void dispatchResults() {
+ localResults.stream().forEach(x -> {
+ addResultsFromResponse(x.getCreatedEntities());
+ addResultsFromResponse(x.getUpdatedEntities());
+ addResultsFromResponse(x.getDeletedEntities());
+ });
+
+ clear();
+ }
+
+ private void pause(int retryCount) {
+ try {
+ Thread.sleep(1000 * retryCount);
+ } catch (InterruptedException e) {
+ LOG.error("pause: Interrupted!", e);
+ }
+ }
+
+ private void addResultsFromResponse(List<AtlasEntityHeader> entities) {
+ if (CollectionUtils.isEmpty(entities)) {
+ return;
+ }
+
+ for (AtlasEntityHeader eh : entities) {
+ addResult(eh.getGuid());
+ }
+ }
+
+ private void clear() {
+ localResults.clear();
+ entityBuffer.clear();
+ clearCache();
+ currentBatch.set(0);
+ }
+
+ private void clearCache() {
+ GraphTransactionInterceptor.clearCache();
+ RequestContext.get().clearCache();
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java
new file mode 100644
index 0000000..69d33b2
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityConsumerBuilder.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.store.graph.v2.bulkimport.pc;
+
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.pc.WorkItemBuilder;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasTypeRegistry;
+
+import java.util.concurrent.BlockingQueue;
+
+public class EntityConsumerBuilder implements WorkItemBuilder<EntityConsumer, AtlasEntity.AtlasEntityWithExtInfo> {
+ private AtlasGraph atlasGraph;
+ private AtlasEntityStore entityStore;
+ private final EntityGraphRetriever entityGraphRetriever;
+ private final AtlasTypeRegistry typeRegistry;
+ private int batchSize;
+
+ public EntityConsumerBuilder(AtlasGraph atlasGraph, AtlasEntityStore entityStore,
+ EntityGraphRetriever entityGraphRetriever, AtlasTypeRegistry typeRegistry, int batchSize) {
+ this.atlasGraph = atlasGraph;
+ this.entityStore = entityStore;
+ this.entityGraphRetriever = entityGraphRetriever;
+ this.typeRegistry = typeRegistry;
+ this.batchSize = batchSize;
+ }
+
+ @Override
+ public EntityConsumer build(BlockingQueue<AtlasEntity.AtlasEntityWithExtInfo> queue) {
+ return new EntityConsumer(atlasGraph, entityStore, entityGraphRetriever, typeRegistry, queue, this.batchSize);
+ }
+}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
new file mode 100644
index 0000000..16bb49e
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/pc/EntityCreationManager.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.store.graph.v2.bulkimport.pc;
+
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.pc.StatusReporter;
+import org.apache.atlas.pc.WorkItemBuilder;
+import org.apache.atlas.pc.WorkItemManager;
+import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
+import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EntityCreationManager<AtlasEntityWithExtInfo> extends WorkItemManager {
+ private static final Logger LOG = LoggerFactory.getLogger(EntityCreationManager.class);
+ private static final String WORKER_PREFIX = "migration-import";
+
+ private final StatusReporter<String, String> statusReporter;
+ private final AtlasImportResult importResult;
+ private String currentTypeName;
+ private float currentPercent;
+ private EntityImportStream entityImportStream;
+
+ public EntityCreationManager(WorkItemBuilder builder, int batchSize, int numWorkers, AtlasImportResult importResult) {
+ super(builder, WORKER_PREFIX, batchSize, numWorkers, true);
+ this.importResult = importResult;
+
+ this.statusReporter = new StatusReporter<>();
+ }
+
+ public int read(EntityImportStream entityStream) {
+ int currentIndex = 0;
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo;
+ this.entityImportStream = entityStream;
+ while ((entityWithExtInfo = entityStream.getNextEntityWithExtInfo()) != null) {
+ AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
+ if (entity == null) {
+ continue;
+ }
+
+ try {
+ produce(currentIndex++, entity.getTypeName(), entityWithExtInfo);
+ } catch (Throwable e) {
+ LOG.warn("Exception: {}", entity.getGuid(), e);
+ break;
+ }
+ }
+ return currentIndex;
+ }
+
+ private void produce(int currentIndex, String typeName, AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) {
+ String previousTypeName = getCurrentTypeName();
+
+ if (StringUtils.isNotEmpty(typeName)
+ && StringUtils.isNotEmpty(previousTypeName)
+ && !StringUtils.equals(previousTypeName, typeName)) {
+ LOG.info("Waiting: '{}' to complete...", previousTypeName);
+ super.drain();
+ LOG.info("Switching entity type processing: From: '{}' To: '{}'...", previousTypeName, typeName);
+ }
+
+ setCurrentTypeName(typeName);
+ statusReporter.produced(entityWithExtInfo.getEntity().getGuid(), String.format("%s:%s", entityWithExtInfo.getEntity().getTypeName(), currentIndex));
+ super.checkProduce(entityWithExtInfo);
+ extractResults();
+ }
+
+ public void extractResults() {
+ Object result;
+ while (((result = getResults().poll())) != null) {
+ statusReporter.processed((String) result);
+ }
+
+ logStatus();
+ }
+
+ private void logStatus() {
+ String ack = statusReporter.ack();
+ if (StringUtils.isEmpty(ack)) {
+ return;
+ }
+
+ String[] split = ack.split(":");
+ if (split.length == 0 || split.length < 2) {
+ return;
+ }
+
+ importResult.incrementMeticsCounter(split[0]);
+ this.currentPercent = updateImportMetrics(split[0], Integer.parseInt(split[1]), this.entityImportStream.size(), getCurrentPercent());
+ }
+
+ private static float updateImportMetrics(String typeNameGuid, int currentIndex, int streamSize, float currentPercent) {
+ String lastEntityImported = String.format("entity:last-imported:%s:(%s)", typeNameGuid, currentIndex);
+ return BulkImporterImpl.updateImportProgress(LOG, currentIndex, streamSize, currentPercent, lastEntityImported);
+ }
+
+ private String getCurrentTypeName() {
+ return this.currentTypeName;
+ }
+
+ private void setCurrentTypeName(String typeName) {
+ this.currentTypeName = typeName;
+ }
+
+ private float getCurrentPercent() {
+ return this.currentPercent;
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/TestModules.java b/repository/src/test/java/org/apache/atlas/TestModules.java
index 06e0ebc..a298934 100644
--- a/repository/src/test/java/org/apache/atlas/TestModules.java
+++ b/repository/src/test/java/org/apache/atlas/TestModules.java
@@ -36,7 +36,9 @@ import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.repository.audit.EntityAuditListener;
import org.apache.atlas.repository.audit.EntityAuditListenerV2;
import org.apache.atlas.repository.audit.EntityAuditRepository;
+import org.apache.atlas.repository.graph.FullTextMapperV2;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graph.IFullTextMapper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDBMigrator;
import org.apache.atlas.repository.graphdb.janus.migration.GraphDBGraphSONMigrator;
@@ -61,6 +63,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasRelationshipStoreV2;
import org.apache.atlas.repository.store.graph.v2.AtlasTypeDefGraphStoreV2;
import org.apache.atlas.repository.store.graph.v2.BulkImporterImpl;
import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper;
+import org.apache.atlas.repository.store.graph.v2.IAtlasEntityChangeNotifier;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.service.Service;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -144,6 +147,8 @@ public class TestModules {
bind(AtlasEntityStore.class).to(AtlasEntityStoreV2.class);
bind(AtlasRelationshipStore.class).to(AtlasRelationshipStoreV2.class);
+ bind(IAtlasEntityChangeNotifier.class).to(AtlasEntityChangeNotifier.class);
+ bind(IFullTextMapper.class).to(FullTextMapperV2.class);
// bind the DiscoveryService interface to an implementation
bind(AtlasDiscoveryService.class).to(EntityDiscoveryService.class).asEagerSingleton();