You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2025/02/03 16:05:32 UTC
(atlas) branch atlas-2.5 updated: ATLAS-4965: checkstyle compliance updates - atlas-repository module (#277)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch atlas-2.5
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/atlas-2.5 by this push:
new aabb14241 ATLAS-4965: checkstyle compliance updates - atlas-repository module (#277)
aabb14241 is described below
commit aabb142418d07e86b19efb0447c599412f7880c0
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Mon Feb 3 08:03:59 2025 -0800
ATLAS-4965: checkstyle compliance updates - atlas-repository module (#277)
(cherry picked from commit 5be0223d4ce81e4c2e69a1c596e471dc923ef40c)
---
dev-support/checkstyle-suppressions.xml | 1 +
repository/pom.xml | 5 +
.../org/apache/atlas/GraphTransactionAdvisor.java | 3 +
.../apache/atlas/GraphTransactionInterceptor.java | 368 +-
.../atlas/discovery/AtlasDiscoveryService.java | 35 +-
.../atlas/discovery/AtlasLineageService.java | 1 -
.../discovery/ClassificationSearchProcessor.java | 174 +-
.../atlas/discovery/EntityDiscoveryService.java | 1108 ++--
.../atlas/discovery/EntityLineageService.java | 198 +-
.../atlas/discovery/EntitySearchProcessor.java | 125 +-
.../atlas/discovery/FreeTextSearchProcessor.java | 65 +-
.../atlas/discovery/FullTextSearchProcessor.java | 43 +-
.../atlas/discovery/GraphIndexQueryBuilder.java | 41 +-
.../discovery/RelationshipSearchProcessor.java | 80 +-
.../apache/atlas/discovery/SearchAggregator.java | 11 +-
.../atlas/discovery/SearchAggregatorImpl.java | 72 +-
.../org/apache/atlas/discovery/SearchContext.java | 284 +-
.../org/apache/atlas/discovery/SearchIndexer.java | 7 +-
.../apache/atlas/discovery/SearchProcessor.java | 1090 ++--
.../atlas/discovery/SuggestionsProviderImpl.java | 9 +-
.../atlas/discovery/TermSearchProcessor.java | 48 +-
.../org/apache/atlas/entitytransform/Action.java | 45 +-
.../entitytransform/AtlasEntityTransformer.java | 2 -
.../atlas/entitytransform/BaseEntityHandler.java | 135 +-
.../apache/atlas/entitytransform/Condition.java | 39 +-
.../entitytransform/HdfsPathEntityHandler.java | 22 +-
.../entitytransform/HiveColumnEntityHandler.java | 32 +-
.../entitytransform/HiveDatabaseEntityHandler.java | 19 +-
.../HiveStorageDescriptorEntityHandler.java | 28 +-
.../entitytransform/HiveTableEntityHandler.java | 44 +-
.../entitytransform/TransformationConstants.java | 18 +-
.../atlas/entitytransform/TransformerContext.java | 6 +-
.../atlas/glossary/GlossaryCategoryUtils.java | 270 +-
.../org/apache/atlas/glossary/GlossaryService.java | 678 ++-
.../apache/atlas/glossary/GlossaryTermUtils.java | 843 ++-
.../org/apache/atlas/glossary/GlossaryUtils.java | 44 +-
.../org/apache/atlas/listener/ChangedTypeDefs.java | 40 +-
.../atlas/listener/EntityChangeListenerV2.java | 18 +-
.../atlas/listener/TypeDefChangeListener.java | 5 +-
.../main/java/org/apache/atlas/query/AtlasDSL.java | 20 +-
.../java/org/apache/atlas/query/DSLVisitor.java | 42 +-
.../java/org/apache/atlas/query/GremlinClause.java | 6 +-
.../org/apache/atlas/query/GremlinClauseList.java | 27 +-
.../java/org/apache/atlas/query/GremlinQuery.java | 6 +-
.../apache/atlas/query/GremlinQueryComposer.java | 297 +-
.../org/apache/atlas/query/IdentifierHelper.java | 236 +-
.../main/java/org/apache/atlas/query/Lookup.java | 8 +-
.../java/org/apache/atlas/query/QueryParams.java | 42 +-
.../apache/atlas/query/RegistryBasedLookup.java | 188 +-
.../apache/atlas/query/SelectClauseComposer.java | 44 +-
.../apache/atlas/query/antlr4/AtlasDSLLexer.java | 665 +--
.../apache/atlas/query/antlr4/AtlasDSLParser.java | 5915 +++++++++++---------
.../query/antlr4/AtlasDSLParserBaseVisitor.java | 704 ++-
.../atlas/query/antlr4/AtlasDSLParserListener.java | 938 ++--
.../atlas/query/antlr4/AtlasDSLParserVisitor.java | 548 +-
.../atlas/query/executors/DSLQueryExecutor.java | 6 +-
.../GremlinClauseToTraversalTranslator.java | 68 +-
.../query/executors/ScriptEngineBasedExecutor.java | 50 +-
.../query/executors/SelectClauseProjections.java | 31 +-
.../query/executors/TraversalBasedExecutor.java | 42 +-
.../atlas/repository/IndexCreationException.java | 10 +-
.../apache/atlas/repository/IndexException.java | 9 +-
.../atlas/repository/RepositoryException.java | 7 +-
.../audit/AbstractStorageBasedAuditRepository.java | 233 +-
.../audit/AtlasAuditReductionService.java | 246 +-
.../atlas/repository/audit/AtlasAuditService.java | 67 +-
.../audit/CassandraBasedAuditRepository.java | 412 +-
.../repository/audit/EntityAuditListener.java | 203 +-
.../repository/audit/EntityAuditListenerV2.java | 83 +-
.../repository/audit/EntityAuditRepository.java | 23 +-
.../audit/HBaseBasedAuditRepository.java | 848 +--
.../audit/InMemoryEntityAuditRepository.java | 141 +-
.../audit/NoopEntityAuditRepository.java | 18 +-
.../repository/audit/TypeDefAuditListener.java | 22 +-
.../converters/AtlasAbstractFormatConverter.java | 14 +-
.../converters/AtlasArrayFormatConverter.java | 49 +-
.../AtlasClassificationFormatConverter.java | 18 +-
.../converters/AtlasEntityFormatConverter.java | 43 +-
.../converters/AtlasEnumFormatConverter.java | 41 +-
.../converters/AtlasFormatConverter.java | 19 +-
.../converters/AtlasFormatConverters.java | 27 +-
.../converters/AtlasInstanceConverter.java | 82 +-
.../converters/AtlasMapFormatConverter.java | 44 +-
.../converters/AtlasObjectIdConverter.java | 14 +-
.../converters/AtlasPrimitiveFormatConverter.java | 11 +-
.../converters/AtlasStructFormatConverter.java | 92 +-
.../repository/converters/TypeConverterUtil.java | 279 +-
.../atlas/repository/graph/AtlasEdgeLabel.java | 94 +-
.../atlas/repository/graph/AtlasGraphProvider.java | 96 +-
.../atlas/repository/graph/FullTextMapperV2.java | 134 +-
.../repository/graph/GraphBackedSearchIndexer.java | 645 ++-
.../apache/atlas/repository/graph/GraphHelper.java | 1555 ++---
.../repository/graph/IAtlasGraphProvider.java | 7 +-
.../atlas/repository/graph/IFullTextMapper.java | 8 +-
.../repository/graph/IndexChangeListener.java | 8 +-
.../repository/graph/IndexRecoveryService.java | 128 +-
.../atlas/repository/graph/SolrIndexHelper.java | 40 +-
.../repository/impexp/AtlasServerService.java | 80 +-
.../atlas/repository/impexp/AuditsWriter.java | 218 +-
.../atlas/repository/impexp/EntitiesExtractor.java | 22 +-
.../impexp/ExportImportAuditService.java | 124 +-
.../atlas/repository/impexp/ExportService.java | 281 +-
.../repository/impexp/ExportTypeProcessor.java | 38 +-
.../atlas/repository/impexp/ExtractStrategy.java | 9 +-
.../repository/impexp/HdfsPathEntityCreator.java | 57 +-
.../atlas/repository/impexp/ImportService.java | 164 +-
.../atlas/repository/impexp/ImportTransformer.java | 199 +-
.../atlas/repository/impexp/ImportTransforms.java | 92 +-
.../repository/impexp/ImportTransformsShaper.java | 28 +-
.../repository/impexp/ImportTypeDefProcessor.java | 17 +-
.../impexp/IncrementalExportEntityProvider.java | 84 +-
.../impexp/MigrationProgressService.java | 52 +-
.../impexp/RelationshipAttributesExtractor.java | 71 +-
.../impexp/StartEntityFetchByExportRequest.java | 133 +-
.../impexp/TableReplicationRequestProcessor.java | 75 +-
.../repository/impexp/TypeAttributeDifference.java | 79 +-
.../atlas/repository/impexp/VertexExtractor.java | 82 +-
.../repository/impexp/ZipExportFileNames.java | 5 +-
.../apache/atlas/repository/impexp/ZipSink.java | 38 +-
.../apache/atlas/repository/impexp/ZipSource.java | 290 +-
.../atlas/repository/impexp/ZipSourceDirect.java | 180 +-
.../impexp/ZipSourceWithBackingDirectory.java | 202 +-
.../repository/migration/DataMigrationService.java | 34 +-
.../migration/DataMigrationStatusService.java | 67 +-
.../atlas/repository/migration/FileWatcher.java | 39 +-
.../migration/ZipFileMigrationImporter.java | 120 +-
.../repository/ogm/AbstractDataTransferObject.java | 14 +-
.../atlas/repository/ogm/AtlasAuditEntryDTO.java | 19 +-
.../atlas/repository/ogm/AtlasServerDTO.java | 21 +-
.../apache/atlas/repository/ogm/DTORegistry.java | 19 +-
.../apache/atlas/repository/ogm/DataAccess.java | 59 +-
.../atlas/repository/ogm/DataTransferObject.java | 3 +-
.../repository/ogm/ExportImportAuditEntryDTO.java | 31 +-
.../ogm/glossary/AbstractGlossaryDTO.java | 12 +-
.../ogm/glossary/AtlasGlossaryCategoryDTO.java | 69 +-
.../repository/ogm/glossary/AtlasGlossaryDTO.java | 67 +-
.../ogm/glossary/AtlasGlossaryTermDTO.java | 135 +-
.../ogm/metrics/AtlasMetricsStatDTO.java | 45 +-
.../ogm/profiles/AtlasSavedSearchDTO.java | 8 +-
.../ogm/profiles/AtlasUserProfileDTO.java | 14 +-
.../patches/AddMandatoryAttributesPatch.java | 23 +-
.../repository/patches/AtlasPatchHandler.java | 16 +-
.../repository/patches/AtlasPatchManager.java | 42 +-
.../repository/patches/AtlasPatchRegistry.java | 56 +-
.../repository/patches/AtlasPatchService.java | 6 +-
.../patches/ClassificationTextPatch.java | 15 +-
.../patches/ConcurrentPatchProcessor.java | 134 +-
.../repository/patches/EdgePatchProcessor.java | 110 +-
.../repository/patches/IndexConsistencyPatch.java | 8 +-
.../atlas/repository/patches/PatchContext.java | 21 +-
.../repository/patches/ProcessImpalaNamePatch.java | 28 +-
.../atlas/repository/patches/ProcessNamePatch.java | 43 +-
.../atlas/repository/patches/ReIndexPatch.java | 94 +-
.../patches/RelationshipTypeNamePatch.java | 29 +-
.../patches/SuggestionsRequestHandlerPatch.java | 1 -
.../repository/patches/SuperTypesUpdatePatch.java | 28 +-
.../repository/patches/UniqueAttributePatch.java | 62 +-
.../patches/UpdateCompositeIndexStatusPatch.java | 2 +
.../bootstrap/AtlasTypeDefStoreInitializer.java | 481 +-
.../repository/store/graph/AtlasDefStore.java | 6 +-
.../repository/store/graph/AtlasEntityStore.java | 13 +-
.../store/graph/AtlasRelationshipStore.java | 6 +-
.../store/graph/AtlasTypeDefGraphStore.java | 512 +-
.../atlas/repository/store/graph/BulkImporter.java | 7 +-
.../store/graph/EntityCorrelationStore.java | 9 +-
.../store/graph/EntityGraphDiscovery.java | 9 +-
.../store/graph/EntityGraphDiscoveryContext.java | 103 +-
.../repository/store/graph/EntityResolver.java | 7 +-
.../store/graph/v1/DeleteHandlerDelegate.java | 21 +-
.../repository/store/graph/v1/DeleteHandlerV1.java | 523 +-
.../store/graph/v1/HardDeleteHandlerV1.java | 1 -
.../store/graph/v1/SoftDeleteHandlerV1.java | 5 +-
.../store/graph/v2/AtlasAbstractDefStoreV2.java | 57 +-
.../graph/v2/AtlasBusinessMetadataDefStoreV2.java | 171 +-
.../graph/v2/AtlasClassificationDefStoreV2.java | 134 +-
.../store/graph/v2/AtlasEntityChangeNotifier.java | 140 +-
.../store/graph/v2/AtlasEntityComparator.java | 6 +-
.../store/graph/v2/AtlasEntityDefStoreV2.java | 105 +-
.../graph/v2/AtlasEntityGraphDiscoveryV2.java | 148 +-
.../store/graph/v2/AtlasEntityStoreV2.java | 619 +-
.../store/graph/v2/AtlasEntityStream.java | 5 +-
.../store/graph/v2/AtlasEntityStreamForImport.java | 45 +-
.../store/graph/v2/AtlasEnumDefStoreV2.java | 88 +-
.../store/graph/v2/AtlasGraphUtilsV2.java | 393 +-
.../graph/v2/AtlasRelationshipDefStoreV2.java | 289 +-
.../store/graph/v2/AtlasRelationshipStoreV2.java | 230 +-
.../store/graph/v2/AtlasStructDefStoreV2.java | 655 +--
.../store/graph/v2/AtlasTypeDefGraphStoreV2.java | 241 +-
.../store/graph/v2/AttributeMutationContext.java | 56 +-
.../store/graph/v2/BulkImporterImpl.java | 85 +-
.../store/graph/v2/ClassificationAssociator.java | 108 +-
.../store/graph/v2/EntityGraphMapper.java | 3212 ++++++-----
.../store/graph/v2/EntityGraphRetriever.java | 439 +-
.../store/graph/v2/EntityImportStream.java | 16 +-
.../store/graph/v2/EntityMutationContext.java | 79 +-
.../store/graph/v2/EntityStateChecker.java | 62 +-
.../repository/store/graph/v2/EntityStream.java | 5 +-
.../store/graph/v2/IAtlasEntityChangeNotifier.java | 6 +-
.../store/graph/v2/IDBasedEntityResolver.java | 19 +-
.../store/graph/v2/InMemoryMapEntityStream.java | 8 +-
.../graph/v2/UniqAttrBasedEntityResolver.java | 10 +-
.../v2/bulkimport/EntityChangeNotifierNop.java | 43 +-
.../graph/v2/bulkimport/FullTextMapperV2Nop.java | 19 +-
.../store/graph/v2/bulkimport/ImportStrategy.java | 6 +-
.../store/graph/v2/bulkimport/MigrationImport.java | 60 +-
.../store/graph/v2/bulkimport/RegularImport.java | 95 +-
.../graph/v2/bulkimport/pc/EntityConsumer.java | 141 +-
.../v2/bulkimport/pc/EntityConsumerBuilder.java | 48 +-
.../v2/bulkimport/pc/EntityCreationManager.java | 66 +-
.../tasks/AuditReductionEntityRetrievalTask.java | 67 +-
.../store/graph/v2/tasks/AuditReductionTask.java | 77 +-
.../graph/v2/tasks/AuditReductionTaskFactory.java | 72 +-
.../tasks/ClassificationPropagateTaskFactory.java | 17 +-
.../v2/tasks/ClassificationPropagationTasks.java | 7 +-
.../store/graph/v2/tasks/ClassificationTask.java | 75 +-
.../searchdownload/SearchResultDownloadTask.java | 155 +-
.../SearchResultDownloadTaskFactory.java | 72 +-
.../repository/userprofile/UserProfileService.java | 28 +-
.../apache/atlas/repository/util/FilterUtil.java | 97 +-
.../apache/atlas/repository/util/UniqueList.java | 14 +-
.../org/apache/atlas/services/MetricsService.java | 348 +-
.../org/apache/atlas/store/AtlasTypeDefStore.java | 8 +-
.../java/org/apache/atlas/tasks/AbstractTask.java | 14 +-
.../java/org/apache/atlas/tasks/TaskExecutor.java | 24 +-
.../java/org/apache/atlas/tasks/TaskFactory.java | 9 +-
.../apache/atlas/tasks/TaskFactoryRegistry.java | 10 +-
.../org/apache/atlas/tasks/TaskManagement.java | 75 +-
.../java/org/apache/atlas/tasks/TaskRegistry.java | 48 +-
.../atlas/util/AtlasGremlin2QueryProvider.java | 3 +-
.../atlas/util/AtlasGremlinQueryProvider.java | 24 +-
.../org/apache/atlas/util/AtlasMetricJVMUtil.java | 38 +-
.../org/apache/atlas/util/AtlasMetricsCounter.java | 138 +-
.../org/apache/atlas/util/AtlasMetricsUtil.java | 241 +-
.../atlas/util/AtlasRepositoryConfiguration.java | 87 +-
.../org/apache/atlas/util/AttributeValueMap.java | 34 +-
.../main/java/org/apache/atlas/util/BeanUtil.java | 16 +-
.../main/java/org/apache/atlas/util/FileUtils.java | 56 +-
.../org/apache/atlas/util/IndexedInstance.java | 39 +-
.../org/apache/atlas/util/SearchPredicateUtil.java | 1150 ++--
.../java/org/apache/atlas/util/SearchTracker.java | 5 +-
.../test/java/org/apache/atlas/BasicTestSetup.java | 741 +--
.../test/java/org/apache/atlas/TestModules.java | 52 +-
.../atlas/discovery/AtlasDiscoveryServiceTest.java | 995 ++--
.../ClassificationSearchProcessorTest.java | 506 +-
.../atlas/discovery/EntitySearchProcessorTest.java | 467 +-
.../discovery/FreeTextSearchProcessorTest.java | 100 +-
.../discovery/RelationshipSearchProcessorTest.java | 50 +-
.../entitytransform/TransformationHandlerTest.java | 73 +-
.../apache/atlas/glossary/GlossaryServiceTest.java | 279 +-
.../atlas/glossary/PaginationHelperTest.java | 44 +-
.../atlas/patches/AtlasPatchRegistryTest.java | 7 +-
.../org/apache/atlas/pc/StatusReporterTest.java | 108 +-
.../org/apache/atlas/pc/WorkItemConsumerTest.java | 87 +-
.../atlas/pc/WorkItemConsumerWithResultsTest.java | 90 +-
.../org/apache/atlas/pc/WorkItemManagerTest.java | 79 +-
.../atlas/pc/WorkItemManagerWithResultsTest.java | 124 +-
.../org/apache/atlas/query/BaseDSLComposer.java | 30 +-
.../org/apache/atlas/query/DSLQueriesTest.java | 338 +-
.../atlas/query/GremlinQueryComposerTest.java | 264 +-
.../org/apache/atlas/repository/AtlasTestBase.java | 30 +-
.../atlas/repository/audit/AdminPurgeTest.java | 89 +-
.../repository/audit/AtlasAuditServiceTest.java | 73 +-
.../repository/audit/AuditRepositoryTestBase.java | 110 +-
.../audit/CassandraAuditRepositoryTest.java | 38 +-
.../graph/RecoveryInfoManagementTest.java | 19 +-
.../repository/impexp/AtlasImportRequestTest.java | 11 +-
.../repository/impexp/AtlasServerServiceTest.java | 63 +-
.../impexp/DataMigrationStatusServiceTest.java | 20 +-
.../impexp/ExportImportAuditServiceTest.java | 56 +-
.../repository/impexp/ExportIncrementalTest.java | 196 +-
.../atlas/repository/impexp/ExportServiceTest.java | 236 +-
.../repository/impexp/ExportSkipLineageTest.java | 37 +-
.../impexp/HdfsPathEntityCreatorTest.java | 21 +-
.../impexp/ImportReactivateTableTest.java | 66 +-
.../atlas/repository/impexp/ImportServiceTest.java | 315 +-
.../impexp/ImportTransformerJSONTest.java | 20 +-
.../repository/impexp/ImportTransformerTest.java | 54 +-
.../impexp/ImportTransformsShaperTest.java | 29 +-
.../repository/impexp/ImportTransformsTest.java | 117 +-
.../IncrementalExportEntityProviderTest.java | 16 +-
.../RelationshipAttributesExtractorTest.java | 102 +-
.../impexp/ReplicationEntityAttributeTest.java | 90 +-
.../StartEntityFetchByExportRequestTest.java | 82 +-
.../TableReplicationRequestProcessorTest.java | 31 +-
.../impexp/TypeAttributeDifferenceTest.java | 164 +-
.../atlas/repository/impexp/UniqueListTest.java | 10 +-
.../atlas/repository/impexp/ZipDirectTest.java | 37 +-
.../impexp/ZipFileResourceTestUtils.java | 95 +-
.../atlas/repository/impexp/ZipSinkTest.java | 154 +-
.../atlas/repository/impexp/ZipSourceTest.java | 136 +-
.../migration/ComplexAttributesTest.java | 33 +-
.../repository/migration/HiveParititionTest.java | 22 +-
.../atlas/repository/migration/HiveStocksTest.java | 15 +-
.../repository/migration/MigrationBaseAsserts.java | 44 +-
.../migration/MigrationProgressServiceTest.java | 72 +-
.../atlas/repository/migration/PathTest.java | 44 +-
.../migration/RelationshipCacheGeneratorTest.java | 28 +-
.../repository/migration/TypesDefScrubberTest.java | 47 +-
.../migration/TypesWithClassificationTest.java | 33 +-
.../migration/TypesWithCollectionsFinderTest.java | 8 +-
.../store/graph/AtlasTypeDefGraphStoreTest.java | 452 +-
.../store/graph/v1/SoftReferenceTest.java | 75 +-
.../v2/AtlasBusinessMetadataDefStoreV2Test.java | 417 +-
.../v2/AtlasClassificationDefStoreV2Test.java | 106 +-
.../store/graph/v2/AtlasComplexAttributesTest.java | 261 +-
.../store/graph/v2/AtlasEntityComparatorTest.java | 128 +-
.../store/graph/v2/AtlasEntityDefStoreV2Test.java | 26 +-
.../store/graph/v2/AtlasEntityStoreV2Test.java | 634 ++-
.../store/graph/v2/AtlasEntityTestBase.java | 75 +-
.../graph/v2/AtlasRelationshipDefStoreV2Test.java | 169 +-
.../v2/AtlasRelationshipStoreHardDeleteV2Test.java | 32 +-
.../v2/AtlasRelationshipStoreSoftDeleteV2Test.java | 39 +-
.../graph/v2/AtlasRelationshipStoreV2Test.java | 191 +-
.../store/graph/v2/BulkImportPercentTest.java | 160 +-
.../repository/store/graph/v2/BulkLoadingTest.java | 7 +-
.../graph/v2/ClassificationAssociatorTest.java | 212 +-
.../store/graph/v2/DifferentialAuditsTest.java | 31 +-
.../store/graph/v2/EntityCorrelationStoreTest.java | 21 +-
.../v2/InverseReferenceUpdateHardDeleteV2Test.java | 25 +-
.../v2/InverseReferenceUpdateSoftDeleteV2Test.java | 30 +-
.../graph/v2/InverseReferenceUpdateV2Test.java | 103 +-
.../ClassificationPropagationTest.java | 477 +-
.../ClassificationPropagationWithTasksTest.java | 133 +-
.../userprofile/UserProfileServiceTest.java | 22 +-
.../apache/atlas/services/MetricsServiceTest.java | 144 +-
.../org/apache/atlas/tasks/BaseTaskFixture.java | 23 +-
.../org/apache/atlas/tasks/TaskExecutorTest.java | 68 +-
.../org/apache/atlas/tasks/TaskManagementTest.java | 57 +-
.../org/apache/atlas/tasks/TaskRegistryTest.java | 65 +-
.../atlas/utils/ObjectUpdateSynchronizerTest.java | 109 +-
.../org/apache/atlas/utils/TestLoadModelUtils.java | 131 +-
.../apache/atlas/utils/TestResourceFileUtils.java | 32 +-
332 files changed, 25937 insertions(+), 24298 deletions(-)
diff --git a/dev-support/checkstyle-suppressions.xml b/dev-support/checkstyle-suppressions.xml
index ee19b03e0..64c9173c8 100644
--- a/dev-support/checkstyle-suppressions.xml
+++ b/dev-support/checkstyle-suppressions.xml
@@ -32,5 +32,6 @@
<suppress files="AtlasClientV2.java" checks="TypeName"/>
<suppress files="AtlasGraph.java" checks="MethodName"/>
<suppress files="AtlasGraphTraversalSource.java" checks="MethodName"/>
+ <suppress files="DeleteHandlerV1.java" checks="MethodName"/>
<suppress files="Id.java" checks="MethodName"/>
</suppressions>
diff --git a/repository/pom.xml b/repository/pom.xml
index 2f9954618..06034304d 100644
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -31,6 +31,11 @@
<name>Apache Atlas Repository</name>
<description>Apache Atlas Repository Module</description>
+ <properties>
+ <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
+ <checkstyle.skip>false</checkstyle.skip>
+ </properties>
+
<dependencies>
<dependency>
<groupId>com.carrotsearch</groupId>
diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java
index 9751a870c..dfdc4ecbb 100644
--- a/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java
+++ b/repository/src/main/java/org/apache/atlas/GraphTransactionAdvisor.java
@@ -27,6 +27,7 @@ import org.springframework.aop.support.StaticMethodMatcherPointcut;
import org.springframework.stereotype.Component;
import javax.inject.Inject;
+
import java.lang.reflect.Method;
@Component
@@ -37,9 +38,11 @@ public class GraphTransactionAdvisor extends AbstractPointcutAdvisor {
@Override
public boolean matches(Method method, Class<?> targetClass) {
boolean annotationPresent = method.isAnnotationPresent(GraphTransaction.class);
+
if (annotationPresent) {
LOG.info("GraphTransaction intercept for {}.{}", targetClass.getName(), method.getName());
}
+
return annotationPresent;
}
};
diff --git a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
index c8b7ff8d9..40587d2de 100644
--- a/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
+++ b/repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
@@ -17,7 +17,6 @@
package org.apache.atlas;
-import com.google.common.annotations.VisibleForTesting;
import org.aopalliance.intercept.MethodInterceptor;
import org.aopalliance.intercept.MethodInvocation;
import org.apache.atlas.annotation.GraphTransaction;
@@ -35,6 +34,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import javax.ws.rs.core.Response;
+
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collections;
@@ -48,72 +48,149 @@ import java.util.concurrent.locks.ReentrantLock;
public class GraphTransactionInterceptor implements MethodInterceptor {
private static final Logger LOG = LoggerFactory.getLogger(GraphTransactionInterceptor.class);
- @VisibleForTesting
- private static final ObjectUpdateSynchronizer OBJECT_UPDATE_SYNCHRONIZER = new ObjectUpdateSynchronizer();
- private static final ThreadLocal<List<PostTransactionHook>> postTransactionHooks = new ThreadLocal<>();
- private static final ThreadLocal<Boolean> isTxnOpen = ThreadLocal.withInitial(() -> Boolean.FALSE);
- private static final ThreadLocal<Boolean> innerFailure = ThreadLocal.withInitial(() -> Boolean.FALSE);
- private static final ThreadLocal<Map<String, AtlasVertex>> guidVertexCache = ThreadLocal.withInitial(() -> new HashMap<>());
+ private static final ObjectUpdateSynchronizer OBJECT_UPDATE_SYNCHRONIZER = new ObjectUpdateSynchronizer();
+ private static final ThreadLocal<List<PostTransactionHook>> postTransactionHooks = new ThreadLocal<>();
+ private static final ThreadLocal<Boolean> isTxnOpen = ThreadLocal.withInitial(() -> Boolean.FALSE);
+ private static final ThreadLocal<Boolean> innerFailure = ThreadLocal.withInitial(() -> Boolean.FALSE);
+ private static final ThreadLocal<Map<String, AtlasVertex>> guidVertexCache = ThreadLocal.withInitial(HashMap::new);
+ private static final ThreadLocal<Map<Object, String>> vertexGuidCache = ThreadLocal.withInitial(HashMap::new);
+ private static final ThreadLocal<Map<Object, AtlasEntity.Status>> vertexStateCache = ThreadLocal.withInitial(HashMap::new);
+ private static final ThreadLocal<Map<Object, AtlasEntity.Status>> edgeStateCache = ThreadLocal.withInitial(HashMap::new);
private final AtlasGraph graph;
private final TaskManagement taskManagement;
- private static final ThreadLocal<Map<Object, String>> vertexGuidCache =
- new ThreadLocal<Map<Object, String>>() {
- @Override
- public Map<Object, String> initialValue() {
- return new HashMap<Object, String>();
- }
- };
-
- private static final ThreadLocal<Map<Object, AtlasEntity.Status>> vertexStateCache =
- new ThreadLocal<Map<Object, AtlasEntity.Status>>() {
- @Override
- public Map<Object, AtlasEntity.Status> initialValue() {
- return new HashMap<Object, AtlasEntity.Status>();
- }
- };
-
- private static final ThreadLocal<Map<Object, AtlasEntity.Status>> edgeStateCache =
- new ThreadLocal<Map<Object, AtlasEntity.Status>>() {
- @Override
- public Map<Object, AtlasEntity.Status> initialValue() {
- return new HashMap<Object, AtlasEntity.Status>();
- }
- };
-
@Inject
public GraphTransactionInterceptor(AtlasGraph graph, TaskManagement taskManagement) {
this.graph = graph;
this.taskManagement = taskManagement;
}
+ public static void lockObjectAndReleasePostCommit(final String guid) {
+ OBJECT_UPDATE_SYNCHRONIZER.lockObject(guid);
+ }
+
+ public static void lockObjectAndReleasePostCommit(final List<String> guids) {
+ OBJECT_UPDATE_SYNCHRONIZER.lockObject(guids);
+ }
+
+ public static void addToVertexCache(String guid, AtlasVertex vertex) {
+ Map<String, AtlasVertex> cache = guidVertexCache.get();
+
+ cache.put(guid, vertex);
+ }
+
+ public static void removeFromVertexCache(String guid) {
+ Map<String, AtlasVertex> cache = guidVertexCache.get();
+
+ cache.remove(guid);
+ }
+
+ public static AtlasVertex getVertexFromCache(String guid) {
+ Map<String, AtlasVertex> cache = guidVertexCache.get();
+
+ return cache.get(guid);
+ }
+
+ public static void clearCache() {
+ guidVertexCache.get().clear();
+ vertexGuidCache.get().clear();
+ vertexStateCache.get().clear();
+ edgeStateCache.get().clear();
+ }
+
+ public static void addToVertexGuidCache(Object vertexId, String guid) {
+ if (guid == null) {
+ removeFromVertexGuidCache(vertexId);
+ } else {
+ Map<Object, String> cache = vertexGuidCache.get();
+
+ cache.put(vertexId, guid);
+ }
+ }
+
+ public static void removeFromVertexGuidCache(Object vertexId) {
+ Map<Object, String> cache = vertexGuidCache.get();
+
+ cache.remove(vertexId);
+ }
+
+ public static String getVertexGuidFromCache(Object vertexId) {
+ Map<Object, String> cache = vertexGuidCache.get();
+
+ return cache.get(vertexId);
+ }
+
+ public static void addToVertexStateCache(Object vertexId, AtlasEntity.Status status) {
+ if (status == null) {
+ removeFromVertexStateCache(vertexId);
+ } else {
+ Map<Object, AtlasEntity.Status> cache = vertexStateCache.get();
+
+ cache.put(vertexId, status);
+ }
+ }
+
+ public static void removeFromVertexStateCache(Object vertexId) {
+ Map<Object, AtlasEntity.Status> cache = vertexStateCache.get();
+
+ cache.remove(vertexId);
+ }
+
+ public static AtlasEntity.Status getVertexStateFromCache(Object vertexId) {
+ Map<Object, AtlasEntity.Status> cache = vertexStateCache.get();
+
+ return cache.get(vertexId);
+ }
+
+ public static void addToEdgeStateCache(Object edgeId, AtlasEntity.Status status) {
+ if (status == null) {
+ removeFromEdgeStateCache(edgeId);
+ } else {
+ Map<Object, AtlasEntity.Status> cache = edgeStateCache.get();
+
+ cache.put(edgeId, status);
+ }
+ }
+
+ public static void removeFromEdgeStateCache(Object edgeId) {
+ Map<Object, AtlasEntity.Status> cache = edgeStateCache.get();
+
+ cache.remove(edgeId);
+ }
+
+ public static AtlasEntity.Status getEdgeStateFromCache(Object edgeId) {
+ Map<Object, AtlasEntity.Status> cache = edgeStateCache.get();
+
+ return cache.get(edgeId);
+ }
+
@Override
public Object invoke(MethodInvocation invocation) throws Throwable {
- Method method = invocation.getMethod();
- String invokingClass = method.getDeclaringClass().getSimpleName();
- String invokedMethodName = method.getName();
- boolean logRollback = method.getAnnotation(GraphTransaction.class) == null || method.getAnnotation(GraphTransaction.class).logRollback();
+ Method method = invocation.getMethod();
+ String invokingClass = method.getDeclaringClass().getSimpleName();
+ String invokedMethodName = method.getName();
+ GraphTransaction annotation = method.getAnnotation(GraphTransaction.class);
+ boolean logRollback = annotation == null || annotation.logRollback();
boolean isInnerTxn = isTxnOpen.get();
+
// Outermost txn marks any subsequent transaction as inner
isTxnOpen.set(Boolean.TRUE);
- if (LOG.isDebugEnabled() && isInnerTxn) {
+ if (isInnerTxn) {
LOG.debug("Txn entry-point {}.{} is inner txn. Commit/Rollback will be ignored", invokingClass, invokedMethodName);
}
- boolean isSuccess = false;
- MetricRecorder metric = null;
+ boolean isSuccess = false;
+ MetricRecorder metric = null;
try {
try {
Object response = invocation.proceed();
if (isInnerTxn) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Ignoring commit for nested/inner transaction {}.{}", invokingClass, invokedMethodName);
- }
+ LOG.debug("Ignoring commit for nested/inner transaction {}.{}", invokingClass, invokedMethodName);
} else {
metric = RequestContext.get().startMetricRecord("graphCommit");
@@ -125,13 +202,13 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
return response;
} catch (Throwable t) {
if (isInnerTxn) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Ignoring rollback for nested/inner transaction {}.{}", invokingClass, invokedMethodName);
- }
+ LOG.debug("Ignoring rollback for nested/inner transaction {}.{}", invokingClass, invokedMethodName);
+
innerFailure.set(true);
} else {
doRollback(logRollback, t);
}
+
throw t;
}
} finally {
@@ -139,9 +216,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
// Only outer txn can mark as closed
if (!isInnerTxn) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Closing outer txn");
- }
+ LOG.debug("Closing outer txn");
// Reset the boolean flags
isTxnOpen.set(Boolean.FALSE);
@@ -151,9 +226,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
List<PostTransactionHook> trxHooks = postTransactionHooks.get();
if (trxHooks != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing post-txn hooks");
- }
+ LOG.debug("Processing post-txn hooks");
postTransactionHooks.remove();
@@ -175,11 +248,20 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
}
}
+ boolean logException(Throwable t) {
+ if (t instanceof AtlasBaseException) {
+ Response.Status httpCode = ((AtlasBaseException) t).getAtlasErrorCode().getHttpCode();
+
+ return httpCode != Response.Status.NOT_FOUND && httpCode != Response.Status.NO_CONTENT;
+ } else {
+ return !(t instanceof NotFoundException);
+ }
+ }
+
private void doCommitOrRollback(final String invokingClass, final String invokedMethodName) {
if (innerFailure.get()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Inner/Nested call threw exception. Rollback on txn entry-point, {}.{}", invokingClass, invokedMethodName);
- }
+ LOG.debug("Inner/Nested call threw exception. Rollback on txn entry-point, {}.{}", invokingClass, invokedMethodName);
+
graph.rollback();
} else {
doCommit(invokingClass, invokedMethodName);
@@ -189,9 +271,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
private void doCommit(final String invokingClass, final String invokedMethodName) {
graph.commit();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Graph commit txn {}.{}", invokingClass, invokedMethodName);
- }
+ LOG.debug("Graph commit txn {}.{}", invokingClass, invokedMethodName);
}
private void doRollback(boolean logRollback, final Throwable t) {
@@ -206,39 +286,6 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
graph.rollback();
}
- public static void lockObjectAndReleasePostCommit(final String guid) {
- OBJECT_UPDATE_SYNCHRONIZER.lockObject(guid);
- }
-
- public static void lockObjectAndReleasePostCommit(final List<String> guids) {
- OBJECT_UPDATE_SYNCHRONIZER.lockObject(guids);
- }
-
- public static void addToVertexCache(String guid, AtlasVertex vertex) {
- Map<String, AtlasVertex> cache = guidVertexCache.get();
-
- cache.put(guid, vertex);
- }
-
- public static void removeFromVertexCache(String guid) {
- Map<String, AtlasVertex> cache = guidVertexCache.get();
-
- cache.remove(guid);
- }
-
- public static AtlasVertex getVertexFromCache(String guid) {
- Map<String, AtlasVertex> cache = guidVertexCache.get();
-
- return cache.get(guid);
- }
-
- public static void clearCache() {
- guidVertexCache.get().clear();
- vertexGuidCache.get().clear();
- vertexStateCache.get().clear();
- edgeStateCache.get().clear();
- }
-
private void submitTasks() {
if (CollectionUtils.isEmpty(RequestContext.get().getQueuedTasks()) || taskManagement == null) {
return;
@@ -247,84 +294,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
taskManagement.addAll(RequestContext.get().getQueuedTasks());
}
- boolean logException(Throwable t) {
- if (t instanceof AtlasBaseException) {
- Response.Status httpCode = ((AtlasBaseException) t).getAtlasErrorCode().getHttpCode();
- return httpCode != Response.Status.NOT_FOUND && httpCode != Response.Status.NO_CONTENT;
- } else if (t instanceof NotFoundException) {
- return false;
- } else {
- return true;
- }
- }
-
- public static void addToVertexGuidCache(Object vertexId, String guid) {
-
- if (guid == null) {
- removeFromVertexGuidCache(vertexId);
- } else {
- Map<Object, String> cache = vertexGuidCache.get();
- cache.put(vertexId, guid);
- }
- }
-
- public static void removeFromVertexGuidCache(Object vertexId) {
- Map<Object, String> cache = vertexGuidCache.get();
-
- cache.remove(vertexId);
- }
-
- public static String getVertexGuidFromCache(Object vertexId) {
- Map<Object, String> cache = vertexGuidCache.get();
-
- return cache.get(vertexId);
- }
-
- public static void addToVertexStateCache(Object vertexId, AtlasEntity.Status status) {
-
- if (status == null) {
- removeFromVertexStateCache(vertexId);
- } else {
- Map<Object, AtlasEntity.Status> cache = vertexStateCache.get();
- cache.put(vertexId, status);
- }
- }
-
- public static void removeFromVertexStateCache(Object vertexId) {
- Map<Object, AtlasEntity.Status> cache = vertexStateCache.get();
-
- cache.remove(vertexId);
- }
-
- public static AtlasEntity.Status getVertexStateFromCache(Object vertexId) {
- Map<Object, AtlasEntity.Status> cache = vertexStateCache.get();
-
- return cache.get(vertexId);
- }
-
- public static void addToEdgeStateCache(Object edgeId, AtlasEntity.Status status) {
-
- if (status == null) {
- removeFromEdgeStateCache(edgeId);
- } else {
- Map<Object, AtlasEntity.Status> cache = edgeStateCache.get();
- cache.put(edgeId, status);
- }
- }
-
- public static void removeFromEdgeStateCache(Object edgeId) {
- Map<Object, AtlasEntity.Status> cache = edgeStateCache.get();
-
- cache.remove(edgeId);
- }
-
- public static AtlasEntity.Status getEdgeStateFromCache(Object edgeId) {
- Map<Object, AtlasEntity.Status> cache = edgeStateCache.get();
-
- return cache.get(edgeId);
- }
-
- public static abstract class PostTransactionHook {
+ public abstract static class PostTransactionHook {
protected PostTransactionHook() {
List<PostTransactionHook> trxHooks = postTransactionHooks.get();
@@ -354,70 +324,63 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
return --refCount;
}
- public int getRefCount() { return refCount; }
+ public int getRefCount() {
+ return refCount;
+ }
}
-
public static class ObjectUpdateSynchronizer {
private final Map<String, RefCountedReentrantLock> guidLockMap = new ConcurrentHashMap<>();
- private final ThreadLocal<List<String>> lockedGuids = new ThreadLocal<List<String>>() {
- @Override
- protected List<String> initialValue() {
- return new ArrayList<>();
- }
- };
+ private final ThreadLocal<List<String>> lockedGuids = ThreadLocal.withInitial(ArrayList::new);
- public void lockObject(final List<?> guids) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> lockObject(): guids: {}", guids);
- }
+ public void lockObject(final List<String> guids) {
+ LOG.debug("==> lockObject(): guids: {}", guids);
+
+ Collections.sort(guids);
- Collections.sort((List<String>) guids);
- for (String g : (List<String>) guids) {
+ for (String g : guids) {
lockObject(g);
}
}
- private void lockObject(final String guid) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> lockObject(): guid: {}, guidLockMap.size: {}", guid, guidLockMap.size());
- }
-
- ReentrantLock lock = getOrCreateObjectLock(guid);
- lock.lock();
+ public void releaseLockedObjects() {
+ List<String> guids = lockedGuids.get();
- lockedGuids.get().add(guid);
+ LOG.debug("==> releaseLockedObjects(): lockedGuids.size: {}", guids.size());
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== lockObject(): guid: {}, guidLockMap.size: {}", guid, guidLockMap.size());
+ for (String guid : guids) {
+ releaseObjectLock(guid);
}
+
+ guids.clear();
+
+ LOG.debug("<== releaseLockedObjects(): lockedGuids.size: {}", guids.size());
}
- public void releaseLockedObjects() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> releaseLockedObjects(): lockedGuids.size: {}", lockedGuids.get().size());
- }
+ private void lockObject(final String guid) {
+ LOG.debug("==> lockObject(): guid: {}, guidLockMap.size: {}", guid, guidLockMap.size());
- for (String guid : lockedGuids.get()) {
- releaseObjectLock(guid);
- }
+ ReentrantLock lock = getOrCreateObjectLock(guid);
- lockedGuids.get().clear();
+ lock.lock();
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== releaseLockedObjects(): lockedGuids.size: {}", lockedGuids.get().size());
- }
+ lockedGuids.get().add(guid);
+
+ LOG.debug("<== lockObject(): guid: {}, guidLockMap.size: {}", guid, guidLockMap.size());
}
private RefCountedReentrantLock getOrCreateObjectLock(String guid) {
synchronized (guidLockMap) {
RefCountedReentrantLock ret = guidLockMap.get(guid);
+
if (ret == null) {
ret = new RefCountedReentrantLock();
+
guidLockMap.put(guid, ret);
}
ret.increment();
+
return ret;
}
}
@@ -425,6 +388,7 @@ public class GraphTransactionInterceptor implements MethodInterceptor {
private RefCountedReentrantLock releaseObjectLock(String guid) {
synchronized (guidLockMap) {
RefCountedReentrantLock lock = guidLockMap.get(guid);
+
if (lock != null && lock.isHeldByCurrentThread()) {
int refCount = lock.decrement();
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index acab4ce0a..9e0f1973d 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -18,9 +18,14 @@
package org.apache.atlas.discovery;
-
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.discovery.*;
+import org.apache.atlas.model.discovery.AtlasQuickSearchResult;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.discovery.AtlasSearchResultDownloadStatus;
+import org.apache.atlas.model.discovery.AtlasSuggestionsResult;
+import org.apache.atlas.model.discovery.QuickSearchParameters;
+import org.apache.atlas.model.discovery.RelationshipSearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import org.apache.atlas.model.tasks.AtlasTask;
import org.apache.atlas.repository.Constants.AtlasAuditAgingType;
@@ -32,7 +37,6 @@ import java.util.Set;
public interface AtlasDiscoveryService {
/**
- *
* @param query search query in DSL format.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
@@ -41,7 +45,6 @@ public interface AtlasDiscoveryService {
AtlasSearchResult searchUsingDslQuery(String query, int limit, int offset) throws AtlasBaseException;
/**
- *
* @param query query
* @param typeName type name
* @param classification classification
@@ -50,7 +53,6 @@ public interface AtlasDiscoveryService {
String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification);
/**
- *
* @param query search query.
* @param excludeDeletedEntities exclude deleted entities in search result.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
@@ -60,7 +62,6 @@ public interface AtlasDiscoveryService {
AtlasSearchResult searchUsingFullTextQuery(String query, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
/**
- *
* @param query search query.
* @param type entity type.
* @param classification classification name.
@@ -72,10 +73,11 @@ public interface AtlasDiscoveryService {
* @return AtlasSearchResult
*/
AtlasSearchResult searchUsingBasicQuery(String query, String type, String classification, String attrName,
- String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
+ String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
/**
* Search for entities matching the search criteria
+ *
* @param searchParameters Search criteria
* @return Matching entities
* @throws AtlasBaseException
@@ -84,6 +86,7 @@ public interface AtlasDiscoveryService {
/**
* Search for guids of entities matching the search criteria
+ *
* @param searchParameters Search criteria
* @return GUIDs of matching entities
* @throws AtlasBaseException
@@ -92,39 +95,35 @@ public interface AtlasDiscoveryService {
/**
* Search for relations (edges) matching the search criteria
+ *
* @param searchParameters Search criteria
* @return Matching Edge Entities
* @throws AtlasBaseException
*/
- AtlasSearchResult searchRelationsWithParameters(RelationshipSearchParameters searchParameters) throws AtlasBaseException;
+ AtlasSearchResult searchRelationsWithParameters(RelationshipSearchParameters searchParameters) throws AtlasBaseException;
/**
- *
* @param guid unique ID of the entity.
* @param relation relation name.
* @param getApproximateCount
* @param searchParameters
* @return AtlasSearchResult
*/
- AtlasSearchResult searchRelatedEntities(String guid, String relation, boolean getApproximateCount, SearchParameters searchParameters) throws AtlasBaseException;
+ AtlasSearchResult searchRelatedEntities(String guid, String relation, boolean getApproximateCount, SearchParameters searchParameters) throws AtlasBaseException;
/**
- *
- *
* @param savedSearch Search to be saved
* @throws AtlasBaseException
*/
AtlasUserSavedSearch addSavedSearch(String currentUser, AtlasUserSavedSearch savedSearch) throws AtlasBaseException;
/**
- *
* @param savedSearch Search to be saved
* @throws AtlasBaseException
*/
AtlasUserSavedSearch updateSavedSearch(String currentUser, AtlasUserSavedSearch savedSearch) throws AtlasBaseException;
/**
- *
* @param userName Name of the user for whom the saved searches are to be retrieved
* @return List of saved searches for the user
* @throws AtlasBaseException
@@ -132,7 +131,6 @@ public interface AtlasDiscoveryService {
List<AtlasUserSavedSearch> getSavedSearches(String currentUser, String userName) throws AtlasBaseException;
/**
- *
* @param guid Guid for the saved search
* @return Search object identified by the guid
* @throws AtlasBaseException
@@ -140,7 +138,6 @@ public interface AtlasDiscoveryService {
AtlasUserSavedSearch getSavedSearchByGuid(String currentUser, String guid) throws AtlasBaseException;
/**
- *
* @param userName Name of the user who the search belongs
* @param searchName Name of the search to be retrieved
* @return Search object identified by the name
@@ -149,7 +146,6 @@ public interface AtlasDiscoveryService {
AtlasUserSavedSearch getSavedSearchByName(String currentUser, String userName, String searchName) throws AtlasBaseException;
/**
- *
* @param currentUser User requesting the operation
* @param guid Guid used to look up Saved Search
* @throws AtlasBaseException
@@ -158,6 +154,7 @@ public interface AtlasDiscoveryService {
/**
* Search for entities matching the search criteria
+ *
* @param searchParameters Search criteria
* @return Matching entities
* @throws AtlasBaseException
@@ -166,6 +163,7 @@ public interface AtlasDiscoveryService {
/**
* Should return top 5 suggestion strings for the given prefix.
+ *
* @param prefixString the prefix string
* @param fieldName field from which to retrieve suggestions
* @return top 5 suggestion strings for the given prefix.
@@ -174,12 +172,12 @@ public interface AtlasDiscoveryService {
/**
* Creates task to search and download the results of Basic and DSL search
+ *
* @param taskParams parameters of AtlasTask
*/
void createAndQueueSearchResultDownloadTask(Map<String, Object> taskParams) throws AtlasBaseException;
/**
- *
* @return AtlasSearchResultDownloadStatus
* @throws IOException
*/
@@ -187,6 +185,7 @@ public interface AtlasDiscoveryService {
/**
* Creates task to age out audits
+ *
* @param taskParams parameters of AtlasTask
* @return Task created to perform audit aging
*/
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java
index 0b518b904..24d6b1f1c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasLineageService.java
@@ -18,7 +18,6 @@
package org.apache.atlas.discovery;
-
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index dfcc441cc..8329c0f9b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -21,7 +21,12 @@ import org.apache.atlas.SortOrder;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.graphdb.AtlasEdge;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.util.SearchPredicateUtil;
@@ -35,7 +40,14 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
/**
* This class is needed when this is a registered classification type or wildcard search,
@@ -43,35 +55,34 @@ import java.util.*;
* tag-filter for special classification types, including wildcard search - classification name contains *)
*/
public class ClassificationSearchProcessor extends SearchProcessor {
+ private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
- private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
- private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
-
- private final AtlasIndexQuery indexQuery;
- private final AtlasIndexQuery classificationIndexQuery;
- private final AtlasGraphQuery tagGraphQueryWithAttributes;
- private final Predicate traitPredicate;
- private final Predicate isEntityPredicate;
- private Predicate activePredicate;
+ private final AtlasIndexQuery indexQuery;
+ private final AtlasIndexQuery classificationIndexQuery;
+ private final AtlasGraphQuery tagGraphQueryWithAttributes;
+ private final Predicate traitPredicate;
+ private final Predicate isEntityPredicate;
+ private Predicate activePredicate;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
// eg, an entity A has classification "cls" and B has "cls 1"
// when user execute a exact search for "cls", only A should be returned
// but both A and B are returned. To avoid this, we should filter the res.
- private boolean whiteSpaceFilter = false;
+ private boolean whiteSpaceFilter;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
- final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
- final Set<String> indexAttributes = new HashSet<>();
- final Set<String> graphAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
- final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
- final boolean isWildcardSearch = context.isWildCardSearch();
- final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
+ final Set<String> indexAttributes = new HashSet<>();
+ final Set<String> graphAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
+ final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
+ final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
+ final boolean isWildcardSearch = context.isWildCardSearch();
+ final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
processSearchAttributes(classificationTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
@@ -84,72 +95,68 @@ public class ClassificationSearchProcessor extends SearchProcessor {
each of above cases with either has empty/or not tagFilters
*/
final boolean useIndexSearchForEntity = (CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) &&
- !context.hasAttributeFilter(filterCriteria) &&
- (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
+ !context.hasAttributeFilter(filterCriteria) &&
+ (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well.
*/
final boolean useIndexSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
- classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
- !isWildcardSearch) &&
- (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
- CollectionUtils.isNotEmpty(indexAttributes) &&
- canApplyIndexFilter(classificationTypes, filterCriteria, false);
+ classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
+ !isWildcardSearch) &&
+ (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
+ CollectionUtils.isNotEmpty(indexAttributes) &&
+ canApplyIndexFilter(classificationTypes, filterCriteria, false);
final boolean useGraphSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
- classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
- !isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes));
+ classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
+ !isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes));
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
if (context.getSearchParameters().getExcludeDeletedEntities()) {
- activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
- .generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
+ activePredicate = SearchPredicateUtil.getEQPredicateGenerator().generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
}
- Predicate attributePredicate = null;
- Predicate typeNamePredicate = null;
-
- AtlasGraph graph = context.getGraph();
+ Predicate attributePredicate = null;
+ Predicate typeNamePredicate = null;
+ AtlasGraph graph = context.getGraph();
// index query directly on entity
if (useIndexSearchForEntity) {
-
StringBuilder queryString = new StringBuilder();
+
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
if (isWildcardSearch) {
-
// tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else {
if (classificationTypes.iterator().next() == SearchContext.MATCH_ALL_NOT_CLASSIFIED) {
// tagFilters is not allowed in unique classificationType search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
-
} else {
-
// only registered classification will search for subtypes
graphIndexQueryBuilder.addClassificationAndSubTypesQueryFilter(queryString);
+
whiteSpaceFilter = true;
}
}
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
- indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
- indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
- indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
- LOG.debug("Using query string '{}'.", indexQuery);
+ indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
+ indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
+ indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
+
+ LOG.debug("Using query string '{}'.", indexQuery);
} else {
indexQuery = null;
}
// index query directly on classification
if (useIndexSearchForClassification) {
-
StringBuilder queryString = new StringBuilder();
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
@@ -158,22 +165,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
constructFilterQuery(queryString, classificationTypes, filterCriteria, indexAttributes);
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
+
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
- typeNamePredicate = isClassificationRootType() ? null :
- SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
-
} else {
classificationIndexQuery = null;
}
// only registered classification will search with tag filters
if (useGraphSearchForClassification) {
-
AtlasGraphQuery query = graph.query();
if (!isClassificationRootType()) {
@@ -182,10 +187,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
- typeNamePredicate = isClassificationRootType() ? null :
- SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, allAttributes);
-
} else {
tagGraphQueryWithAttributes = null;
}
@@ -200,21 +203,18 @@ public class ClassificationSearchProcessor extends SearchProcessor {
@Override
public List<AtlasVertex> execute() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> ClassificationSearchProcessor.execute({})", context);
- }
-
- List<AtlasVertex> ret = new ArrayList<>();
+ LOG.debug("==> ClassificationSearchProcessor.execute({})", context);
- AtlasPerfTracer perf = null;
+ List<AtlasVertex> ret = new ArrayList<>();
+ AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ClassificationSearchProcessor.execute(" + context + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "ClassificationSearchProcessor.execute(" + context + ")");
}
try {
- final int limit = context.getSearchParameters().getLimit();
- Integer marker = context.getMarker();
+ final int limit = context.getSearchParameters().getLimit();
+ Integer marker = context.getMarker();
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like non-active-entities or duplicate-entities (same entity pointed to by multiple
@@ -225,6 +225,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (indexQuery == null) {
marker = null;
}
+
// if marker is provided, start query with marker offset
int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
int qryOffset = marker != null ? marker : 0;
@@ -234,8 +235,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap = new LinkedHashMap<>();
final List<AtlasVertex> classificationVertices = new ArrayList<>();
- final String sortBy = context.getSearchParameters().getSortBy();
- final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+ final String sortBy = context.getSearchParameters().getSortBy();
+ final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
for (; ret.size() < limit; qryOffset += limit) {
offsetEntityVertexMap.clear();
@@ -250,24 +251,24 @@ public class ClassificationSearchProcessor extends SearchProcessor {
boolean isLastResultPage = true;
if (indexQuery != null) {
- Iterator<AtlasIndexQuery.Result> queryResult;
+ final Iterator<AtlasIndexQuery.Result> queryResult;
+
if (StringUtils.isNotEmpty(sortBy)) {
Order qrySortOrder = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
+
queryResult = indexQuery.vertices(qryOffset, limit, sortBy, qrySortOrder);
} else {
queryResult = indexQuery.vertices(qryOffset, limit);
}
- offsetEntityVertexMap = getVerticesFromIndexQueryResult(queryResult, offsetEntityVertexMap, qryOffset);
- isLastResultPage = offsetEntityVertexMap.size() < limit;
+ offsetEntityVertexMap = getVerticesFromIndexQueryResult(queryResult, offsetEntityVertexMap, qryOffset);
+ isLastResultPage = offsetEntityVertexMap.size() < limit;
// Do in-memory filtering
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, traitPredicate);
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, isEntityPredicate);
-
} else {
if (classificationIndexQuery != null) {
-
Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
getVerticesFromIndexQueryResult(queryResult, classificationVertices);
@@ -276,7 +277,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
} else if (tagGraphQueryWithAttributes != null) {
-
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
getVertices(queryResult, classificationVertices);
@@ -297,6 +297,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
for (AtlasEdge edge : edges) {
AtlasVertex entityVertex = edge.getOutVertex();
+
resultCount++;
String guid = AtlasGraphUtilsV2.getIdFromVertex(entityVertex);
@@ -315,8 +316,10 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (whiteSpaceFilter) {
offsetEntityVertexMap = filterWhiteSpaceClassification(offsetEntityVertexMap);
}
+
// Do in-memory filtering
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, isEntityPredicate);
+
if (activePredicate != null) {
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, activePredicate);
}
@@ -334,42 +337,41 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (marker != null) {
nextOffset = resultIdx + 1;
}
-
} finally {
AtlasPerfTracer.log(perf);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== ClassificationSearchProcessor.execute({}): ret.size()={}", context, ret.size());
- }
+ LOG.debug("<== ClassificationSearchProcessor.execute({}): ret.size()={}", context, ret.size());
return ret;
}
+ @Override
+ public long getResultCount() {
+ return (indexQuery != null) ? indexQuery.vertexTotals() : -1;
+ }
+
@Override
public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> ClassificationSearchProcessor.filter({})", offsetEntityVertexMap.size());
- }
+ LOG.debug("==> ClassificationSearchProcessor.filter({})", offsetEntityVertexMap.size());
if (inMemoryPredicate != null) {
//in case of classification type + index attributes
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, traitPredicate);
//filter attributes (filterCriteria). Find classification vertex(typeName = classification) from entity vertex (traitName = classification)
- final Set<String> processedGuids = new HashSet<>();
+ final Set<String> processedGuids = new HashSet<>();
LinkedHashMap<Integer, AtlasVertex> matchEntityVertices = new LinkedHashMap<>();
if (MapUtils.isNotEmpty(offsetEntityVertexMap)) {
for (Map.Entry<Integer, AtlasVertex> offsetToEntity : offsetEntityVertexMap.entrySet()) {
-
- AtlasVertex entityVertex = offsetToEntity.getValue();
- Iterable<AtlasEdge> edges = entityVertex.getEdges(AtlasEdgeDirection.OUT, Constants.CLASSIFICATION_LABEL);
+ AtlasVertex entityVertex = offsetToEntity.getValue();
+ Iterable<AtlasEdge> edges = entityVertex.getEdges(AtlasEdgeDirection.OUT, Constants.CLASSIFICATION_LABEL);
for (AtlasEdge edge : edges) {
AtlasVertex classificationVertex = edge.getInVertex();
+ AtlasVertex matchVertex = (AtlasVertex) CollectionUtils.find(Collections.singleton(classificationVertex), inMemoryPredicate);
- AtlasVertex matchVertex = (AtlasVertex) CollectionUtils.find(Collections.singleton(classificationVertex), inMemoryPredicate);
if (matchVertex != null) {
String guid = AtlasGraphUtilsV2.getIdFromVertex(entityVertex);
@@ -380,14 +382,13 @@ public class ClassificationSearchProcessor extends SearchProcessor {
matchEntityVertices.put(offsetToEntity.getKey(), entityVertex);
processedGuids.add(guid);
break;
-
}
}
}
}
+
offsetEntityVertexMap.clear();
offsetEntityVertexMap.putAll(matchEntityVertices);
-
} else {
//in case of only classsification type
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, traitPredicate);
@@ -396,15 +397,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
offsetEntityVertexMap = super.filter(offsetEntityVertexMap);
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== ClassificationSearchProcessor.filter(): ret.size()={}", offsetEntityVertexMap.size());
- }
+ LOG.debug("<== ClassificationSearchProcessor.filter(): ret.size()={}", offsetEntityVertexMap.size());
return offsetEntityVertexMap;
}
-
- @Override
- public long getResultCount() {
- return (indexQuery != null) ? indexQuery.vertexTotals() : -1;
- }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index dc3acf695..626fbb057 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -70,10 +70,10 @@ import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery;
import org.apache.atlas.util.SearchPredicateUtil;
@@ -92,6 +92,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
+
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
@@ -108,7 +109,10 @@ import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
-import static org.apache.atlas.AtlasErrorCode.*;
+import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
+import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
+import static org.apache.atlas.AtlasErrorCode.PENDING_TASKS_ALREADY_IN_PROGRESS;
+import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
@@ -120,35 +124,31 @@ import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.
@Component
public class EntityDiscoveryService implements AtlasDiscoveryService {
private static final Logger LOG = LoggerFactory.getLogger(EntityDiscoveryService.class);
+
private static final String DEFAULT_SORT_ATTRIBUTE_NAME = "name";
- private final AtlasGraph graph;
- private final EntityGraphRetriever entityRetriever;
- private final AtlasGremlinQueryProvider gremlinQueryProvider;
- private final AtlasTypeRegistry typeRegistry;
- private final GraphBackedSearchIndexer indexer;
- private final SearchTracker searchTracker;
- private final int maxResultSetSize;
- private final int maxTypesLengthInIdxQuery;
- private final int maxTagsLengthInIdxQuery;
- private final String indexSearchPrefix;
- private final UserProfileService userProfileService;
- private final SuggestionsProvider suggestionsProvider;
- private final DSLQueryExecutor dslQueryExecutor;
- private final TaskManagement taskManagement;
+ private final AtlasGraph graph;
+ private final EntityGraphRetriever entityRetriever;
+ private final AtlasGremlinQueryProvider gremlinQueryProvider;
+ private final AtlasTypeRegistry typeRegistry;
+ private final GraphBackedSearchIndexer indexer;
+ private final SearchTracker searchTracker;
+ private final int maxResultSetSize;
+ private final int maxTypesLengthInIdxQuery;
+ private final int maxTagsLengthInIdxQuery;
+ private final String indexSearchPrefix;
+ private final UserProfileService userProfileService;
+ private final SuggestionsProvider suggestionsProvider;
+ private final DSLQueryExecutor dslQueryExecutor;
+ private final TaskManagement taskManagement;
@Inject
- EntityDiscoveryService(AtlasTypeRegistry typeRegistry,
- AtlasGraph graph,
- GraphBackedSearchIndexer indexer,
- SearchTracker searchTracker,
- UserProfileService userProfileService,
- TaskManagement taskManagement) throws AtlasException {
+ EntityDiscoveryService(AtlasTypeRegistry typeRegistry, AtlasGraph graph, GraphBackedSearchIndexer indexer, SearchTracker searchTracker, UserProfileService userProfileService, TaskManagement taskManagement) throws AtlasException {
this.graph = graph;
this.entityRetriever = new EntityGraphRetriever(this.graph, typeRegistry);
this.indexer = indexer;
this.searchTracker = searchTracker;
- this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
+ this.gremlinQueryProvider = AtlasGremlinQueryProvider.getInstance();
this.typeRegistry = typeRegistry;
this.maxResultSetSize = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
this.maxTypesLengthInIdxQuery = ApplicationProperties.get().getInt(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
@@ -156,13 +156,57 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
this.indexSearchPrefix = AtlasGraphUtilsV2.getIndexSearchPrefix();
this.userProfileService = userProfileService;
this.suggestionsProvider = new SuggestionsProviderImpl(graph, typeRegistry);
- this.dslQueryExecutor = AtlasConfiguration.DSL_EXECUTOR_TRAVERSAL.getBoolean()
- ? new TraversalBasedExecutor(typeRegistry, graph, entityRetriever)
- : new ScriptEngineBasedExecutor(typeRegistry, graph, entityRetriever);
+ this.dslQueryExecutor = AtlasConfiguration.DSL_EXECUTOR_TRAVERSAL.getBoolean() ? new TraversalBasedExecutor(typeRegistry, graph, entityRetriever) : new ScriptEngineBasedExecutor(typeRegistry, graph, entityRetriever);
this.taskManagement = taskManagement;
+
LOG.info("DSL Executor: {}", this.dslQueryExecutor.getClass().getSimpleName());
}
+ public static SearchParameters createSearchParameters(QuickSearchParameters quickSearchParameters) {
+ SearchParameters searchParameters = new SearchParameters();
+
+ searchParameters.setQuery(quickSearchParameters.getQuery());
+ searchParameters.setTypeName(quickSearchParameters.getTypeName());
+ searchParameters.setExcludeDeletedEntities(quickSearchParameters.getExcludeDeletedEntities());
+ searchParameters.setIncludeSubTypes(quickSearchParameters.getIncludeSubTypes());
+ searchParameters.setLimit(quickSearchParameters.getLimit());
+ searchParameters.setOffset(quickSearchParameters.getOffset());
+ searchParameters.setEntityFilters(quickSearchParameters.getEntityFilters());
+ searchParameters.setAttributes(quickSearchParameters.getAttributes());
+ searchParameters.setSortBy(quickSearchParameters.getSortBy());
+ searchParameters.setSortOrder(quickSearchParameters.getSortOrder());
+ searchParameters.setExcludeHeaderAttributes(quickSearchParameters.getExcludeHeaderAttributes());
+
+ return searchParameters;
+ }
+
+ public static SearchParameters createSearchParameters(RelationshipSearchParameters relationshipSearchParameters) {
+ SearchParameters searchParameters = new SearchParameters();
+
+ searchParameters.setRelationshipName(relationshipSearchParameters.getRelationshipName());
+ searchParameters.setIncludeSubTypes(relationshipSearchParameters.isIncludeSubTypes());
+ searchParameters.setLimit(relationshipSearchParameters.getLimit());
+ searchParameters.setOffset(relationshipSearchParameters.getOffset());
+ searchParameters.setRelationshipFilters(relationshipSearchParameters.getRelationshipFilters());
+ searchParameters.setSortBy(relationshipSearchParameters.getSortBy());
+ searchParameters.setSortOrder(relationshipSearchParameters.getSortOrder());
+ searchParameters.setMarker(relationshipSearchParameters.getMarker());
+
+ return searchParameters;
+ }
+
+ @VisibleForTesting
+ static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesLengthInIdxQuery) {
+ AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
+ String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
+
+ if (StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
+ return typeAndSubTypesQryStr;
+ }
+
+ return "";
+ }
+
@Override
@GraphTransaction
public AtlasSearchResult searchUsingDslQuery(String dslQuery, int limit, int offset) throws AtlasBaseException {
@@ -173,17 +217,33 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return ret;
}
+ @Override
+ public String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification) {
+ String queryStr = query == null ? "" : query;
+
+ if (StringUtils.isNotEmpty(typeName)) {
+ queryStr = escapeTypeName(typeName) + " " + queryStr;
+ }
+
+ if (StringUtils.isNotEmpty(classification)) {
+ // isa works with a type name only - like hive_column isa PII; it doesn't work with more complex query
+ if (StringUtils.isEmpty(query)) {
+ queryStr += (" isa " + classification);
+ }
+ }
+
+ return queryStr;
+ }
+
@Override
@GraphTransaction
- public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset)
- throws AtlasBaseException {
+ public AtlasSearchResult searchUsingFullTextQuery(String fullTextQuery, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(fullTextQuery, AtlasQueryType.FULL_TEXT);
QueryParams params = QueryParams.getNormalizedParams(limit, offset);
AtlasIndexQuery idxQuery = toAtlasIndexQuery(fullTextQuery);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Executing Full text query: {}", fullTextQuery);
- }
+ LOG.debug("Executing Full text query: {}", fullTextQuery);
+
ret.setFullTextResult(getIndexQueryResults(idxQuery, params, excludeDeletedEntities));
ret.setApproximateCount(idxQuery.vertexTotals());
@@ -194,15 +254,10 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
- public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, String attrName,
- String attrValuePrefix, boolean excludeDeletedEntities, int limit,
- int offset) throws AtlasBaseException {
-
+ public AtlasSearchResult searchUsingBasicQuery(String query, String typeName, String classification, String attrName, String attrValuePrefix, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.BASIC);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Executing basic search query: {} with type: {} and classification: {}", query, typeName, classification);
- }
+ LOG.debug("Executing basic search query: {} with type: {} and classification: {}", query, typeName, classification);
final QueryParams params = QueryParams.getNormalizedParams(limit, offset);
Set<String> typeNames = null;
@@ -250,7 +305,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
if (attribute == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, typeName);
}
-
} else {
// if attrName is null|empty iterate defaultAttrNames to get attribute value
final List<String> defaultAttrNames = new ArrayList<>(Arrays.asList("qualifiedName", "name"));
@@ -267,7 +321,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
// (guids cannot be searched in fulltext)
isGuidPrefixSearch = true;
query = null;
-
} else {
attrQualifiedName = attribute.getQualifiedName();
@@ -277,9 +330,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("Executing attribute search attrName: {} and attrValue: {}", attrName, attrValuePrefix);
- }
+ LOG.debug("Executing attribute search attrName: {} and attrValue: {}", attrName, attrValuePrefix);
}
// if query was provided, perform indexQuery and filter for typeName & classification in memory; this approach
@@ -294,11 +345,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
final AtlasIndexQuery qry = graph.indexQuery(Constants.FULLTEXT_INDEX, idxQuery, indexQueryOffset);
final Iterator<Result<?, ?>> qryResult = qry.vertices();
- if (LOG.isDebugEnabled()) {
- LOG.debug("indexQuery: query=" + idxQuery + "; offset=" + indexQueryOffset);
- }
+ LOG.debug("indexQuery: query={}; offset={}", idxQuery, indexQueryOffset);
- if(!qryResult.hasNext()) {
+ if (!qryResult.hasNext()) {
break;
}
@@ -398,7 +447,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
Object result = graph.executeGremlinScript(scriptEngine, bindings, basicQuery, false);
if (result instanceof List && CollectionUtils.isNotEmpty((List) result)) {
- List queryResult = (List) result;
+ List queryResult = (List) result;
Object firstElement = queryResult.get(0);
if (firstElement instanceof AtlasVertex) {
@@ -423,59 +472,19 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return ret;
}
- @Override
- @GraphTransaction
- public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
- String query = quickSearchParameters.getQuery();
- if (StringUtils.isNotEmpty(query) && !AtlasStructType.AtlasAttribute.hastokenizeChar(query)) {
- query = query + "*";
- }
- quickSearchParameters.setQuery(query);
-
- SearchContext searchContext = new SearchContext(createSearchParameters(quickSearchParameters),
- typeRegistry,
- graph,
- indexer.getVertexIndexKeys());
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("Generating the search results for the query {} .", searchContext.getSearchParameters().getQuery());
- }
-
- AtlasSearchResult searchResult = searchWithSearchContext(searchContext);
-
- if(LOG.isDebugEnabled()) {
- LOG.debug("Generating the aggregated metrics for the query {} .", searchContext.getSearchParameters().getQuery());
- }
-
- // load the facet fields and attributes.
- Set<String> aggregationFields = getAggregationFields();
- Set<AtlasAttribute> aggregationAttributes = getAggregationAtlasAttributes();
- SearchAggregator searchAggregator = new SearchAggregatorImpl(searchContext);
- Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = searchAggregator.getAggregatedMetrics(aggregationFields, aggregationAttributes);
- AtlasQuickSearchResult ret = new AtlasQuickSearchResult(searchResult, aggregatedMetrics);
-
- return ret;
- }
-
- @Override
- @GraphTransaction
- public AtlasSuggestionsResult getSuggestions(String prefixString, String fieldName) {
- return suggestionsProvider.getSuggestions(prefixString, fieldName);
- }
-
@Override
@GraphTransaction
public AtlasSearchResult searchWithParameters(SearchParameters searchParameters) throws AtlasBaseException {
String query = searchParameters.getQuery();
if (StringUtils.isNotEmpty(query)) {
-
String modifiedString = StringUtils.strip(query, "*");
if (AtlasStructType.AtlasAttribute.hastokenizeChar(modifiedString)) {
searchParameters.setQuery(modifiedString);
}
}
+
return searchWithSearchContext(new SearchContext(searchParameters, typeRegistry, graph, indexer.getVertexIndexKeys()));
}
@@ -487,413 +496,563 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
- public void createAndQueueSearchResultDownloadTask(Map<String, Object> taskParams) throws AtlasBaseException {
+ public AtlasSearchResult searchRelationsWithParameters(RelationshipSearchParameters searchParameters) throws AtlasBaseException {
+ SearchContext searchContext = new SearchContext(createSearchParameters(searchParameters), typeRegistry, graph, null);
- List<AtlasTask> pendingTasks = taskManagement.getPendingTasksByType(SearchResultDownloadTaskFactory.SEARCH_RESULT_DOWNLOAD);
- if (CollectionUtils.isNotEmpty(pendingTasks) && pendingTasks.size() > SearchResultDownloadTaskFactory.MAX_PENDING_TASKS_ALLOWED) {
- throw new AtlasBaseException(PENDING_TASKS_ALREADY_IN_PROGRESS, String.valueOf(pendingTasks.size()));
- }
- AtlasTask task = taskManagement.createTask(SearchResultDownloadTaskFactory.SEARCH_RESULT_DOWNLOAD, RequestContext.getCurrentUser(), taskParams);
- RequestContext.get().queueTask(task);
+ return searchRelationsWithSearchContext(searchContext);
}
@Override
- public AtlasSearchResultDownloadStatus getSearchResultDownloadStatus() throws IOException {
- List<AtlasTask> pendingTasks = taskManagement.getPendingTasksByType(SearchResultDownloadTaskFactory.SEARCH_RESULT_DOWNLOAD);
- List<AtlasTask> currentUserPendingTasks = pendingTasks.stream().filter(task -> task.getCreatedBy()
- .equals(RequestContext.getCurrentUser())).collect(Collectors.toList());
+ @GraphTransaction
+ public AtlasSearchResult searchRelatedEntities(String guid, String relation, boolean getApproximateCount, SearchParameters searchParameters) throws AtlasBaseException {
+ AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP);
- List<AtlasSearchDownloadRecord> searchDownloadRecords = new ArrayList<>();
- for (AtlasTask pendingTask : currentUserPendingTasks) {
- String fileName = (String) pendingTask.getParameters().get(SearchResultDownloadTask.CSV_FILE_NAME_KEY);
- AtlasSearchDownloadRecord searchDownloadRecord = new AtlasSearchDownloadRecord(pendingTask.getStatus(), fileName, pendingTask.getCreatedBy(), pendingTask.getCreatedTime(), pendingTask.getStartTime());
- searchDownloadRecords.add(searchDownloadRecord);
+ if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "guid: '" + guid + "', relation: '" + relation + "'");
}
- File fileDir = new File(SearchResultDownloadTask.DOWNLOAD_DIR_PATH, RequestContext.getCurrentUser());
- if (fileDir.exists()) {
- File[] currentUserFiles = fileDir.listFiles();
- if (currentUserFiles != null) {
- for (File file : currentUserFiles) {
- BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
- Date createdTime = new Date(attr.creationTime().toMillis());
- AtlasSearchDownloadRecord searchDownloadRecord = new AtlasSearchDownloadRecord(AtlasTask.Status.COMPLETE, file.getName(), RequestContext.getCurrentUser(), createdTime);
- searchDownloadRecords.add(searchDownloadRecord);
- }
- }
- }
- AtlasSearchResultDownloadStatus result = new AtlasSearchResultDownloadStatus();
- result.setSearchDownloadRecords(searchDownloadRecords);
- return result;
- }
+ //validate entity
+ AtlasVertex entityVertex = entityRetriever.getEntityVertex(guid);
+ String entityTypeName = GraphHelper.getTypeName(entityVertex);
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName);
- @Override
- @GraphTransaction
- public AtlasTask createAndQueueAuditReductionTask(Map<String, Object> taskParams, String taskType) throws AtlasBaseException {
- List<AtlasTask> pendingTasks = taskManagement.getPendingTasksByType(taskType);
- if (CollectionUtils.isNotEmpty(pendingTasks) && pendingTasks.size() > AuditReductionTaskFactory.MAX_PENDING_TASKS_ALLOWED) {
- throw new AtlasBaseException(PENDING_TASKS_ALREADY_IN_PROGRESS, String.valueOf(pendingTasks.size()));
+ if (entityType == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_TYPE, entityTypeName, guid);
}
- AtlasTask task = taskManagement.createTask(taskType, RequestContext.getCurrentUser(), taskParams);
- RequestContext.get().queueTask(task);
-
- return task;
- }
-
- @Override
- @GraphTransaction
- public AtlasSearchResult searchRelationsWithParameters(RelationshipSearchParameters searchParameters) throws AtlasBaseException {
- SearchContext searchContext = new SearchContext(createSearchParameters(searchParameters),
- typeRegistry,
- graph,
- null);
- return searchRelationsWithSearchContext(searchContext);
- }
-
- private AtlasSearchResult searchRelationsWithSearchContext(SearchContext searchContext) throws AtlasBaseException {
- SearchParameters searchParameters = searchContext.getSearchParameters();
- AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
- final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(),searchParameters.getOffset());
- searchParameters.setLimit(params.limit());
- searchParameters.setOffset(params.offset());
+ //validate relation
+ AtlasEntityType endEntityType = null;
+ AtlasAttribute attribute = entityType.getAttribute(relation);
- if (!searchContext.needRelationshipProcessor()) {
- return ret;
+ if (attribute == null) {
+ attribute = entityType.getRelationshipAttribute(relation, null);
}
- RelationshipSearchProcessor rsp = new RelationshipSearchProcessor(searchContext, indexer.getEdgeIndexKeys());
- List<AtlasEdge> edges = rsp.executeEdges();
+ if (attribute != null) {
+ //get end entity type through relationship attribute
+ endEntityType = attribute.getReferencedEntityType(typeRegistry);
+
+ if (endEntityType == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_ATTRIBUTE, relation, attribute.getTypeName());
+ }
- ret.setApproximateCount(rsp.getResultCount());
+ relation = attribute.getRelationshipEdgeLabel();
+ } else {
+ //get end entity type through label
+ String endEntityTypeName = GraphHelper.getReferencedEntityTypeName(entityVertex, relation);
- String nextMarker = rsp.getNextMarker();
- if (StringUtils.isNotEmpty(nextMarker)) {
- ret.setNextMarker(nextMarker);
+ if (StringUtils.isNotEmpty(endEntityTypeName)) {
+ endEntityType = typeRegistry.getEntityTypeByName(endEntityTypeName);
+ }
}
- for (AtlasEdge edge : edges) {
- AtlasRelationshipHeader relation = entityRetriever.mapEdgeToAtlasRelationshipHeader(edge);
- ret.addRelation(relation);
+ //validate sortBy attribute
+ String sortBy = searchParameters.getSortBy();
+ SortOrder sortOrder = searchParameters.getSortOrder();
+ int offset = searchParameters.getOffset();
+ int limit = searchParameters.getLimit();
+ String sortByAttributeName = DEFAULT_SORT_ATTRIBUTE_NAME;
+
+ if (StringUtils.isNotEmpty(sortBy)) {
+ sortByAttributeName = sortBy;
}
- return ret;
- }
+ if (endEntityType != null) {
+ AtlasAttribute sortByAttribute = endEntityType.getAttribute(sortByAttributeName);
- public Set<String> searchEntityGUIDs(AtlasAuditAgingType auditAgingType, Set<String> entityTypes, SearchContext searchContext) throws AtlasBaseException {
- SearchParameters searchParameters = searchContext.getSearchParameters();
- final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(),searchParameters.getOffset());
- String searchID = searchTracker.add(searchContext); // For future cancellations
+ if (sortByAttribute == null) {
+ sortByAttributeName = null;
+ sortOrder = null;
- searchParameters.setLimit(params.limit());
- searchParameters.setOffset(params.offset());
+ if (StringUtils.isNotEmpty(sortBy)) {
+ LOG.info("Invalid sortBy Attribute {} for entityType {}, Ignoring Sorting", sortBy, endEntityType.getTypeName());
+ } else {
+ LOG.info("Invalid Default sortBy Attribute {} for entityType {}, Ignoring Sorting", DEFAULT_SORT_ATTRIBUTE_NAME, endEntityType.getTypeName());
+ }
+ } else {
+ sortByAttributeName = sortByAttribute.getVertexPropertyName();
- Set<String> guids = new HashSet<>();
- try {
- List<AtlasVertex> resultList = searchContext.getSearchProcessor().execute();
- do {
- for (AtlasVertex atlasVertex : resultList) {
- if (atlasVertex != null && checkVertexMatchesSearchCriteria(atlasVertex, auditAgingType, entityTypes)) {
- guids.add(atlasVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class));
- }
+ if (sortOrder == null) {
+ sortOrder = ASCENDING;
}
- searchParameters.setOffset(searchParameters.getOffset() + searchParameters.getLimit());
- resultList = searchContext.getSearchProcessor().execute();
- } while (CollectionUtils.isNotEmpty(resultList));
- LOG.info("Total {} entities are eligible for Audit aging", guids.size());
- } catch (Throwable t) {
- LOG.error("Error while retrieving eligible entities for audit aging");
- } finally {
- searchTracker.remove(searchID);
+ }
+ } else {
+ sortOrder = null;
+
+ LOG.info("Invalid sortBy Attribute {}, Ignoring Sorting", sortBy);
}
- return guids;
- }
+ //get relationship(end vertices) vertices
+ GraphTraversal gt = graph.V(entityVertex.getId()).bothE(relation).otherV();
- private boolean checkVertexMatchesSearchCriteria(AtlasVertex vertex, AtlasAuditAgingType auditAgingType, Set<String> entityTypes) {
- if (CollectionUtils.isEmpty(entityTypes)) {
- return true;
+ if (searchParameters.getExcludeDeletedEntities()) {
+ gt.has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
}
- String typeName = vertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
- boolean typeNameMatchesCriteria = entityTypes.contains(typeName);
- return (auditAgingType == AtlasAuditAgingType.DEFAULT) ? !typeNameMatchesCriteria : typeNameMatchesCriteria;
- }
-
- private AtlasSearchResult searchWithSearchContext(SearchContext searchContext) throws AtlasBaseException {
- SearchParameters searchParameters = searchContext.getSearchParameters();
- AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
- final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(),searchParameters.getOffset());
- String searchID = searchTracker.add(searchContext); // For future cancellations
- searchParameters.setLimit(params.limit());
- searchParameters.setOffset(params.offset());
+ if (sortOrder != null) {
+ if (sortOrder == ASCENDING) {
+ gt.order().by(sortByAttributeName, Order.asc);
+ } else {
+ gt.order().by(sortByAttributeName, Order.desc);
+ }
+ }
- try {
- List<AtlasVertex> resultList = searchContext.getSearchProcessor().execute();
+ gt.range(offset, offset + limit);
- ret.setApproximateCount(searchContext.getSearchProcessor().getResultCount());
+ List<AtlasEntityHeader> resultList = new ArrayList<>();
- String nextMarker = searchContext.getSearchProcessor().getNextMarker();
- if (StringUtils.isNotEmpty(nextMarker)) {
- ret.setNextMarker(nextMarker);
+ while (gt.hasNext()) {
+ Vertex v = (Vertex) gt.next();
+
+ if (v != null && v.property(Constants.GUID_PROPERTY_KEY).isPresent()) {
+ String endVertexGuid = v.property(Constants.GUID_PROPERTY_KEY).value().toString();
+ AtlasVertex vertex = entityRetriever.getEntityVertex(endVertexGuid);
+ AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(vertex, searchParameters.getAttributes());
+
+ if (searchParameters.getIncludeClassificationAttributes()) {
+ entity.setClassifications(entityRetriever.getAllClassifications(vertex));
+ }
+
+ resultList.add(entity);
}
+ }
- //If excludeHeaderAttributes is true, only primitive attributes requested in 'attributes' field will be sent in the response
- Set<String> attributes = searchParameters.getAttributes();
- if (searchContext.excludeHeaderAttributes()) {
+ ret.setEntities(resultList);
- AtlasSearchResult.AttributeSearchResult attributeSearchResult = new AtlasSearchResult.AttributeSearchResult();
- attributeSearchResult.setName(new ArrayList<>(attributes));
+ if (ret.getEntities() == null) {
+ ret.setEntities(new ArrayList<>());
+ }
- Collection<List<Object>> values = new ArrayList<>();
- for (AtlasVertex vertex : resultList) {
- List<Object> row = new ArrayList<>();
+ //set approximate count
+ //state of the edge and endVertex will be same
+ if (getApproximateCount) {
+ Iterator<AtlasEdge> edges = GraphHelper.getAdjacentEdgesByLabel(entityVertex, AtlasEdgeDirection.BOTH, relation);
- for (String attrName : attributes) {
- AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
- AtlasAttribute attribute = entityType.getAttribute(attrName);
- Object value = vertex.getProperty(attribute.getVertexPropertyName(), Object.class);
+ if (searchParameters.getExcludeDeletedEntities()) {
+ List<AtlasEdge> edgeList = new ArrayList<>();
- row.add(value != null ? value : StringUtils.EMPTY);
- }
- values.add(row);
- }
- attributeSearchResult.setValues(new ArrayList<>(values));
+ edges.forEachRemaining(edgeList::add);
- ret.setAttributes(attributeSearchResult);
- return ret;
+ Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator().generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
+
+ CollectionUtils.filter(edgeList, activePredicate);
+
+ ret.setApproximateCount(edgeList.size());
+ } else {
+ ret.setApproximateCount(IteratorUtils.size(edges));
}
+ }
- // By default any attribute that shows up in the search parameter should be sent back in the response
- // If additional values are requested then the entityAttributes will be a superset of the all search attributes
- // and the explicitly requested attribute(s)
- Set<String> resultAttributes = new HashSet<>();
- Set<String> entityAttributes = new HashSet<>();
+ scrubSearchResults(ret);
- if (CollectionUtils.isNotEmpty(searchParameters.getAttributes())) {
- resultAttributes.addAll(searchParameters.getAttributes());
+ return ret;
+ }
+
+ @Override
+ public AtlasUserSavedSearch addSavedSearch(String currentUser, AtlasUserSavedSearch savedSearch) throws AtlasBaseException {
+ try {
+ if (StringUtils.isEmpty(savedSearch.getOwnerName())) {
+ savedSearch.setOwnerName(currentUser);
}
- if (CollectionUtils.isNotEmpty(searchContext.getEntityAttributes())) {
- resultAttributes.addAll(searchContext.getEntityAttributes());
+ checkSavedSearchOwnership(currentUser, savedSearch);
+
+ return userProfileService.addSavedSearch(savedSearch);
+ } catch (AtlasBaseException e) {
+ LOG.error("addSavedSearch({})", savedSearch, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public AtlasUserSavedSearch updateSavedSearch(String currentUser, AtlasUserSavedSearch savedSearch) throws AtlasBaseException {
+ try {
+ if (StringUtils.isEmpty(savedSearch.getOwnerName())) {
+ savedSearch.setOwnerName(currentUser);
}
- if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
+ checkSavedSearchOwnership(currentUser, savedSearch);
- AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
+ return userProfileService.updateSavedSearch(savedSearch);
+ } catch (AtlasBaseException e) {
+ LOG.error("updateSavedSearch({})", savedSearch, e);
+ throw e;
+ }
+ }
- for (String resultAttribute : resultAttributes) {
- AtlasAttribute attribute = entityType.getAttribute(resultAttribute);
+ @Override
+ public List<AtlasUserSavedSearch> getSavedSearches(String currentUser, String userName) throws AtlasBaseException {
+ try {
+ if (StringUtils.isEmpty(userName)) {
+ userName = currentUser;
+ } else if (!StringUtils.equals(currentUser, userName)) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "invalid data");
+ }
- if (attribute == null) {
- attribute = entityType.getRelationshipAttribute(resultAttribute, null);
- }
+ return userProfileService.getSavedSearches(userName);
+ } catch (AtlasBaseException e) {
+ LOG.error("getSavedSearches({})", userName, e);
+ throw e;
+ }
+ }
- if (attribute != null) {
- AtlasType attributeType = attribute.getAttributeType();
+ @Override
+ public AtlasUserSavedSearch getSavedSearchByGuid(String currentUser, String guid) throws AtlasBaseException {
+ try {
+ AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(guid);
- if (attributeType instanceof AtlasArrayType) {
- attributeType = ((AtlasArrayType) attributeType).getElementType();
- }
+ checkSavedSearchOwnership(currentUser, savedSearch);
- if (attributeType instanceof AtlasEntityType || attributeType instanceof AtlasObjectIdType) {
- entityAttributes.add(resultAttribute);
- }
- }
- }
+ return savedSearch;
+ } catch (AtlasBaseException e) {
+ LOG.error("getSavedSearchByGuid({})", guid, e);
+ throw e;
+ }
+ }
+
+ @Override
+ public AtlasUserSavedSearch getSavedSearchByName(String currentUser, String userName, String searchName) throws AtlasBaseException {
+ try {
+ if (StringUtils.isEmpty(userName)) {
+ userName = currentUser;
+ } else if (!StringUtils.equals(currentUser, userName)) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "invalid data");
}
- for (AtlasVertex atlasVertex : resultList) {
- AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, resultAttributes);
+ return userProfileService.getSavedSearch(userName, searchName);
+ } catch (AtlasBaseException e) {
+ LOG.error("getSavedSearchByName({}, {})", userName, searchName, e);
+ throw e;
+ }
+ }
- if(searchParameters.getIncludeClassificationAttributes()) {
- entity.setClassifications(entityRetriever.getAllClassifications(atlasVertex));
- }
+ @Override
+ public void deleteSavedSearch(String currentUser, String guid) throws AtlasBaseException {
+ try {
+ AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(guid);
- ret.addEntity(entity);
+ checkSavedSearchOwnership(currentUser, savedSearch);
- // populate ret.referredEntities
- for (String entityAttribute : entityAttributes) {
- Object attrValue = entity.getAttribute(entityAttribute);
+ userProfileService.deleteSavedSearch(guid);
+ } catch (AtlasBaseException e) {
+ LOG.error("deleteSavedSearch({})", guid, e);
+ throw e;
+ }
+ }
- if (attrValue instanceof AtlasObjectId) {
- AtlasObjectId objId = (AtlasObjectId) attrValue;
+ @Override
+ @GraphTransaction
+ public AtlasQuickSearchResult quickSearch(QuickSearchParameters quickSearchParameters) throws AtlasBaseException {
+ String query = quickSearchParameters.getQuery();
- if (ret.getReferredEntities() == null) {
- ret.setReferredEntities(new HashMap<>());
- }
+ if (StringUtils.isNotEmpty(query) && !AtlasStructType.AtlasAttribute.hastokenizeChar(query)) {
+ query = query + "*";
+ }
- if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
- ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
- }
- } else if (attrValue instanceof Collection) {
- Collection objIds = (Collection) attrValue;
+ quickSearchParameters.setQuery(query);
- for (Object obj : objIds) {
- if (obj instanceof AtlasObjectId) {
- AtlasObjectId objId = (AtlasObjectId) obj;
+ SearchContext searchContext = new SearchContext(createSearchParameters(quickSearchParameters), typeRegistry, graph, indexer.getVertexIndexKeys());
- if (ret.getReferredEntities() == null) {
- ret.setReferredEntities(new HashMap<>());
- }
+ LOG.debug("Generating the search results for the query {}", searchContext.getSearchParameters().getQuery());
- if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
- ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
- }
- }
- }
- }
+ AtlasSearchResult searchResult = searchWithSearchContext(searchContext);
+
+ LOG.debug("Generating the aggregated metrics for the query {}", searchContext.getSearchParameters().getQuery());
+
+ // load the facet fields and attributes.
+ Set<String> aggregationFields = getAggregationFields();
+ Set<AtlasAttribute> aggregationAttributes = getAggregationAtlasAttributes();
+ SearchAggregator searchAggregator = new SearchAggregatorImpl(searchContext);
+ Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = searchAggregator.getAggregatedMetrics(aggregationFields, aggregationAttributes);
+
+ return new AtlasQuickSearchResult(searchResult, aggregatedMetrics);
+ }
+
+ @Override
+ @GraphTransaction
+ public AtlasSuggestionsResult getSuggestions(String prefixString, String fieldName) {
+ return suggestionsProvider.getSuggestions(prefixString, fieldName);
+ }
+
+ @Override
+ @GraphTransaction
+ public void createAndQueueSearchResultDownloadTask(Map<String, Object> taskParams) throws AtlasBaseException {
+ List<AtlasTask> pendingTasks = taskManagement.getPendingTasksByType(SearchResultDownloadTaskFactory.SEARCH_RESULT_DOWNLOAD);
+
+ if (CollectionUtils.isNotEmpty(pendingTasks) && pendingTasks.size() > SearchResultDownloadTaskFactory.MAX_PENDING_TASKS_ALLOWED) {
+ throw new AtlasBaseException(PENDING_TASKS_ALREADY_IN_PROGRESS, String.valueOf(pendingTasks.size()));
+ }
+
+ AtlasTask task = taskManagement.createTask(SearchResultDownloadTaskFactory.SEARCH_RESULT_DOWNLOAD, RequestContext.getCurrentUser(), taskParams);
+
+ RequestContext.get().queueTask(task);
+ }
+
+ @Override
+ public AtlasSearchResultDownloadStatus getSearchResultDownloadStatus() throws IOException {
+ List<AtlasTask> pendingTasks = taskManagement.getPendingTasksByType(SearchResultDownloadTaskFactory.SEARCH_RESULT_DOWNLOAD);
+ List<AtlasTask> currentUserPendingTasks = pendingTasks.stream().filter(task -> task.getCreatedBy().equals(RequestContext.getCurrentUser())).collect(Collectors.toList());
+ List<AtlasSearchDownloadRecord> searchDownloadRecords = new ArrayList<>();
+
+ for (AtlasTask pendingTask : currentUserPendingTasks) {
+ String fileName = (String) pendingTask.getParameters().get(SearchResultDownloadTask.CSV_FILE_NAME_KEY);
+ AtlasSearchDownloadRecord searchDownloadRecord = new AtlasSearchDownloadRecord(pendingTask.getStatus(), fileName, pendingTask.getCreatedBy(), pendingTask.getCreatedTime(), pendingTask.getStartTime());
+
+ searchDownloadRecords.add(searchDownloadRecord);
+ }
+
+ File fileDir = new File(SearchResultDownloadTask.DOWNLOAD_DIR_PATH, RequestContext.getCurrentUser());
+
+ if (fileDir.exists()) {
+ File[] currentUserFiles = fileDir.listFiles();
+
+ if (currentUserFiles != null) {
+ for (File file : currentUserFiles) {
+ BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class);
+ Date createdTime = new Date(attr.creationTime().toMillis());
+ AtlasSearchDownloadRecord searchDownloadRecord = new AtlasSearchDownloadRecord(AtlasTask.Status.COMPLETE, file.getName(), RequestContext.getCurrentUser(), createdTime);
+
+ searchDownloadRecords.add(searchDownloadRecord);
}
}
- } finally {
- searchTracker.remove(searchID);
}
- scrubSearchResults(ret);
+ AtlasSearchResultDownloadStatus result = new AtlasSearchResultDownloadStatus();
- return ret;
+ result.setSearchDownloadRecords(searchDownloadRecords);
+
+ return result;
}
@Override
@GraphTransaction
- public AtlasSearchResult searchRelatedEntities(String guid, String relation, boolean getApproximateCount, SearchParameters searchParameters) throws AtlasBaseException {
- AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP);
+ public AtlasTask createAndQueueAuditReductionTask(Map<String, Object> taskParams, String taskType) throws AtlasBaseException {
+ List<AtlasTask> pendingTasks = taskManagement.getPendingTasksByType(taskType);
- if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "guid: '" + guid + "', relation: '" + relation + "'");
+ if (CollectionUtils.isNotEmpty(pendingTasks) && pendingTasks.size() > AuditReductionTaskFactory.MAX_PENDING_TASKS_ALLOWED) {
+ throw new AtlasBaseException(PENDING_TASKS_ALREADY_IN_PROGRESS, String.valueOf(pendingTasks.size()));
}
- //validate entity
- AtlasVertex entityVertex = entityRetriever.getEntityVertex(guid);
- String entityTypeName = GraphHelper.getTypeName(entityVertex);
- AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entityTypeName);
+ AtlasTask task = taskManagement.createTask(taskType, RequestContext.getCurrentUser(), taskParams);
- if (entityType == null) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_TYPE, entityTypeName, guid);
+ RequestContext.get().queueTask(task);
+
+ return task;
+ }
+
+ public Set<String> searchEntityGUIDs(AtlasAuditAgingType auditAgingType, Set<String> entityTypes, SearchContext searchContext) throws AtlasBaseException {
+ SearchParameters searchParameters = searchContext.getSearchParameters();
+ final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(), searchParameters.getOffset());
+ String searchID = searchTracker.add(searchContext); // For future cancellations
+ Set<String> guids = new HashSet<>();
+
+ searchParameters.setLimit(params.limit());
+ searchParameters.setOffset(params.offset());
+
+ try {
+ List<AtlasVertex> resultList = searchContext.getSearchProcessor().execute();
+
+ do {
+ for (AtlasVertex atlasVertex : resultList) {
+ if (atlasVertex != null && checkVertexMatchesSearchCriteria(atlasVertex, auditAgingType, entityTypes)) {
+ guids.add(atlasVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class));
+ }
+ }
+
+ searchParameters.setOffset(searchParameters.getOffset() + searchParameters.getLimit());
+
+ resultList = searchContext.getSearchProcessor().execute();
+ } while (CollectionUtils.isNotEmpty(resultList));
+
+ LOG.info("Total {} entities are eligible for Audit aging", guids.size());
+ } catch (Throwable t) {
+ LOG.error("Error while retrieving eligible entities for audit aging");
+ } finally {
+ searchTracker.remove(searchID);
}
- //validate relation
- AtlasEntityType endEntityType = null;
- AtlasAttribute attribute = entityType.getAttribute(relation);
+ return guids;
+ }
- if (attribute == null) {
- attribute = entityType.getRelationshipAttribute(relation, null);
+ public int getMaxResultSetSize() {
+ return maxResultSetSize;
+ }
+
+ private AtlasSearchResult searchRelationsWithSearchContext(SearchContext searchContext) throws AtlasBaseException {
+ SearchParameters searchParameters = searchContext.getSearchParameters();
+ AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
+ final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(), searchParameters.getOffset());
+
+ searchParameters.setLimit(params.limit());
+ searchParameters.setOffset(params.offset());
+
+ if (!searchContext.needRelationshipProcessor()) {
+ return ret;
}
- if (attribute != null) {
- //get end entity type through relationship attribute
- endEntityType = attribute.getReferencedEntityType(typeRegistry);
+ RelationshipSearchProcessor rsp = new RelationshipSearchProcessor(searchContext, indexer.getEdgeIndexKeys());
+ List<AtlasEdge> edges = rsp.executeEdges();
- if (endEntityType == null) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_RELATIONSHIP_ATTRIBUTE, relation, attribute.getTypeName());
- }
- relation = attribute.getRelationshipEdgeLabel();
- } else {
- //get end entity type through label
- String endEntityTypeName = GraphHelper.getReferencedEntityTypeName(entityVertex, relation);
+ ret.setApproximateCount(rsp.getResultCount());
- if (StringUtils.isNotEmpty(endEntityTypeName)) {
- endEntityType = typeRegistry.getEntityTypeByName(endEntityTypeName);
- }
+ String nextMarker = rsp.getNextMarker();
+
+ if (StringUtils.isNotEmpty(nextMarker)) {
+ ret.setNextMarker(nextMarker);
}
- //validate sortBy attribute
- String sortBy = searchParameters.getSortBy();
- SortOrder sortOrder = searchParameters.getSortOrder();
- int offset = searchParameters.getOffset();
- int limit = searchParameters.getLimit();
- String sortByAttributeName = DEFAULT_SORT_ATTRIBUTE_NAME;
+ for (AtlasEdge edge : edges) {
+ AtlasRelationshipHeader relation = entityRetriever.mapEdgeToAtlasRelationshipHeader(edge);
- if (StringUtils.isNotEmpty(sortBy)) {
- sortByAttributeName = sortBy;
+ ret.addRelation(relation);
}
- if (endEntityType != null) {
- AtlasAttribute sortByAttribute = endEntityType.getAttribute(sortByAttributeName);
+ return ret;
+ }
- if (sortByAttribute == null) {
- sortByAttributeName = null;
- sortOrder = null;
+ private boolean checkVertexMatchesSearchCriteria(AtlasVertex vertex, AtlasAuditAgingType auditAgingType, Set<String> entityTypes) {
+ if (CollectionUtils.isEmpty(entityTypes)) {
+ return true;
+ }
- if (StringUtils.isNotEmpty(sortBy)) {
- LOG.info("Invalid sortBy Attribute {} for entityType {}, Ignoring Sorting", sortBy, endEntityType.getTypeName());
- } else {
- LOG.info("Invalid Default sortBy Attribute {} for entityType {}, Ignoring Sorting", DEFAULT_SORT_ATTRIBUTE_NAME, endEntityType.getTypeName());
+ String typeName = vertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
+ boolean typeNameMatchesCriteria = entityTypes.contains(typeName);
+
+ return (auditAgingType == AtlasAuditAgingType.DEFAULT) ? !typeNameMatchesCriteria : typeNameMatchesCriteria;
+ }
+
+ private AtlasSearchResult searchWithSearchContext(SearchContext searchContext) throws AtlasBaseException {
+ SearchParameters searchParameters = searchContext.getSearchParameters();
+ AtlasSearchResult ret = new AtlasSearchResult(searchParameters);
+ final QueryParams params = QueryParams.getNormalizedParams(searchParameters.getLimit(), searchParameters.getOffset());
+ String searchID = searchTracker.add(searchContext); // For future cancellations
+
+ searchParameters.setLimit(params.limit());
+ searchParameters.setOffset(params.offset());
+
+ try {
+ List<AtlasVertex> resultList = searchContext.getSearchProcessor().execute();
+
+ ret.setApproximateCount(searchContext.getSearchProcessor().getResultCount());
+
+ String nextMarker = searchContext.getSearchProcessor().getNextMarker();
+
+ if (StringUtils.isNotEmpty(nextMarker)) {
+ ret.setNextMarker(nextMarker);
+ }
+
+ //If excludeHeaderAttributes is true, only primitive attributes requested in 'attributes' field will be sent in the response
+ Set<String> attributes = searchParameters.getAttributes();
+
+ if (searchContext.excludeHeaderAttributes()) {
+ Collection<List<Object>> values = new ArrayList<>();
+ AtlasSearchResult.AttributeSearchResult attributeSearchResult = new AtlasSearchResult.AttributeSearchResult();
+
+ attributeSearchResult.setName(new ArrayList<>(attributes));
+
+ for (AtlasVertex vertex : resultList) {
+ List<Object> row = new ArrayList<>();
+
+ for (String attrName : attributes) {
+ AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
+ AtlasAttribute attribute = entityType.getAttribute(attrName);
+ Object value = vertex.getProperty(attribute.getVertexPropertyName(), Object.class);
+
+ row.add(value != null ? value : StringUtils.EMPTY);
+ }
+
+ values.add(row);
}
- } else {
- sortByAttributeName = sortByAttribute.getVertexPropertyName();
+ attributeSearchResult.setValues(new ArrayList<>(values));
+
+ ret.setAttributes(attributeSearchResult);
+
+ return ret;
+ }
+
+ // By default any attribute that shows up in the search parameter should be sent back in the response
+ // If additional values are requested then the entityAttributes will be a superset of the all search attributes
+ // and the explicitly requested attribute(s)
+ Set<String> resultAttributes = new HashSet<>();
+ Set<String> entityAttributes = new HashSet<>();
- if (sortOrder == null) {
- sortOrder = ASCENDING;
- }
+ if (CollectionUtils.isNotEmpty(searchParameters.getAttributes())) {
+ resultAttributes.addAll(searchParameters.getAttributes());
}
- } else {
- sortOrder = null;
- if (StringUtils.isNotEmpty(sortBy)) {
- LOG.info("Invalid sortBy Attribute {}, Ignoring Sorting", sortBy);
+ if (CollectionUtils.isNotEmpty(searchContext.getEntityAttributes())) {
+ resultAttributes.addAll(searchContext.getEntityAttributes());
}
- }
- //get relationship(end vertices) vertices
- GraphTraversal gt = graph.V(entityVertex.getId()).bothE(relation).otherV();
+ if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
+ AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
- if (searchParameters.getExcludeDeletedEntities()) {
- gt.has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
- }
+ for (String resultAttribute : resultAttributes) {
+ AtlasAttribute attribute = entityType.getAttribute(resultAttribute);
- if (sortOrder != null) {
- if (sortOrder == ASCENDING) {
- gt.order().by(sortByAttributeName, Order.asc);
- } else {
- gt.order().by(sortByAttributeName, Order.desc);
- }
- }
+ if (attribute == null) {
+ attribute = entityType.getRelationshipAttribute(resultAttribute, null);
+ }
- gt.range(offset, offset + limit);
+ if (attribute != null) {
+ AtlasType attributeType = attribute.getAttributeType();
- List<AtlasEntityHeader> resultList = new ArrayList<>();
- while (gt.hasNext()) {
- Vertex v = (Vertex) gt.next();
+ if (attributeType instanceof AtlasArrayType) {
+ attributeType = ((AtlasArrayType) attributeType).getElementType();
+ }
- if (v != null && v.property(Constants.GUID_PROPERTY_KEY).isPresent()) {
- String endVertexGuid = v.property(Constants.GUID_PROPERTY_KEY).value().toString();
- AtlasVertex vertex = entityRetriever.getEntityVertex(endVertexGuid);
- AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(vertex, searchParameters.getAttributes());
+ if (attributeType instanceof AtlasEntityType || attributeType instanceof AtlasObjectIdType) {
+ entityAttributes.add(resultAttribute);
+ }
+ }
+ }
+ }
+
+ for (AtlasVertex atlasVertex : resultList) {
+ AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(atlasVertex, resultAttributes);
if (searchParameters.getIncludeClassificationAttributes()) {
- entity.setClassifications(entityRetriever.getAllClassifications(vertex));
+ entity.setClassifications(entityRetriever.getAllClassifications(atlasVertex));
}
- resultList.add(entity);
- }
- }
- ret.setEntities(resultList);
+ ret.addEntity(entity);
- if (ret.getEntities() == null) {
- ret.setEntities(new ArrayList<>());
- }
+ // populate ret.referredEntities
+ for (String entityAttribute : entityAttributes) {
+ Object attrValue = entity.getAttribute(entityAttribute);
- //set approximate count
- //state of the edge and endVertex will be same
- if (getApproximateCount) {
- Iterator<AtlasEdge> edges = GraphHelper.getAdjacentEdgesByLabel(entityVertex, AtlasEdgeDirection.BOTH, relation);
+ if (attrValue instanceof AtlasObjectId) {
+ AtlasObjectId objId = (AtlasObjectId) attrValue;
- if (searchParameters.getExcludeDeletedEntities()) {
- List<AtlasEdge> edgeList = new ArrayList<>();
- edges.forEachRemaining(edgeList::add);
+ if (ret.getReferredEntities() == null) {
+ ret.setReferredEntities(new HashMap<>());
+ }
- Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator().generatePredicate
- (Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
+ if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
+ ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ }
+ } else if (attrValue instanceof Collection) {
+ Collection objIds = (Collection) attrValue;
- CollectionUtils.filter(edgeList, activePredicate);
- ret.setApproximateCount(edgeList.size());
+ for (Object obj : objIds) {
+ if (obj instanceof AtlasObjectId) {
+ AtlasObjectId objId = (AtlasObjectId) obj;
- } else {
- ret.setApproximateCount(IteratorUtils.size(edges));
+ if (ret.getReferredEntities() == null) {
+ ret.setReferredEntities(new HashMap<>());
+ }
+ if (!ret.getReferredEntities().containsKey(objId.getGuid())) {
+ ret.getReferredEntities().put(objId.getGuid(), entityRetriever.toAtlasEntityHeader(objId.getGuid()));
+ }
+ }
+ }
+ }
+ }
}
+ } finally {
+ searchTracker.remove(searchID);
}
scrubSearchResults(ret);
@@ -901,21 +1060,16 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return ret;
}
- public int getMaxResultSetSize() {
- return maxResultSetSize;
- }
-
private String getQueryForFullTextSearch(String userKeyedString, String typeName, String classification) {
- String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery);
- String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsLengthInIdxQuery);
+ String typeFilter = getTypeFilter(typeRegistry, typeName, maxTypesLengthInIdxQuery);
+ String classificationFilter = getClassificationFilter(typeRegistry, classification, maxTagsLengthInIdxQuery);
+ StringBuilder queryText = new StringBuilder();
- StringBuilder queryText = new StringBuilder();
-
- if (! StringUtils.isEmpty(userKeyedString)) {
+ if (!StringUtils.isEmpty(userKeyedString)) {
queryText.append(userKeyedString);
}
- if (! StringUtils.isEmpty(typeFilter)) {
+ if (!StringUtils.isEmpty(typeFilter)) {
if (queryText.length() > 0) {
queryText.append(" AND ");
}
@@ -923,7 +1077,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
queryText.append(typeFilter);
}
- if (! StringUtils.isEmpty(classificationFilter)) {
+ if (!StringUtils.isEmpty(classificationFilter)) {
if (queryText.length() > 0) {
queryText.append(" AND ");
}
@@ -931,7 +1085,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
queryText.append(classificationFilter);
}
- return String.format(indexSearchPrefix + "\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText.toString());
+ return String.format(indexSearchPrefix + "\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, queryText);
}
private List<AtlasFullTextResult> getIndexQueryResults(AtlasIndexQuery query, QueryParams params, boolean excludeDeletedEntities) throws AtlasBaseException {
@@ -950,7 +1104,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
if (guid != null) {
AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(vertex);
- Double score = idxQueryResult.getScore();
+ Double score = idxQueryResult.getScore();
+
ret.add(new AtlasFullTextResult(entity, score));
}
}
@@ -960,6 +1115,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
private AtlasIndexQuery toAtlasIndexQuery(String fullTextQuery) {
String graphQuery = String.format(indexSearchPrefix + "\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, fullTextQuery);
+
return graph.indexQuery(Constants.FULLTEXT_INDEX, graphQuery);
}
@@ -971,19 +1127,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
AtlasClassificationType type = typeRegistry.getClassificationTypeByName(classificationName);
String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
- if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
- return typeAndSubTypesQryStr;
- }
-
- return "";
- }
-
- @VisibleForTesting
- static String getTypeFilter(AtlasTypeRegistry typeRegistry, String typeName, int maxTypesLengthInIdxQuery) {
- AtlasEntityType type = typeRegistry.getEntityTypeByName(typeName);
- String typeAndSubTypesQryStr = type != null ? type.getTypeAndAllSubTypesQryStr() : null;
-
- if(StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
+ if (StringUtils.isNotEmpty(typeAndSubTypesQryStr) && typeAndSubTypesQryStr.length() <= maxTypesLengthInIdxQuery) {
return typeAndSubTypesQryStr;
}
@@ -994,150 +1138,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return new HashSet<>(Arrays.asList(ACTIVE.toString(), DELETED.toString()));
}
-
- @Override
- public AtlasUserSavedSearch addSavedSearch(String currentUser, AtlasUserSavedSearch savedSearch) throws AtlasBaseException {
- try {
- if (StringUtils.isEmpty(savedSearch.getOwnerName())) {
- savedSearch.setOwnerName(currentUser);
- }
-
- checkSavedSearchOwnership(currentUser, savedSearch);
-
- return userProfileService.addSavedSearch(savedSearch);
- } catch (AtlasBaseException e) {
- LOG.error("addSavedSearch({})", savedSearch, e);
- throw e;
- }
- }
-
-
- @Override
- public AtlasUserSavedSearch updateSavedSearch(String currentUser, AtlasUserSavedSearch savedSearch) throws AtlasBaseException {
- try {
- if (StringUtils.isEmpty(savedSearch.getOwnerName())) {
- savedSearch.setOwnerName(currentUser);
- }
-
- checkSavedSearchOwnership(currentUser, savedSearch);
-
- return userProfileService.updateSavedSearch(savedSearch);
- } catch (AtlasBaseException e) {
- LOG.error("updateSavedSearch({})", savedSearch, e);
- throw e;
- }
- }
-
- @Override
- public List<AtlasUserSavedSearch> getSavedSearches(String currentUser, String userName) throws AtlasBaseException {
- try {
- if (StringUtils.isEmpty(userName)) {
- userName = currentUser;
- } else if (!StringUtils.equals(currentUser, userName)) {
- throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "invalid data");
- }
-
- return userProfileService.getSavedSearches(userName);
- } catch (AtlasBaseException e) {
- LOG.error("getSavedSearches({})", userName, e);
- throw e;
- }
- }
-
- @Override
- public AtlasUserSavedSearch getSavedSearchByGuid(String currentUser, String guid) throws AtlasBaseException {
- try {
- AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(guid);
-
- checkSavedSearchOwnership(currentUser, savedSearch);
-
- return savedSearch;
- } catch (AtlasBaseException e) {
- LOG.error("getSavedSearchByGuid({})", guid, e);
- throw e;
- }
- }
-
- @Override
- public AtlasUserSavedSearch getSavedSearchByName(String currentUser, String userName, String searchName) throws AtlasBaseException {
- try {
- if (StringUtils.isEmpty(userName)) {
- userName = currentUser;
- } else if (!StringUtils.equals(currentUser, userName)) {
- throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "invalid data");
- }
-
- return userProfileService.getSavedSearch(userName, searchName);
- } catch (AtlasBaseException e) {
- LOG.error("getSavedSearchByName({}, {})", userName, searchName, e);
- throw e;
- }
- }
-
- @Override
- public void deleteSavedSearch(String currentUser, String guid) throws AtlasBaseException {
- try {
- AtlasUserSavedSearch savedSearch = userProfileService.getSavedSearch(guid);
-
- checkSavedSearchOwnership(currentUser, savedSearch);
-
- userProfileService.deleteSavedSearch(guid);
- } catch (AtlasBaseException e) {
- LOG.error("deleteSavedSearch({})", guid, e);
- throw e;
- }
- }
-
- @Override
- public String getDslQueryUsingTypeNameClassification(String query, String typeName, String classification) {
- String queryStr = query == null ? "" : query;
-
- if (StringUtils.isNotEmpty(typeName)) {
- queryStr = escapeTypeName(typeName) + " " + queryStr;
- }
-
- if (StringUtils.isNotEmpty(classification)) {
- // isa works with a type name only - like hive_column isa PII; it doesn't work with more complex query
- if (StringUtils.isEmpty(query)) {
- queryStr += (" isa " + classification);
- }
- }
- return queryStr;
- }
-
- public static SearchParameters createSearchParameters(QuickSearchParameters quickSearchParameters) {
- SearchParameters searchParameters = new SearchParameters();
-
- searchParameters.setQuery(quickSearchParameters.getQuery());
- searchParameters.setTypeName(quickSearchParameters.getTypeName());
- searchParameters.setExcludeDeletedEntities(quickSearchParameters.getExcludeDeletedEntities());
- searchParameters.setIncludeSubTypes(quickSearchParameters.getIncludeSubTypes());
- searchParameters.setLimit(quickSearchParameters.getLimit());
- searchParameters.setOffset(quickSearchParameters.getOffset());
- searchParameters.setEntityFilters(quickSearchParameters.getEntityFilters());
- searchParameters.setAttributes(quickSearchParameters.getAttributes());
- searchParameters.setSortBy(quickSearchParameters.getSortBy());
- searchParameters.setSortOrder(quickSearchParameters.getSortOrder());
- searchParameters.setExcludeHeaderAttributes(quickSearchParameters.getExcludeHeaderAttributes());
-
- return searchParameters;
- }
-
- public static SearchParameters createSearchParameters(RelationshipSearchParameters relationshipSearchParameters) {
- SearchParameters searchParameters = new SearchParameters();
-
- searchParameters.setRelationshipName(relationshipSearchParameters.getRelationshipName());
- searchParameters.setIncludeSubTypes(relationshipSearchParameters.isIncludeSubTypes());
- searchParameters.setLimit(relationshipSearchParameters.getLimit());
- searchParameters.setOffset(relationshipSearchParameters.getOffset());
- searchParameters.setRelationshipFilters(relationshipSearchParameters.getRelationshipFilters());
- searchParameters.setSortBy(relationshipSearchParameters.getSortBy());
- searchParameters.setSortOrder(relationshipSearchParameters.getSortOrder());
- searchParameters.setMarker(relationshipSearchParameters.getMarker());
-
- return searchParameters;
- }
-
private String escapeTypeName(String typeName) {
String ret;
@@ -1182,7 +1182,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
AtlasEntityType typeAsset = typeRegistry.getEntityTypeByName(ASSET_ENTITY_TYPE);
AtlasAttribute atttOwner = typeAsset != null ? typeAsset.getAttribute(OWNER_ATTRIBUTE) : null;
- if(atttOwner == null) {
+ if (atttOwner == null) {
String msg = String.format("Unable to resolve the attribute %s.%s", ASSET_ENTITY_TYPE, OWNER_ATTRIBUTE);
LOG.error(msg);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
index aa881f2bd..99c91fc54 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
@@ -18,7 +18,6 @@
package org.apache.atlas.discovery;
-
import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.annotation.GraphTransaction;
@@ -31,8 +30,8 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.atlas.model.lineage.AtlasLineageInfo;
-import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageInfoOnDemand;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageDirection;
+import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageInfoOnDemand;
import org.apache.atlas.model.lineage.AtlasLineageInfo.LineageRelation;
import org.apache.atlas.model.lineage.LineageOnDemandConstraints;
import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -56,6 +55,7 @@ import org.springframework.stereotype.Service;
import javax.inject.Inject;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -99,18 +99,17 @@ public class EntityLineageService implements AtlasLineageService {
@Inject
EntityLineageService(AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph) {
- this.graph = atlasGraph;
- this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
- this.entityRetriever = new EntityGraphRetriever(atlasGraph, typeRegistry);
- this.atlasTypeRegistry = typeRegistry;
+ this.graph = atlasGraph;
+ this.gremlinQueryProvider = AtlasGremlinQueryProvider.getInstance();
+ this.entityRetriever = new EntityGraphRetriever(atlasGraph, typeRegistry);
+ this.atlasTypeRegistry = typeRegistry;
}
@Override
@GraphTransaction
public AtlasLineageInfo getAtlasLineageInfo(String guid, LineageDirection direction, int depth) throws AtlasBaseException {
AtlasLineageInfo ret;
-
- boolean isDataSet = validateEntityTypeAndCheckIfDataSet(guid);
+ boolean isDataSet = validateEntityTypeAndCheckIfDataSet(guid);
if (LINEAGE_USING_GREMLIN) {
ret = getLineageInfoV1(guid, direction, depth, isDataSet);
@@ -124,16 +123,14 @@ public class EntityLineageService implements AtlasLineageService {
@Override
@GraphTransaction
public AtlasLineageInfo getAtlasLineageInfo(String guid, Map<String, LineageOnDemandConstraints> lineageConstraintsMap) throws AtlasBaseException {
- AtlasLineageInfo ret;
-
if (MapUtils.isEmpty(lineageConstraintsMap)) {
lineageConstraintsMap = new HashMap<>();
+
lineageConstraintsMap.put(guid, getDefaultLineageConstraints(guid));
}
- boolean isDataSet = validateEntityTypeAndCheckIfDataSet(guid);
-
- ret = getLineageInfoOnDemand(guid, lineageConstraintsMap, isDataSet);
+ boolean isDataSet = validateEntityTypeAndCheckIfDataSet(guid);
+ AtlasLineageInfo ret = getLineageInfoOnDemand(guid, lineageConstraintsMap, isDataSet);
appendLineageOnDemandPayload(ret, lineageConstraintsMap);
@@ -143,6 +140,60 @@ public class EntityLineageService implements AtlasLineageService {
return ret;
}
+ @Override
+ @GraphTransaction
+ public SchemaDetails getSchemaForHiveTableByName(final String datasetName) throws AtlasBaseException {
+ if (StringUtils.isEmpty(datasetName)) {
+ // TODO: Complete error handling here
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST);
+ }
+
+ AtlasEntityType hiveTable = atlasTypeRegistry.getEntityTypeByName("hive_table");
+ Map<String, Object> lookupAttributes = new HashMap<>();
+
+ lookupAttributes.put("qualifiedName", datasetName);
+
+ String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(hiveTable, lookupAttributes);
+
+ return getSchemaForHiveTableByGuid(guid);
+ }
+
+ @Override
+ @GraphTransaction
+ public SchemaDetails getSchemaForHiveTableByGuid(final String guid) throws AtlasBaseException {
+ if (StringUtils.isEmpty(guid)) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST);
+ }
+
+ SchemaDetails ret = new SchemaDetails();
+ AtlasEntityType hiveColumn = atlasTypeRegistry.getEntityTypeByName("hive_column");
+
+ ret.setDataType(AtlasTypeUtil.toClassTypeDefinition(hiveColumn));
+
+ AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(guid);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)), "read entity schema: guid=", guid);
+
+ Map<String, AtlasEntity> referredEntities = entityWithExtInfo.getReferredEntities();
+ List<String> columnIds = getColumnIds(entity);
+
+ if (MapUtils.isNotEmpty(referredEntities)) {
+ List<Map<String, Object>> rows = referredEntities.entrySet()
+ .stream()
+ .filter(e -> isColumn(columnIds, e))
+ .map(e -> AtlasTypeUtil.toMap(e.getValue()))
+ .collect(Collectors.toList());
+ ret.setRows(rows);
+ }
+
+ return ret;
+ }
+
+ public boolean isLineageOnDemandEnabled() {
+ return AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getBoolean();
+ }
+
private boolean validateEntityTypeAndCheckIfDataSet(String guid) throws AtlasBaseException {
AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeaderWithClassifications(guid);
@@ -171,6 +222,7 @@ public class EntityLineageService implements AtlasLineageService {
if (lineageInfo == null || MapUtils.isEmpty(lineageConstraintsMap)) {
return;
}
+
lineageInfo.setLineageOnDemandPayload(lineageConstraintsMap);
}
@@ -181,61 +233,12 @@ public class EntityLineageService implements AtlasLineageService {
}
}
- @Override
- @GraphTransaction
- public SchemaDetails getSchemaForHiveTableByName(final String datasetName) throws AtlasBaseException {
- if (StringUtils.isEmpty(datasetName)) {
- // TODO: Complete error handling here
- throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST);
- }
-
- AtlasEntityType hive_table = atlasTypeRegistry.getEntityTypeByName("hive_table");
-
- Map<String, Object> lookupAttributes = new HashMap<>();
- lookupAttributes.put("qualifiedName", datasetName);
- String guid = AtlasGraphUtilsV2.getGuidByUniqueAttributes(hive_table, lookupAttributes);
-
- return getSchemaForHiveTableByGuid(guid);
- }
-
- @Override
- @GraphTransaction
- public SchemaDetails getSchemaForHiveTableByGuid(final String guid) throws AtlasBaseException {
- if (StringUtils.isEmpty(guid)) {
- throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST);
- }
- SchemaDetails ret = new SchemaDetails();
- AtlasEntityType hive_column = atlasTypeRegistry.getEntityTypeByName("hive_column");
-
- ret.setDataType(AtlasTypeUtil.toClassTypeDefinition(hive_column));
-
- AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(guid);
- AtlasEntity entity = entityWithExtInfo.getEntity();
-
- AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(atlasTypeRegistry, AtlasPrivilege.ENTITY_READ, new AtlasEntityHeader(entity)),
- "read entity schema: guid=", guid);
-
- Map<String, AtlasEntity> referredEntities = entityWithExtInfo.getReferredEntities();
- List<String> columnIds = getColumnIds(entity);
-
- if (MapUtils.isNotEmpty(referredEntities)) {
- List<Map<String, Object>> rows = referredEntities.entrySet()
- .stream()
- .filter(e -> isColumn(columnIds, e))
- .map(e -> AtlasTypeUtil.toMap(e.getValue()))
- .collect(Collectors.toList());
- ret.setRows(rows);
- }
-
- return ret;
- }
-
private List<String> getColumnIds(AtlasEntity entity) {
List<String> ret = new ArrayList<>();
Object columnObjs = entity.getAttribute(COLUMNS);
if (columnObjs instanceof List) {
- for (Object pkObj : (List) columnObjs) {
+ for (Object pkObj : (List<?>) columnObjs) {
if (pkObj instanceof AtlasObjectId) {
ret.add(((AtlasObjectId) pkObj).getGuid());
}
@@ -266,19 +269,18 @@ public class EntityLineageService implements AtlasLineageService {
private AtlasLineageInfo getLineageInfo(String guid, LineageDirection direction, int depth, boolean isDataSet) throws AtlasBaseException {
final Map<String, Object> bindings = new HashMap<>();
String lineageQuery = getLineageQuery(guid, direction, depth, isDataSet, bindings);
- List results = executeGremlinScript(bindings, lineageQuery);
+ List<?> results = executeGremlinScript(bindings, lineageQuery);
Map<String, AtlasEntityHeader> entities = new HashMap<>();
Set<LineageRelation> relations = new HashSet<>();
if (CollectionUtils.isNotEmpty(results)) {
for (Object result : results) {
if (result instanceof Map) {
- for (final Object o : ((Map) result).entrySet()) {
- final Map.Entry entry = (Map.Entry) o;
- Object value = entry.getValue();
+ for (final Map.Entry<?, ?> entry : ((Map<?, ?>) result).entrySet()) {
+ Object value = entry.getValue();
if (value instanceof List) {
- for (Object elem : (List) value) {
+ for (Object elem : (List<?>) value) {
if (elem instanceof AtlasEdge) {
processEdge((AtlasEdge) elem, entities, relations);
} else {
@@ -317,7 +319,7 @@ public class EntityLineageService implements AtlasLineageService {
if (direction == OUTPUT || direction == BOTH) {
traverseEdges(datasetVertex, false, depth, ret);
}
- } else {
+ } else {
AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
// make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1'
@@ -350,51 +352,51 @@ public class EntityLineageService implements AtlasLineageService {
}
private LineageOnDemandConstraints getDefaultLineageConstraints(String guid) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("No lineage on-demand constraints provided for guid: {}, configuring with default values direction: {}, inputRelationsLimit: {}, outputRelationsLimit: {}, depth: {}",
- guid, BOTH, LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT, LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT, LINEAGE_ON_DEMAND_DEFAULT_DEPTH);
- }
+ LOG.debug("No lineage on-demand constraints provided for guid: {}, configuring with default values direction: {}, inputRelationsLimit: {}, outputRelationsLimit: {}, depth: {}",
+ guid, BOTH, LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT, LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT, LINEAGE_ON_DEMAND_DEFAULT_DEPTH);
return new LineageOnDemandConstraints();
}
private LineageOnDemandConstraints getAndValidateLineageConstraintsByGuid(String guid, Map<String, LineageOnDemandConstraints> lineageConstraintsMap) {
-
if (lineageConstraintsMap == null || !lineageConstraintsMap.containsKey(guid)) {
return getDefaultLineageConstraints(guid);
}
- LineageOnDemandConstraints lineageConstraintsByGuid = lineageConstraintsMap.get(guid);;
+ LineageOnDemandConstraints lineageConstraintsByGuid = lineageConstraintsMap.get(guid);
+
if (lineageConstraintsByGuid == null) {
return getDefaultLineageConstraints(guid);
}
if (Objects.isNull(lineageConstraintsByGuid.getDirection())) {
LOG.info("No lineage on-demand direction provided for guid: {}, configuring with default value {}", guid, LineageDirection.BOTH);
+
lineageConstraintsByGuid.setDirection(BOTH);
}
if (lineageConstraintsByGuid.getInputRelationsLimit() == 0) {
LOG.info("No lineage on-demand inputRelationsLimit provided for guid: {}, configuring with default value {}", guid, LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT);
+
lineageConstraintsByGuid.setInputRelationsLimit(LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT);
}
if (lineageConstraintsByGuid.getOutputRelationsLimit() == 0) {
LOG.info("No lineage on-demand outputRelationsLimit provided for guid: {}, configuring with default value {}", guid, LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT);
+
lineageConstraintsByGuid.setOutputRelationsLimit(LINEAGE_ON_DEMAND_DEFAULT_NODE_COUNT);
}
if (lineageConstraintsByGuid.getDepth() == 0) {
LOG.info("No lineage on-demand depth provided for guid: {}, configuring with default value {}", guid, LINEAGE_ON_DEMAND_DEFAULT_DEPTH);
+
lineageConstraintsByGuid.setDepth(LINEAGE_ON_DEMAND_DEFAULT_DEPTH);
}
return lineageConstraintsByGuid;
-
}
private AtlasLineageInfo getLineageInfoOnDemand(String guid, Map<String, LineageOnDemandConstraints> lineageConstraintsMap, boolean isDataSet) throws AtlasBaseException {
-
LineageOnDemandConstraints lineageConstraintsByGuid = getAndValidateLineageConstraintsByGuid(guid, lineageConstraintsMap);
if (lineageConstraintsByGuid == null) {
@@ -424,7 +426,7 @@ public class EntityLineageService implements AtlasLineageService {
if (direction == OUTPUT || direction == BOTH) {
traverseEdgesOnDemand(datasetVertex, false, depth, new HashSet<>(), lineageConstraintsMap, ret);
}
- } else {
+ } else {
AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);
// make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1'
@@ -439,7 +441,6 @@ public class EntityLineageService implements AtlasLineageService {
traverseEdgesOnDemand(processEdges, false, depth, lineageConstraintsMap, ret);
}
-
}
return ret;
@@ -478,7 +479,7 @@ public class EntityLineageService implements AtlasLineageService {
private void traverseEdgesOnDemand(Iterable<AtlasEdge> processEdges, boolean isInput, int depth, Map<String, LineageOnDemandConstraints> lineageConstraintsMap, AtlasLineageInfo ret) throws AtlasBaseException {
for (AtlasEdge processEdge : processEdges) {
- boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE);
+ boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE);
if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, lineageConstraintsMap, ret)) {
break;
@@ -486,9 +487,8 @@ public class EntityLineageService implements AtlasLineageService {
addEdgeToResult(processEdge, ret);
}
- AtlasVertex datasetVertex = processEdge.getInVertex();
-
- String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex);
+ AtlasVertex datasetVertex = processEdge.getInVertex();
+ String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex);
LineageOnDemandConstraints inGuidLineageConstrains = getAndValidateLineageConstraintsByGuid(inGuid, lineageConstraintsMap);
if (!ret.getRelationsOnDemand().containsKey(inGuid)) {
@@ -507,7 +507,6 @@ public class EntityLineageService implements AtlasLineageService {
Iterable<AtlasEdge> incomingEdges = datasetVertex.getEdges(IN, isInput ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE);
for (AtlasEdge incomingEdge : incomingEdges) {
-
if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, lineageConstraintsMap, ret)) {
break;
} else {
@@ -518,7 +517,6 @@ public class EntityLineageService implements AtlasLineageService {
Iterable<AtlasEdge> outgoingEdges = processVertex.getEdges(OUT, isInput ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE);
for (AtlasEdge outgoingEdge : outgoingEdges) {
-
if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, lineageConstraintsMap, ret)) {
break;
} else {
@@ -536,7 +534,6 @@ public class EntityLineageService implements AtlasLineageService {
}
private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, Map<String, LineageOnDemandConstraints> lineageConstraintsMap, AtlasLineageInfo ret) {
-
if (lineageContainsEdge(ret, atlasEdge)) {
return false;
}
@@ -551,18 +548,20 @@ public class EntityLineageService implements AtlasLineageService {
String outGuid = AtlasGraphUtilsV2.getIdFromVertex(outVertex);
LineageOnDemandConstraints outGuidLineageConstraints = getAndValidateLineageConstraintsByGuid(outGuid, lineageConstraintsMap);
- LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints);
+ LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints);
LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints);
if (inLineageInfo.isInputRelationsReachedLimit()) {
inLineageInfo.setHasMoreInputs(true);
+
hasRelationsLimitReached = true;
- }else {
+ } else {
inLineageInfo.incrementInputRelationsCount();
}
if (outLineageInfo.isOutputRelationsReachedLimit()) {
outLineageInfo.setHasMoreOutputs(true);
+
hasRelationsLimitReached = true;
} else {
outLineageInfo.incrementOutputRelationsCount();
@@ -594,6 +593,7 @@ public class EntityLineageService implements AtlasLineageService {
if (isLineageOnDemandEnabled()) {
return inGuid + SEPARATOR + outGuid;
}
+
return relationGuid;
}
@@ -627,12 +627,12 @@ public class EntityLineageService implements AtlasLineageService {
return vertex.getIdForDisplay();
}
- private List executeGremlinScript(Map<String, Object> bindings, String lineageQuery) throws AtlasBaseException {
- List ret;
+ private List<?> executeGremlinScript(Map<String, Object> bindings, String lineageQuery) throws AtlasBaseException {
+ List<?> ret;
ScriptEngine engine = graph.getGremlinScriptEngine();
try {
- ret = (List) graph.executeGremlinScript(engine, bindings, lineageQuery, false);
+ ret = (List<?>) graph.executeGremlinScript(engine, bindings, lineageQuery, false);
} catch (ScriptException e) {
throw new AtlasBaseException(INSTANCE_LINEAGE_QUERY_FAILED, lineageQuery);
} finally {
@@ -654,14 +654,15 @@ public class EntityLineageService implements AtlasLineageService {
String relationGuid = AtlasGraphUtilsV2.getEncodedProperty(edge, RELATIONSHIP_GUID_PROPERTY_KEY, String.class);
boolean isInputEdge = edge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE);
-
if (!entities.containsKey(inGuid)) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(inVertex);
+
entities.put(inGuid, entityHeader);
}
if (!entities.containsKey(outGuid)) {
AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(outVertex);
+
entities.put(outGuid, entityHeader);
}
@@ -673,6 +674,7 @@ public class EntityLineageService implements AtlasLineageService {
if (visitedEdges != null) {
String visitedEdgeLabel = isInputEdge ? getVisitedEdgeLabel(inGuid, outGuid, relationGuid) : getVisitedEdgeLabel(outGuid, inGuid, relationGuid);
+
visitedEdges.add(visitedEdgeLabel);
}
}
@@ -709,17 +711,11 @@ public class EntityLineageService implements AtlasLineageService {
bindings.put("processDepth", depth - 1);
if (depth < 1) {
- ret = isDataSet ? gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) :
- gremlinQueryProvider.getQuery(FULL_LINEAGE_PROCESS);
+ ret = isDataSet ? gremlinQueryProvider.getQuery(FULL_LINEAGE_DATASET) : gremlinQueryProvider.getQuery(FULL_LINEAGE_PROCESS);
} else {
- ret = isDataSet ? gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_DATASET) :
- gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_PROCESS);
+ ret = isDataSet ? gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_DATASET) : gremlinQueryProvider.getQuery(PARTIAL_LINEAGE_PROCESS);
}
return ret;
}
-
- public boolean isLineageOnDemandEnabled() {
- return AtlasConfiguration.LINEAGE_ON_DEMAND_ENABLED.getBoolean();
- }
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 7f1f2f3b4..e002ecf7f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -34,12 +34,23 @@ import org.apache.commons.collections.PredicateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
import java.util.stream.StreamSupport;
-import static org.apache.atlas.SortOrder.ASCENDING;
-import static org.apache.atlas.discovery.SearchContext.*;
-import static org.apache.atlas.repository.Constants.*;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
+import static org.apache.atlas.discovery.SearchContext.MarkerUtil;
+import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
@@ -57,15 +68,15 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
- final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
- final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
- final Set<String> indexAttributes = new HashSet<>();
- final Set<String> graphAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes = context.getEntityTypeNames();
- final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
- final String sortBy = context.getSearchParameters().getSortBy();
- final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+ final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ final Set<String> indexAttributes = new HashSet<>();
+ final Set<String> graphAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
+ final Set<String> typeAndSubTypes = context.getEntityTypeNames();
+ final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
+ final String sortBy = context.getSearchParameters().getSortBy();
+ final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
final Set<String> classificationTypeAndSubTypes = context.getClassificationTypeNames();
@@ -78,10 +89,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
final Predicate typeNamePredicate;
- final Predicate traitPredicate = buildTraitPredict(classificationTypes);
- final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
- .generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
-
+ final Predicate traitPredicate = buildTraitPredict(classificationTypes);
+ final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator().generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
if (!isEntityRootType()) {
typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
@@ -107,6 +116,7 @@ public class EntitySearchProcessor extends SearchProcessor {
constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes);
+
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
@@ -115,7 +125,6 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (indexQuery.length() > 0) {
-
graphIndexQueryBuilder.addActiveStateQueryFilter(indexQuery);
String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
@@ -138,14 +147,14 @@ public class EntitySearchProcessor extends SearchProcessor {
// If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate
if (filterClassification) {
AtlasClassificationType classificationType = classificationTypes.iterator().next();
- List<AtlasGraphQuery> orConditions = new LinkedList<>();
+ List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
orConditions.add(query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
orConditions.add(query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null)
- .has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
+ .has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
} else {
orConditions.add(query.createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
orConditions.add(query.createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes));
@@ -181,6 +190,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// Filter condition for the STATUS
if (context.getSearchParameters().getExcludeDeletedEntities() && this.indexQuery == null) {
graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
+
if (graphQueryPredicate != null) {
graphQueryPredicate = PredicateUtils.andPredicate(graphQueryPredicate, activePredicate);
} else {
@@ -188,8 +198,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
if (sortBy != null && !sortBy.isEmpty()) {
- final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
- AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
+ final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
+ AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
@@ -198,7 +208,7 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
} else {
- graphQuery = null;
+ graphQuery = null;
graphQueryPredicate = null;
}
@@ -208,54 +218,48 @@ public class EntitySearchProcessor extends SearchProcessor {
Predicate attributesPredicate = constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes);
if (attributesPredicate != null) {
- filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate :
- PredicateUtils.andPredicate(filterGraphQueryPredicate, attributesPredicate);
+ filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate : PredicateUtils.andPredicate(filterGraphQueryPredicate, attributesPredicate);
}
if (filterClassification) {
- filterGraphQueryPredicate = filterGraphQueryPredicate == null ? traitPredicate :
- PredicateUtils.andPredicate(filterGraphQueryPredicate, traitPredicate);
+ filterGraphQueryPredicate = filterGraphQueryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(filterGraphQueryPredicate, traitPredicate);
}
// Filter condition for the STATUS
if (context.getSearchParameters().getExcludeDeletedEntities()) {
- filterGraphQueryPredicate = filterGraphQueryPredicate == null ? activePredicate :
- PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate);
+ filterGraphQueryPredicate = filterGraphQueryPredicate == null ? activePredicate : PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate);
}
-
}
@Override
public List<AtlasVertex> execute() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> EntitySearchProcessor.execute({})", context);
- }
-
- List<AtlasVertex> ret = new ArrayList<>();
+ LOG.debug("==> EntitySearchProcessor.execute({})", context);
- AtlasPerfTracer perf = null;
+ List<AtlasVertex> ret = new ArrayList<>();
+ AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntitySearchProcessor.execute(" + context + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntitySearchProcessor.execute(" + context + ")");
}
try {
- final int limit = context.getSearchParameters().getLimit();
- final Integer marker = context.getMarker();
- final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ final Integer marker = context.getMarker();
+ final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
// when subsequent filtering stages are involved, query should start at 0 even though startIdx can be higher
//
// first 'startIdx' number of entries will be ignored
// if marker is provided, start query with marker offset
int qryOffset;
+
if (marker != null) {
qryOffset = marker;
} else {
qryOffset = (nextProcessor != null || (graphQuery != null && indexQuery != null) || isEntityRootType()) ? 0 : startIdx;
}
- int resultIdx = qryOffset;
+ int resultIdx = qryOffset;
LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap = new LinkedHashMap<>();
for (; ret.size() < limit; qryOffset += limit) {
@@ -271,10 +275,11 @@ public class EntitySearchProcessor extends SearchProcessor {
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> idxQueryResult = executeIndexQuery(context, indexQuery, qryOffset, limit);
- offsetEntityVertexMap = getVerticesFromIndexQueryResult(idxQueryResult, offsetEntityVertexMap, qryOffset);
+ offsetEntityVertexMap = getVerticesFromIndexQueryResult(idxQueryResult, offsetEntityVertexMap, qryOffset);
} else {
Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset, limit).iterator();
+
offsetEntityVertexMap = getVertices(queryResult, offsetEntityVertexMap, qryOffset);
}
@@ -300,48 +305,42 @@ public class EntitySearchProcessor extends SearchProcessor {
if (marker != null) {
nextOffset = resultIdx + 1;
}
-
} finally {
AtlasPerfTracer.log(perf);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== EntitySearchProcessor.execute({}): ret.size()={}", context, ret.size());
- }
+ LOG.debug("<== EntitySearchProcessor.execute({}): ret.size()={}", context, ret.size());
return ret;
}
@Override
- public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer,AtlasVertex> offsetEntityVertexMap) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> EntitySearchProcessor.filter({})", offsetEntityVertexMap.size());
+ public long getResultCount() {
+ if (indexQuery != null) {
+ return indexQuery.vertexTotals();
+ } else if (graphQuery != null) {
+ return StreamSupport.stream(graphQuery.vertexIds().spliterator(), false).count();
+ } else {
+ return -1L;
}
+ }
+
+ @Override
+ public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap) {
+ LOG.debug("==> EntitySearchProcessor.filter({})", offsetEntityVertexMap.size());
// Since we already have the entity vertices, a in-memory filter will be faster than fetching the same
// vertices again with the required filtering
if (filterGraphQueryPredicate != null) {
LOG.debug("Filtering in-memory");
+
offsetEntityVertexMap = super.filter(offsetEntityVertexMap, filterGraphQueryPredicate);
}
offsetEntityVertexMap = super.filter(offsetEntityVertexMap);
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== EntitySearchProcessor.filter(): ret.size()={}", offsetEntityVertexMap.size());
- }
+ LOG.debug("<== EntitySearchProcessor.filter(): ret.size()={}", offsetEntityVertexMap.size());
return offsetEntityVertexMap;
}
-
- @Override
- public long getResultCount() {
- if (indexQuery != null) {
- return indexQuery.vertexTotals();
- } else if (graphQuery != null) {
- return StreamSupport.stream(graphQuery.vertexIds().spliterator(), false).count();
- } else {
- return -1L;
- }
- }
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index d0143c791..f619c657a 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -22,7 +22,10 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
-import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
+import org.apache.atlas.repository.graphdb.AtlasIndexQueryParameter;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.GraphIndexQueryParameters;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
@@ -36,16 +39,16 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
-
/**
* This class is equivalent to legacy FullTextSearchProcessor--except that it uses a better search techniques using SOLR
* than going through Janus Graph index apis.
*/
public class FreeTextSearchProcessor extends SearchProcessor {
- private static final Logger LOG = LoggerFactory.getLogger(FreeTextSearchProcessor.class);
- private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FreeTextSearchProcessor");
- public static final String SOLR_QT_PARAMETER = "qt"; // org.apache.solr.common.params.CommonParams.QT;
- public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
+ private static final Logger LOG = LoggerFactory.getLogger(FreeTextSearchProcessor.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FreeTextSearchProcessor");
+
+ public static final String SOLR_QT_PARAMETER = "qt"; // org.apache.solr.common.params.CommonParams.QT;
+ public static final String SOLR_REQUEST_HANDLER_NAME = "/freetext";
private static final boolean IS_SOLR_INDEX_BACKEND = isSolrIndexBackend();
@@ -70,39 +73,26 @@ public class FreeTextSearchProcessor extends SearchProcessor {
}
// just use the query string as is
- LOG.debug("Using query string '{}'.", queryString);
+ LOG.debug("Using query string '{}'.", queryString);
indexQuery = context.getGraph().indexQuery(prepareGraphIndexQueryParameters(context, queryString));
}
- private GraphIndexQueryParameters prepareGraphIndexQueryParameters(SearchContext context, StringBuilder queryString) {
- List<AtlasIndexQueryParameter> parameters = new ArrayList<>();
-
- if (IS_SOLR_INDEX_BACKEND) {
- parameters.add(context.getGraph().indexQueryParameter(SOLR_QT_PARAMETER, SOLR_REQUEST_HANDLER_NAME));
- }
-
- return new GraphIndexQueryParameters(Constants.VERTEX_INDEX, queryString.toString(), 0, parameters);
- }
-
@Override
public List<AtlasVertex> execute() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> FreeTextSearchProcessor.execute({})", context);
- }
-
- List<AtlasVertex> ret = new ArrayList<>();
+ LOG.debug("==> FreeTextSearchProcessor.execute({})", context);
- AtlasPerfTracer perf = null;
+ List<AtlasVertex> ret = new ArrayList<>();
+ AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FreeTextSearchProcessor.execute(" + context + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FreeTextSearchProcessor.execute(" + context + ")");
}
try {
- final int limit = context.getSearchParameters().getLimit();
- final Integer marker = context.getMarker();
- final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ final Integer marker = context.getMarker();
+ final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like vertices of non-entity or non-active-entity
@@ -127,7 +117,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
Iterator<AtlasIndexQuery.Result> idxQueryResult = executeIndexQuery(context, indexQuery, qryOffset, limit);
final boolean isLastResultPage;
- int resultCount = 0;
+ int resultCount = 0;
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
@@ -141,11 +131,13 @@ public class FreeTextSearchProcessor extends SearchProcessor {
if (LOG.isDebugEnabled()) {
LOG.debug("FreeTextSearchProcessor.execute(): ignoring non-entity vertex (id={})", vertex.getId());
}
+
continue;
}
//skip internalTypes
AtlasEntityType entityType = context.getTypeRegistry().getEntityTypeByName(entityTypeName);
+
if (entityType != null && entityType.isInternalType()) {
continue;
}
@@ -166,12 +158,12 @@ public class FreeTextSearchProcessor extends SearchProcessor {
}
isLastResultPage = resultCount < limit;
-
offsetEntityVertexMap = super.filter(offsetEntityVertexMap);
resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, offsetEntityVertexMap, marker);
if (isLastResultPage) {
resultIdx = SearchContext.MarkerUtil.MARKER_END - 1;
+
break;
}
}
@@ -182,14 +174,11 @@ public class FreeTextSearchProcessor extends SearchProcessor {
if (marker != null) {
nextOffset = resultIdx + 1;
}
-
} finally {
AtlasPerfTracer.log(perf);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== FreeTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
- }
+ LOG.debug("<== FreeTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
return ret;
}
@@ -199,6 +188,16 @@ public class FreeTextSearchProcessor extends SearchProcessor {
return indexQuery.vertexTotals();
}
+ private GraphIndexQueryParameters prepareGraphIndexQueryParameters(SearchContext context, StringBuilder queryString) {
+ List<AtlasIndexQueryParameter> parameters = new ArrayList<>();
+
+ if (IS_SOLR_INDEX_BACKEND) {
+ parameters.add(context.getGraph().indexQueryParameter(SOLR_QT_PARAMETER, SOLR_REQUEST_HANDLER_NAME));
+ }
+
+ return new GraphIndexQueryParameters(Constants.VERTEX_INDEX, queryString.toString(), 0, parameters);
+ }
+
private static boolean isSolrIndexBackend() {
try {
String indexBackEnd = ApplicationProperties.get().getString(ApplicationProperties.INDEX_BACKEND_CONF);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 464842090..9dda60a51 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -36,7 +36,6 @@ import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-
public class FullTextSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(FullTextSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("FullTextSearchProcessor");
@@ -49,7 +48,7 @@ public class FullTextSearchProcessor extends SearchProcessor {
SearchParameters searchParameters = context.getSearchParameters();
StringBuilder queryString = new StringBuilder();
- queryString.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.ENTITY_TEXT_PROPERTY_KEY).append("\":(").append(searchParameters.getQuery());
+ queryString.append(INDEX_SEARCH_PREFIX).append("\"").append(Constants.ENTITY_TEXT_PROPERTY_KEY).append("\":(").append(searchParameters.getQuery());
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
@@ -59,22 +58,19 @@ public class FullTextSearchProcessor extends SearchProcessor {
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else {
- LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
- searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length());
+ LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
- if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
- context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
String typeAndSubTypeNamesStr = context.getClassificationTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else {
- LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
- searchParameters.getClassification(), typeAndSubTypeNamesStr.length());
+ LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance", searchParameters.getClassification(), typeAndSubTypeNamesStr.length());
}
}
@@ -85,16 +81,13 @@ public class FullTextSearchProcessor extends SearchProcessor {
@Override
public List<AtlasVertex> execute() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> FullTextSearchProcessor.execute({})", context);
- }
-
- List<AtlasVertex> ret = new ArrayList<>();
+ LOG.debug("==> FullTextSearchProcessor.execute({})", context);
- AtlasPerfTracer perf = null;
+ List<AtlasVertex> ret = new ArrayList<>();
+ AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessor.execute(" + context + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "FullTextSearchProcessor.execute(" + context + ")");
}
try {
@@ -141,13 +134,13 @@ public class FullTextSearchProcessor extends SearchProcessor {
continue;
}
//skip internalTypes
- String entityTypeName = AtlasGraphUtilsV2.getTypeName(vertex);
- AtlasEntityType entityType = context.getTypeRegistry().getEntityTypeByName(entityTypeName);
- if (entityType != null && entityType.isInternalType()) {
+ String entityTypeName = AtlasGraphUtilsV2.getTypeName(vertex);
+ AtlasEntityType entityType = context.getTypeRegistry().getEntityTypeByName(entityTypeName);
+
+ if (entityType != null && entityType.isInternalType()) {
continue;
}
-
if (activeOnly && AtlasGraphUtilsV2.getState(vertex) != AtlasEntity.Status.ACTIVE) {
continue;
}
@@ -156,13 +149,12 @@ public class FullTextSearchProcessor extends SearchProcessor {
}
isLastResultPage = resultCount < limit;
-
offsetEntityVertexMap = super.filter(offsetEntityVertexMap);
-
- resultIdx = collectResultVertices(ret,startIdx, limit, resultIdx, offsetEntityVertexMap, marker);
+ resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, offsetEntityVertexMap, marker);
if (isLastResultPage) {
- resultIdx = SearchContext.MarkerUtil.MARKER_END - 1 ;
+ resultIdx = SearchContext.MarkerUtil.MARKER_END - 1;
+
break;
}
}
@@ -170,14 +162,11 @@ public class FullTextSearchProcessor extends SearchProcessor {
if (marker != null) {
nextOffset = resultIdx + 1;
}
-
} finally {
AtlasPerfTracer.log(perf);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
- }
+ LOG.debug("<== FullTextSearchProcessor.execute({}): ret.size()={}", context, ret.size());
return ret;
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
index e8f51a3ab..3c31d0908 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
@@ -17,17 +17,17 @@
*/
package org.apache.atlas.discovery;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchProcessor.INDEX_SEARCH_PREFIX;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.type.AtlasStructType;
-import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang3.StringUtils;
-
public class GraphIndexQueryBuilder {
SearchContext context;
@@ -38,13 +38,14 @@ public class GraphIndexQueryBuilder {
void addClassificationTypeFilter(StringBuilder indexQuery) {
if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) {
String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames(), true);
+
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames)
- .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
- .append('\"').append(':').append(classificationNames).append(")");
+ .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
+ .append('\"').append(':').append(classificationNames).append(")");
}
}
@@ -57,8 +58,8 @@ public class GraphIndexQueryBuilder {
}
indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
- .append("\"").append(":" + classificationTypesQryStr).append(" OR ").append(INDEX_SEARCH_PREFIX)
- .append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + classificationTypesQryStr).append(")");
+ .append("\"").append(":").append(classificationTypesQryStr).append(" OR ").append(INDEX_SEARCH_PREFIX)
+ .append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":").append(classificationTypesQryStr).append(")");
}
}
@@ -68,21 +69,20 @@ public class GraphIndexQueryBuilder {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
- indexQuery.append("( *:* ").append("-").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
- .append("\"").append(":" + "[* TO *]").append(" AND ").append("-")
- .append(INDEX_SEARCH_PREFIX).append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
- .append("\"").append(":" + "[* TO *]").append(")");
+
+ indexQuery.append("( *:* -").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY).append("\":[* TO *] AND -")
+ .append(INDEX_SEARCH_PREFIX).append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\":[* TO *])");
}
}
}
- void addActiveStateQueryFilter(StringBuilder indexQuery){
+ void addActiveStateQueryFilter(StringBuilder indexQuery) {
if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery != null) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
- indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(STATE_PROPERTY_KEY)
- .append("\"").append(":" + "ACTIVE").append(")");
+
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(STATE_PROPERTY_KEY).append("\"").append(":" + "ACTIVE").append(")");
}
}
@@ -92,8 +92,8 @@ public class GraphIndexQueryBuilder {
indexQuery.append(" AND ");
}
- indexQuery.append("(").append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY)
- .append("\":").append(typeAndAllSubTypesQryStr).append(")");
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(Constants.TYPE_NAME_PROPERTY_KEY)
+ .append("\":").append(typeAndAllSubTypesQryStr).append(")");
}
}
@@ -103,8 +103,7 @@ public class GraphIndexQueryBuilder {
indexQuery.append(" AND ");
}
- indexQuery.append("(").append(INDEX_SEARCH_PREFIX + "\"").append(Constants.RELATIONSHIP_TYPE_PROPERTY_KEY)
- .append("\":").append(typeAndAllSubTypesQryStr).append(")");
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(Constants.RELATIONSHIP_TYPE_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr).append(")");
}
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java
index 5e40aa03b..86cd764c6 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/RelationshipSearchProcessor.java
@@ -57,18 +57,20 @@ public class RelationshipSearchProcessor extends SearchProcessor {
RelationshipSearchProcessor(SearchContext context, Set<String> indexedKeys) {
super(context);
+
context.setEdgeIndexKeys(indexedKeys);
- final Set<AtlasRelationshipType> types = context.getRelationshipTypes();
- final SearchParameters.FilterCriteria filterCriteria = context.getSearchParameters().getRelationshipFilters();
- final Set<String> indexAttributes = new HashSet<>();
- final Set<String> graphAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeNames = CollectionUtils.isNotEmpty(types) ? types.stream().map(AtlasRelationshipType::getTypeName).collect(Collectors.toSet()) : null;
- final String typeAndSubTypesQryStr = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(typeNames, true);
- final Predicate typeNamePredicate = SearchPredicateUtil.generateIsRelationshipEdgePredicate(context.getTypeRegistry());
- final String sortBy = context.getSearchParameters().getSortBy();
- final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+ final Set<AtlasRelationshipType> types = context.getRelationshipTypes();
+ final SearchParameters.FilterCriteria filterCriteria = context.getSearchParameters().getRelationshipFilters();
+ final Set<String> indexAttributes = new HashSet<>();
+ final Set<String> graphAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
+ final Set<String> typeNames = CollectionUtils.isNotEmpty(types) ? types.stream().map(AtlasRelationshipType::getTypeName).collect(Collectors.toSet()) : null;
+ final String typeAndSubTypesQryStr = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(typeNames, true);
+ final Predicate typeNamePredicate = SearchPredicateUtil.generateIsRelationshipEdgePredicate(context.getTypeRegistry());
+ final String sortBy = context.getSearchParameters().getSortBy();
+ final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+
inMemoryPredicate = typeNamePredicate;
processSearchAttributes(types, filterCriteria, indexAttributes, graphAttributes, allAttributes);
@@ -87,6 +89,7 @@ public class RelationshipSearchProcessor extends SearchProcessor {
constructFilterQuery(indexQuery, types, filterCriteria, indexAttributes);
Predicate attributePredicate = constructInMemoryPredicate(types, filterCriteria, indexAttributes);
+
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
@@ -95,10 +98,10 @@ public class RelationshipSearchProcessor extends SearchProcessor {
}
if (indexQuery.length() > 0) {
-
String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
- indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
- indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
+
+ indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
+ indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
this.indexQuery = context.getGraph().indexQuery(Constants.EDGE_INDEX, indexQueryString);
} else {
@@ -112,16 +115,18 @@ public class RelationshipSearchProcessor extends SearchProcessor {
if (!typeSearchByIndex) {
query.in(RELATIONSHIP_TYPE_PROPERTY_KEY, types);
}
+
graphQuery = toGraphFilterQuery(types, filterCriteria, graphAttributes, query);
Predicate attributePredicate = constructInMemoryPredicate(types, filterCriteria, graphAttributes);
+
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
if (StringUtils.isNotEmpty(sortBy)) {
- final AtlasRelationshipType relationshipType = types.iterator().next();
- AtlasStructType.AtlasAttribute sortByAttribute = relationshipType.getAttribute(sortBy);
+ final AtlasRelationshipType relationshipType = types.iterator().next();
+ AtlasStructType.AtlasAttribute sortByAttribute = relationshipType.getAttribute(sortBy);
if (sortByAttribute != null && StringUtils.isNotEmpty(sortByAttribute.getVertexPropertyName())) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
@@ -132,7 +137,6 @@ public class RelationshipSearchProcessor extends SearchProcessor {
} else {
graphQuery = null;
}
-
}
@Override
@@ -140,30 +144,40 @@ public class RelationshipSearchProcessor extends SearchProcessor {
return null;
}
- public List<AtlasEdge> executeEdges() {
- List<AtlasEdge> ret = new ArrayList<>();
+ @Override
+ public long getResultCount() {
+ if (indexQuery != null) {
+ return indexQuery.edgeTotals();
+ } else {
+ return -1L;
+ }
+ }
+ public List<AtlasEdge> executeEdges() {
+ List<AtlasEdge> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
+
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "RelationshipSearchProcessor.execute(" + context + ")");
}
try {
- final int limit = context.getSearchParameters().getLimit();
- final Integer marker = context.getMarker();
- final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
+ final int limit = context.getSearchParameters().getLimit();
+ final Integer marker = context.getMarker();
+ final int startIdx = marker != null ? marker : context.getSearchParameters().getOffset();
// when subsequent filtering stages are involved, query should start at 0 even though startIdx can be higher
// first 'startIdx' number of entries will be ignored
// if marker is provided, start query with marker offset
int qryOffset;
+
if (marker != null) {
qryOffset = marker;
} else {
qryOffset = (graphQuery != null && indexQuery != null) ? 0 : startIdx;
}
- int resultIdx = qryOffset;
+ int resultIdx = qryOffset;
LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap = new LinkedHashMap<>();
for (; ret.size() < limit; qryOffset += limit) {
@@ -175,18 +189,17 @@ public class RelationshipSearchProcessor extends SearchProcessor {
break;
}
- final boolean isLastResultPage;
-
if (indexQuery != null) {
Iterator<AtlasIndexQuery.Result> idxQueryResult = executeIndexQueryForEdge(context, indexQuery, qryOffset, limit);
- offsetEdgeMap = getEdgesFromIndexQueryResult(idxQueryResult, offsetEdgeMap, qryOffset);
+ offsetEdgeMap = getEdgesFromIndexQueryResult(idxQueryResult, offsetEdgeMap, qryOffset);
} else {
Iterator<AtlasEdge> queryResult = graphQuery.edges(qryOffset, limit).iterator();
+
offsetEdgeMap = getEdges(queryResult, offsetEdgeMap, qryOffset);
}
- isLastResultPage = offsetEdgeMap.size() < limit;
+ final boolean isLastResultPage = offsetEdgeMap.size() < limit;
// Do in-memory filtering
offsetEdgeMap = offsetEdgeMap.entrySet()
@@ -205,23 +218,12 @@ public class RelationshipSearchProcessor extends SearchProcessor {
if (marker != null) {
nextOffset = resultIdx + 1;
}
-
} finally {
AtlasPerfTracer.log(perf);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== RelationshipSearchProcessor.execute({}): ret.size()={}", context, ret.size());
- }
- return ret;
- }
+ LOG.debug("<== RelationshipSearchProcessor.execute({}): ret.size()={}", context, ret.size());
- @Override
- public long getResultCount() {
- if (indexQuery != null) {
- return indexQuery.edgeTotals();
- } else {
- return -1L;
- }
+ return ret;
}
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
index b7c27e510..893613c85 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregator.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -32,8 +32,7 @@ public interface SearchAggregator {
* returns aggregation metrics for passed in aggregation fields.
* @param aggregationFields the set of aggregation attribute names.
* @param aggregationAttrbutes the set of aggregationAttributes
- * @return the result of aggreggations by aggregation fields.
+ * @return the result of aggreggations by aggregation fields.
*/
- Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields,
- Set<AtlasStructType.AtlasAttribute> aggregationAttrbutes);
+ Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields, Set<AtlasStructType.AtlasAttribute> aggregationAttrbutes);
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
index 2d095bed2..33d44bf8a 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -31,26 +31,28 @@ import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
public class SearchAggregatorImpl implements SearchAggregator {
private static final Logger LOG = LoggerFactory.getLogger(SearchAggregatorImpl.class);
private final SearchContext searchContext;
-
-
public SearchAggregatorImpl(SearchContext searchContext) {
this.searchContext = searchContext;
}
- public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields,
- Set<AtlasAttribute> aggregationAttributes) {
- SearchParameters searchParameters = searchContext.getSearchParameters();
- AtlasGraph graph = searchContext.getGraph();
- AtlasTypeRegistry typeRegistry = searchContext.getTypeRegistry();
- String queryString = searchParameters.getQuery();
- List<PostProcessor> postProcessors = new ArrayList<>();
+ public Map<String, List<AtlasAggregationEntry>> getAggregatedMetrics(Set<String> aggregationFields, Set<AtlasAttribute> aggregationAttributes) {
+ SearchParameters searchParameters = searchContext.getSearchParameters();
+ AtlasGraph graph = searchContext.getGraph();
+ AtlasTypeRegistry typeRegistry = searchContext.getTypeRegistry();
+ String queryString = searchParameters.getQuery();
+ List<PostProcessor> postProcessors = new ArrayList<>();
postProcessors.add(new ServiceTypeAggregator(searchContext.getTypeRegistry()));
@@ -60,13 +62,13 @@ public class SearchAggregatorImpl implements SearchAggregator {
Map<String, String> indexFieldNameCache = new HashMap<>();
- for (String fieldName: aggregationFields) {
+ for (String fieldName : aggregationFields) {
String indexFieldName = getIndexFieldNameForCommonFieldName(typeRegistry, fieldName);
indexFieldNameCache.put(fieldName, indexFieldName);
}
- for (AtlasAttribute attribute: aggregationAttributes) {
+ for (AtlasAttribute attribute : aggregationAttributes) {
String indexFieldName = attribute.getIndexFieldName();
if (indexFieldName == null) {
@@ -77,23 +79,17 @@ public class SearchAggregatorImpl implements SearchAggregator {
indexFieldNameCache.put(attribute.getQualifiedName(), indexFieldName);
}
- AggregationContext aggregatorContext = new AggregationContext(queryString,
- searchParameters.getEntityFilters(),
- searchForEntityType,
- aggregationFields,
- aggregationAttributes,
- indexFieldNameCache,
- searchParameters.getExcludeDeletedEntities(),
- searchParameters.getIncludeSubTypes());
+ AggregationContext aggregatorContext = new AggregationContext(queryString, searchParameters.getEntityFilters(), searchForEntityType,
+ aggregationFields, aggregationAttributes, indexFieldNameCache, searchParameters.getExcludeDeletedEntities(), searchParameters.getIncludeSubTypes());
Map<String, List<AtlasAggregationEntry>> aggregatedMetrics = graphIndexClient.getAggregatedMetrics(aggregatorContext);
- for (String aggregationMetricName: aggregatedMetrics.keySet()) {
- for (PostProcessor postProcessor: postProcessors) {
+ for (String aggregationMetricName : aggregatedMetrics.keySet()) {
+ for (PostProcessor postProcessor : postProcessors) {
if (postProcessor.needsProcessing(aggregationMetricName)) {
postProcessor.prepareForMetric(aggregationMetricName);
- for (AtlasAggregationEntry aggregationEntry: aggregatedMetrics.get(aggregationMetricName)) {
+ for (AtlasAggregationEntry aggregationEntry : aggregatedMetrics.get(aggregationMetricName)) {
postProcessor.process(aggregationEntry);
}
@@ -102,7 +98,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
}
- for (PostProcessor postProcessor: postProcessors) {
+ for (PostProcessor postProcessor : postProcessors) {
postProcessor.handleCompletion(aggregatedMetrics);
}
@@ -110,37 +106,39 @@ public class SearchAggregatorImpl implements SearchAggregator {
} catch (AtlasException e) {
LOG.error("Error encountered in post processing stage of aggrgation metrics collection. Empty metrics will be returned.", e);
- return Collections.EMPTY_MAP;
+ return Collections.emptyMap();
}
}
private String getIndexFieldNameForCommonFieldName(AtlasTypeRegistry typeRegistry, String fieldName) {
String indexFieldName = typeRegistry.getIndexFieldName(fieldName);
- if(indexFieldName != null) {
+ if (indexFieldName != null) {
return indexFieldName;
}
- if(LOG.isDebugEnabled()) {
- LOG.debug("Could not find index field name from type registry for attribute {}. Will use use the field name as is.", fieldName);
- }
+ LOG.debug("Could not find index field name from type registry for attribute {}. Will use use the field name as is.", fieldName);
return fieldName;
}
- static interface PostProcessor {
+ interface PostProcessor {
boolean needsProcessing(String metricName);
+
void prepareForMetric(String metricName);
+
void process(AtlasAggregationEntry aggregationEntry);
+
void handleMetricCompletion(String metricName);
+
void handleCompletion(Map<String, List<AtlasAggregationEntry>> aggregatedMetrics);
}
- static class ServiceTypeAggregator implements PostProcessor {
+ static class ServiceTypeAggregator implements PostProcessor {
private static final String SERVICE_TYPE = "ServiceType";
private final AtlasTypeRegistry typeRegistry;
- private final List<AtlasAggregationEntry> entries = new ArrayList<>();;
+ private final List<AtlasAggregationEntry> entries = new ArrayList<>();
private final Map<String, AtlasAggregationEntry> entityType2MetricsMap = new HashMap<>();
public ServiceTypeAggregator(AtlasTypeRegistry typeRegistry) {
@@ -157,7 +155,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
Map<String, AtlasAggregationEntry> serviceName2MetricsMap = new HashMap<>();
//prepare the service map to aggregations
- for(String serviceName: typeRegistry.getAllServiceTypes()) {
+ for (String serviceName : typeRegistry.getAllServiceTypes()) {
AtlasAggregationEntry serviceMetrics = new AtlasAggregationEntry(serviceName, 0);
serviceName2MetricsMap.put(serviceName, serviceMetrics);
@@ -166,7 +164,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
}
//prepare the map from entity type to aggregations
- for(AtlasEntityType entityType: typeRegistry.getAllEntityTypes()) {
+ for (AtlasEntityType entityType : typeRegistry.getAllEntityTypes()) {
String serviceName = entityType.getServiceType();
entityType2MetricsMap.put(entityType.getTypeName(), serviceName2MetricsMap.get(serviceName));
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 9a9a5512a..c59a5c440 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -17,7 +17,6 @@
*/
package org.apache.atlas.discovery;
-
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
@@ -32,14 +31,11 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
-import org.apache.atlas.type.AtlasArrayType;
-import org.apache.atlas.type.AtlasBuiltInTypes;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
-import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.commons.collections.CollectionUtils;
@@ -73,44 +69,43 @@ import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIF
* possible chaining of processor(s)
*/
public class SearchContext {
- private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class);
-
- private final AtlasTypeRegistry typeRegistry;
- private final AtlasGraph graph;
- private final Set<AtlasEntityType> entityTypes;
- private final Set<String> indexedKeys;
- private Set<String> edgeIndexKeys;
- private final Set<String> entityAttributes;
- private final Set<String> relationAttributes;
- private final SearchParameters searchParameters;
+ private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class);
+
+ public static final AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
+ public static final AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
+ public static final AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
+ public static final AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
+ public static final AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
+ public static final String TYPENAME_DELIMITER = ",";
+
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasGraph graph;
+ private final Set<AtlasEntityType> entityTypes;
+ private final Set<String> indexedKeys;
+ private final Set<String> entityAttributes;
+ private final Set<String> relationAttributes;
+ private final SearchParameters searchParameters;
private final Set<AtlasClassificationType> classificationTypes;
private final Set<AtlasRelationshipType> relationshipTypes;
private final Set<String> classificationNames;
- private final Set<String> typeAndSubTypes;
- private final Set<String> classificationTypeAndSubTypes;
- private final String typeAndSubTypesQryStr;
- private final String classificationTypeAndSubTypesQryStr;
- private boolean terminateSearch = false;
- private SearchProcessor searchProcessor;
- private Integer marker;
- private boolean hasRelationshipAttributes = false;
-
- public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
- public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
- public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
- public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
- public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
- public final static String TYPENAME_DELIMITER = ",";
-
+ private final Set<String> typeAndSubTypes;
+ private final Set<String> classificationTypeAndSubTypes;
+ private final String typeAndSubTypesQryStr;
+ private final String classificationTypeAndSubTypesQryStr;
+ private Set<String> edgeIndexKeys;
+ private boolean terminateSearch;
+ private SearchProcessor searchProcessor;
+ private Integer marker;
+ private boolean hasRelationshipAttributes;
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
- this.searchParameters = searchParameters;
- this.typeRegistry = typeRegistry;
- this.graph = graph;
- this.indexedKeys = indexedKeys;
- this.entityAttributes = new HashSet<>();
- this.relationAttributes = new HashSet<>();
- this.entityTypes = getEntityTypes(searchParameters.getTypeName());
+ this.searchParameters = searchParameters;
+ this.typeRegistry = typeRegistry;
+ this.graph = graph;
+ this.indexedKeys = indexedKeys;
+ this.entityAttributes = new HashSet<>();
+ this.relationAttributes = new HashSet<>();
+ this.entityTypes = getEntityTypes(searchParameters.getTypeName());
this.classificationNames = getClassificationNames(searchParameters.getClassification());
this.classificationTypes = getClassificationTypes(this.classificationNames);
this.relationshipTypes = getRelationshipTypes(searchParameters.getRelationshipName());
@@ -133,7 +128,7 @@ public class SearchContext {
//Wildcard tag with filter will raise an exception with 400 error code
if (CollectionUtils.isNotEmpty(classificationNames) && hasAttributeFilter(searchParameters.getTagFilters())) {
- for (String classificationName : classificationNames){
+ for (String classificationName : classificationNames) {
//in case of '*' , filters are allowed, but
//in case of regex 'PI*', filters are not allowed ( if present in any of the requested tag)
if (classificationName.contains(WILDCARD_CLASSIFICATIONS) && !classificationName.equals(WILDCARD_CLASSIFICATIONS)) {
@@ -167,19 +162,21 @@ public class SearchContext {
validateAttributes();
//gather all classifications and its corresponding subtypes
- Set<String> classificationTypeAndSubTypes = new HashSet<>();
- String classificationTypeAndSubTypesQryStr = null;
+ Set<String> classificationTypeAndSubTypes = new HashSet<>();
+ String classificationTypeAndSubTypesQryStr = null;
- if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) {
+ if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
for (AtlasClassificationType classificationType : classificationTypes) {
-
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
+
break;
} else {
Set<String> allTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
- classificationTypeAndSubTypes.addAll(allTypes); }
+
+ classificationTypeAndSubTypes.addAll(allTypes);
+ }
}
if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) {
@@ -189,22 +186,24 @@ public class SearchContext {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = "";
}
+
this.classificationTypeAndSubTypes = classificationTypeAndSubTypes;
this.classificationTypeAndSubTypesQryStr = classificationTypeAndSubTypesQryStr;
//gather all types and its corresponding subtypes
- Set<String> typeAndSubTypes = new HashSet<>();
- String typeAndSubTypesQryStr = null;
+ Set<String> typeAndSubTypes = new HashSet<>();
+ String typeAndSubTypesQryStr = null;
if (CollectionUtils.isNotEmpty(entityTypes)) {
for (AtlasEntityType entityType : entityTypes) {
-
if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = ALL_TYPE_QUERY;
+
break;
} else {
- Set<String> allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
+ Set<String> allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
+
typeAndSubTypes.addAll(allTypes);
}
}
@@ -216,6 +215,7 @@ public class SearchContext {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = "";
}
+
this.typeAndSubTypes = typeAndSubTypes;
this.typeAndSubTypesQryStr = typeAndSubTypesQryStr;
@@ -244,43 +244,77 @@ public class SearchContext {
}
}
- public SearchParameters getSearchParameters() { return searchParameters; }
+ public SearchParameters getSearchParameters() {
+ return searchParameters;
+ }
+
+ public AtlasTypeRegistry getTypeRegistry() {
+ return typeRegistry;
+ }
- public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; }
+ public AtlasGraph getGraph() {
+ return graph;
+ }
- public AtlasGraph getGraph() { return graph; }
+ public Set<String> getIndexedKeys() {
+ return indexedKeys;
+ }
- public Set<String> getIndexedKeys() { return indexedKeys; }
+ public Set<String> getEdgeIndexKeys() {
+ return edgeIndexKeys;
+ }
public void setEdgeIndexKeys(Set<String> edgeIndexKeys) {
this.edgeIndexKeys = edgeIndexKeys;
}
- public Set<String> getEdgeIndexKeys() { return edgeIndexKeys; }
-
- public Set<String> getEntityAttributes() { return entityAttributes; }
+ public Set<String> getEntityAttributes() {
+ return entityAttributes;
+ }
- public Set<String> getRelationAttributes() { return relationAttributes; }
+ public Set<String> getRelationAttributes() {
+ return relationAttributes;
+ }
- public Set<AtlasClassificationType> getClassificationTypes() { return classificationTypes; }
+ public Set<AtlasClassificationType> getClassificationTypes() {
+ return classificationTypes;
+ }
- public Set<String> getEntityTypeNames() { return typeAndSubTypes; }
+ public Set<String> getEntityTypeNames() {
+ return typeAndSubTypes;
+ }
- public Set<String> getClassificationTypeNames() { return classificationTypeAndSubTypes; }
+ public Set<String> getClassificationTypeNames() {
+ return classificationTypeAndSubTypes;
+ }
- public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; }
+ public String getEntityTypesQryStr() {
+ return typeAndSubTypesQryStr;
+ }
- public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; }
+ public String getClassificationTypesQryStr() {
+ return classificationTypeAndSubTypesQryStr;
+ }
- public Set<AtlasEntityType> getEntityTypes() { return entityTypes; }
+ public Set<AtlasEntityType> getEntityTypes() {
+ return entityTypes;
+ }
- public SearchProcessor getSearchProcessor() { return searchProcessor; }
+ public SearchProcessor getSearchProcessor() {
+ return searchProcessor;
+ }
- public Set<String> getClassificationNames() {return classificationNames;}
+ public Set<String> getClassificationNames() {
+ return classificationNames;
+ }
- public Integer getMarker() { return marker; }
+ public Integer getMarker() {
+ return marker;
+ }
- public Set<AtlasRelationshipType> getRelationshipTypes() { return relationshipTypes; }
+ public Set<AtlasRelationshipType> getRelationshipTypes() {
+ return relationshipTypes;
+ }
public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType);
@@ -300,9 +334,13 @@ public class SearchContext {
return ret;
}
- public boolean terminateSearch() { return terminateSearch; }
+ public boolean terminateSearch() {
+ return terminateSearch;
+ }
- public void terminateSearch(boolean terminateSearch) { this.terminateSearch = terminateSearch; }
+ public void terminateSearch(boolean terminateSearch) {
+ this.terminateSearch = terminateSearch;
+ }
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
@@ -323,6 +361,17 @@ public class SearchContext {
return toString(new StringBuilder()).toString();
}
+ public boolean excludeHeaderAttributes() {
+ return CollectionUtils.isNotEmpty(entityTypes) &&
+ searchParameters.getExcludeHeaderAttributes() &&
+ CollectionUtils.isNotEmpty(searchParameters.getAttributes()) &&
+ !hasRelationshipAttributes;
+ }
+
+ public boolean hasAttributeFilter(FilterCriteria filterCriteria) {
+ return filterCriteria != null && (CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
+ }
+
boolean needFullTextProcessor() {
return StringUtils.isNotEmpty(searchParameters.getQuery());
}
@@ -332,13 +381,14 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
- return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
+ return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch();
}
- boolean isWildCardSearch () {
+ boolean isWildCardSearch() {
if (CollectionUtils.isNotEmpty(classificationNames)) {
return classificationNames.stream().anyMatch(classification -> classification.contains(WILDCARD_CLASSIFICATIONS));
}
+
return false;
}
@@ -369,11 +419,13 @@ public class SearchContext {
}
String name = structType.getTypeName();
+
if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
name = ALL_ENTITY_TYPES;
} else if (name.equals(MATCH_ALL_CLASSIFICATION_TYPES.getTypeName())) {
name = ALL_CLASSIFICATION_TYPES;
}
+
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, name);
}
}
@@ -381,14 +433,15 @@ public class SearchContext {
private void validateAttributes() throws AtlasBaseException {
Set<String> attributes = searchParameters.getAttributes();
- if (CollectionUtils.isNotEmpty(attributes) && CollectionUtils.isNotEmpty(entityTypes)) {
+ if (CollectionUtils.isNotEmpty(attributes) && CollectionUtils.isNotEmpty(entityTypes)) {
AtlasEntityType entityType = entityTypes.iterator().next();
+
for (String attr : attributes) {
AtlasAttribute attribute = entityType.getAttribute(attr);
if (attribute == null) {
- attribute = entityType.getRelationshipAttribute(attr, null);
+ attribute = entityType.getRelationshipAttribute(attr, null);
hasRelationshipAttributes = attribute != null;
}
@@ -399,22 +452,6 @@ public class SearchContext {
}
}
- public boolean excludeHeaderAttributes() {
- if (CollectionUtils.isNotEmpty(entityTypes) &&
- searchParameters.getExcludeHeaderAttributes() &&
- CollectionUtils.isNotEmpty(searchParameters.getAttributes()) &&
- !hasRelationshipAttributes){
- return true;
- }
-
- return false;
- }
-
- public boolean hasAttributeFilter(FilterCriteria filterCriteria) {
- return filterCriteria != null &&
- (CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
- }
-
private void addProcessor(SearchProcessor processor) {
if (searchProcessor == null) {
searchProcessor = processor;
@@ -432,7 +469,7 @@ public class SearchContext {
ret = MATCH_ALL_CLASSIFIED;
} else if (StringUtils.equals(classificationName, MATCH_ALL_NOT_CLASSIFIED.getTypeName())) {
ret = MATCH_ALL_NOT_CLASSIFIED;
- } else if (StringUtils.equals(classificationName, ALL_CLASSIFICATION_TYPES)){
+ } else if (StringUtils.equals(classificationName, ALL_CLASSIFICATION_TYPES)) {
ret = MATCH_ALL_CLASSIFICATION_TYPES;
} else {
ret = typeRegistry.getClassificationTypeByName(classificationName);
@@ -443,8 +480,7 @@ public class SearchContext {
private Set<AtlasClassificationType> getClassificationTypes(Set<String> classificationNames) {
if (CollectionUtils.isNotEmpty(classificationNames)) {
- return classificationNames.stream().map(n ->
- getClassificationType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+ return classificationNames.stream().map(this::getClassificationType).filter(Objects::nonNull).collect(Collectors.toSet());
}
return null;
@@ -454,7 +490,7 @@ public class SearchContext {
Set<String> classificationNames = new HashSet<>();
if (StringUtils.isNotEmpty(classification)) {
- String[] types = classification.split(TYPENAME_DELIMITER);
+ String[] types = classification.split(TYPENAME_DELIMITER);
Set<String> names = new HashSet<>(Arrays.asList(types));
names.forEach(name -> {
@@ -467,7 +503,6 @@ public class SearchContext {
// Validate if the classification exists
if (CollectionUtils.isEmpty(classificationNames)) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classification);
-
} else if (classificationNames.size() != names.size()) {
names.removeAll(classificationNames);
@@ -479,33 +514,33 @@ public class SearchContext {
}
private AtlasEntityType getEntityType(String entityName) {
- return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES :
- typeRegistry.getEntityTypeByName(entityName);
+ return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES : typeRegistry.getEntityTypeByName(entityName);
}
private Set<AtlasEntityType> getEntityTypes(String typeName) throws AtlasBaseException {
-
Set<AtlasEntityType> entityTypes = null;
+
//split multiple typeNames by comma
if (StringUtils.isNotEmpty(typeName)) {
-
- String[] types = typeName.split(TYPENAME_DELIMITER);
+ String[] types = typeName.split(TYPENAME_DELIMITER);
Set<String> typeNames = new HashSet<>(Arrays.asList(types));
- entityTypes = typeNames.stream().map(n ->
- getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+
+ entityTypes = typeNames.stream().map(this::getEntityType).filter(Objects::nonNull).collect(Collectors.toSet());
// Validate if the type name is incorrect
if (CollectionUtils.isEmpty(entityTypes)) {
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME,typeName);
-
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, typeName);
} else if (entityTypes.size() != typeNames.size()) {
Set<String> validEntityTypes = new HashSet<>();
+
for (AtlasEntityType entityType : entityTypes) {
String name = entityType.getTypeName();
+
if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
validEntityTypes.add(ALL_ENTITY_TYPES);
continue;
}
+
validEntityTypes.add(entityType.getTypeName());
}
@@ -518,7 +553,7 @@ public class SearchContext {
return entityTypes;
}
- private void filterStructTypes(){
+ private void filterStructTypes() {
//if typeName contains ALL_ENTITY_TYPES, remove others as OR condition will not effect any other
if (CollectionUtils.isNotEmpty(entityTypes) && entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) {
entityTypes.clear();
@@ -543,7 +578,6 @@ public class SearchContext {
}
}
-
private AtlasVertex getGlossaryTermVertex(String termName) {
AtlasVertex ret = null;
@@ -551,8 +585,8 @@ public class SearchContext {
AtlasEntityType termType = getTermEntityType();
AtlasAttribute attrName = termType.getAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_QNAME);
AtlasGraphQuery query = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, termType.getTypeName())
- .has(attrName.getVertexPropertyName(), termName)
- .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
+ .has(attrName.getVertexPropertyName(), termName)
+ .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
Iterator<AtlasVertex> results = query.vertices().iterator();
@@ -568,15 +602,17 @@ public class SearchContext {
AtlasAttribute attr = termType.getRelationshipAttribute(TermSearchProcessor.ATLAS_GLOSSARY_TERM_ATTR_ASSIGNED_ENTITIES, EntityGraphRetriever.TERM_RELATION_NAME);
Iterator<AtlasEdge> edges = GraphHelper.getEdgesForLabel(glossaryTerm, attr.getRelationshipEdgeLabel(), attr.getRelationshipEdgeDirection());
- boolean excludeDeletedEntities = searchParameters.getExcludeDeletedEntities();
if (edges != null) {
- while (edges.hasNext()) {
- AtlasEdge edge = edges.next();
+ boolean excludeDeletedEntities = searchParameters.getExcludeDeletedEntities();
+ while (edges.hasNext()) {
+ AtlasEdge edge = edges.next();
AtlasVertex inVertex = edge.getInVertex();
+
if (excludeDeletedEntities && AtlasGraphUtilsV2.getState(inVertex) == AtlasEntity.Status.DELETED) {
continue;
}
+
ret.add(inVertex);
}
}
@@ -590,20 +626,20 @@ public class SearchContext {
private Set<AtlasRelationshipType> getRelationshipTypes(String relationship) throws AtlasBaseException {
Set<AtlasRelationshipType> relationshipTypes = null;
+
//split multiple typeNames by comma
if (StringUtils.isNotEmpty(relationship)) {
-
- String[] types = relationship.split(TYPENAME_DELIMITER);
+ String[] types = relationship.split(TYPENAME_DELIMITER);
Set<String> typeNames = new HashSet<>(Arrays.asList(types));
- relationshipTypes = typeNames.stream().map(n ->
- typeRegistry.getRelationshipTypeByName(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+
+ relationshipTypes = typeNames.stream().map(typeRegistry::getRelationshipTypeByName).filter(Objects::nonNull).collect(Collectors.toSet());
// Validate if the type name is incorrect
if (CollectionUtils.isEmpty(relationshipTypes)) {
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME,relationship);
-
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, relationship);
} else if (relationshipTypes.size() != typeNames.size()) {
Set<String> validEntityTypes = new HashSet<>();
+
for (AtlasRelationshipType entityType : relationshipTypes) {
validEntityTypes.add(entityType.getTypeName());
}
@@ -618,16 +654,15 @@ public class SearchContext {
}
public static class MarkerUtil {
- private final static int IDX_HASH_CODE = 0;
- private final static int IDX_OFFSET = 1;
-
- private final static String MARKER_DELIMITER = ":";
-
@VisibleForTesting
- final static String MARKER_START = "*";
+ static final String MARKER_START = "*";
@VisibleForTesting
- final static int MARKER_END = -1;
+ static final int MARKER_END = -1;
+
+ private static final int IDX_HASH_CODE = 0;
+ private static final int IDX_OFFSET = 1;
+ private static final String MARKER_DELIMITER = ":";
public static String getNextEncMarker(SearchParameters searchParameters, Integer nextOffset) {
if (nextOffset == null) {
@@ -639,6 +674,7 @@ public class SearchContext {
}
String value = searchParameters.hashCode() + MARKER_DELIMITER + nextOffset;
+
return Base64.getEncoder().encodeToString(value.getBytes());
}
@@ -648,6 +684,7 @@ public class SearchContext {
}
String encodedMarker = searchParameters.getMarker();
+
if (StringUtils.equals(encodedMarker, MARKER_START)) {
return 0;
}
@@ -655,17 +692,20 @@ public class SearchContext {
try {
byte[] inputMarkerBytes = Base64.getDecoder().decode(encodedMarker);
String inputMarker = new String(inputMarkerBytes);
+
if (StringUtils.isEmpty(inputMarker) || !inputMarker.contains(MARKER_DELIMITER)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid marker found! Marker does not contain delimiter: " + MARKER_DELIMITER);
}
String[] str = inputMarker.split(MARKER_DELIMITER);
- if (str == null || str.length != 2) {
+
+ if (str.length != 2) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid marker found! Decoding using delimiter did not yield correct result!");
}
int hashCode = Integer.parseInt(str[IDX_HASH_CODE]);
int currentHashCode = searchParameters.hashCode();
+
if (hashCode == currentHashCode && Integer.parseInt(str[IDX_OFFSET]) >= 0) {
return Integer.parseInt(str[IDX_OFFSET]);
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
index b35346ef5..d9208e311 100755
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchIndexer.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -18,7 +18,6 @@
package org.apache.atlas.discovery;
-
/**
* Interface for indexing types.
*/
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 49091dcf3..6594b88a9 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -34,8 +34,14 @@ import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
-import org.apache.atlas.type.*;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasEnumType;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.commons.collections.CollectionUtils;
@@ -51,99 +57,107 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.sql.Timestamp;
import java.time.LocalDateTime;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static org.apache.atlas.repository.Constants.*;
-import static org.apache.atlas.util.SearchPredicateUtil.*;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_VALUE_DELIMITER;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
+import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.util.SearchPredicateUtil.ElementAttributePredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getContainsPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getEQPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getEndsWithPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getGTEPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getGTPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getINPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getInRangePredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getIsNullPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getLIKEPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getLTEPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getLTPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getNEQPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getNotContainsPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getNotEmptyPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getNotNullPredicateGenerator;
+import static org.apache.atlas.util.SearchPredicateUtil.getStartsWithPredicateGenerator;
public abstract class SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(SearchProcessor.class);
- public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
- public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
- public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
- public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
- public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
- public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
- public static final String INDEX_SEARCH_PREFIX = AtlasGraphUtilsV2.getIndexSearchPrefix();
- public static final String AND_STR = " AND ";
- public static final String EMPTY_STRING = "";
- public static final String SPACE_STRING = " ";
- public static final String BRACE_OPEN_STR = "(";
- public static final String BRACE_CLOSE_STR = ")";
- public static final String ALL_TYPE_QUERY = "[* TO *]";
- public static final char CUSTOM_ATTR_SEPARATOR = '=';
- public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
- public static final String CUSTOM_ATTR_SEARCH_FORMAT_GRAPH = "\"%s\":\"%s\"";
- private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
+ public static final Pattern STRAY_AND_PATTERN = Pattern.compile("(AND\\s+)+\\)");
+ public static final Pattern STRAY_OR_PATTERN = Pattern.compile("(OR\\s+)+\\)");
+ public static final Pattern STRAY_ELIPSIS_PATTERN = Pattern.compile("(\\(\\s*)\\)");
+ public static final int MAX_RESULT_SIZE = getApplicationProperty(Constants.INDEX_SEARCH_MAX_RESULT_SET_SIZE, 150);
+ public static final int MAX_QUERY_STR_LENGTH_TYPES = getApplicationProperty(Constants.INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH, 512);
+ public static final int MAX_QUERY_STR_LENGTH_TAGS = getApplicationProperty(Constants.INDEX_SEARCH_TAGS_MAX_QUERY_STR_LENGTH, 512);
+ public static final String INDEX_SEARCH_PREFIX = AtlasGraphUtilsV2.getIndexSearchPrefix();
+ public static final String AND_STR = " AND ";
+ public static final String EMPTY_STRING = "";
+ public static final String SPACE_STRING = " ";
+ public static final String BRACE_OPEN_STR = "(";
+ public static final String BRACE_CLOSE_STR = ")";
+ public static final String ALL_TYPE_QUERY = "[* TO *]";
+ public static final char CUSTOM_ATTR_SEPARATOR = '=';
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT_GRAPH = "\"%s\":\"%s\"";
+
+ private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final Map<SearchParameters.Operator, ElementAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>();
- static
- {
- OPERATOR_MAP.put(SearchParameters.Operator.LT, INDEX_SEARCH_PREFIX + "\"%s\": [* TO %s}");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LT, getLTPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.GT, INDEX_SEARCH_PREFIX + "\"%s\": {%s TO *]");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.GT, getGTPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.LTE, INDEX_SEARCH_PREFIX + "\"%s\": [* TO %s]");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LTE, getLTEPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.GTE, INDEX_SEARCH_PREFIX + "\"%s\": [%s TO *]");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.GTE, getGTEPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.EQ, INDEX_SEARCH_PREFIX + "\"%s\": %s");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.EQ, getEQPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.NEQ, "(*:* -" + INDEX_SEARCH_PREFIX + "\"%s\": %s)");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NEQ, getNEQPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.IN, INDEX_SEARCH_PREFIX + "\"%s\": (%s)"); // this should be a list of quoted strings
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.IN, getINPredicateGenerator()); // this should be a list of quoted strings
-
- OPERATOR_MAP.put(SearchParameters.Operator.LIKE, INDEX_SEARCH_PREFIX + "\"%s\": (%s)"); // this should be regex pattern
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LIKE, getLIKEPredicateGenerator()); // this should be regex pattern
-
- OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, INDEX_SEARCH_PREFIX + "\"%s\": (%s*)");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.STARTS_WITH, getStartsWithPredicateGenerator());
-
- OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, INDEX_SEARCH_PREFIX + "\"%s\": (*%s)");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.ENDS_WITH, getEndsWithPredicateGenerator());
+ protected final SearchContext context;
+ protected SearchProcessor nextProcessor;
+ protected Predicate inMemoryPredicate;
+ protected GraphIndexQueryBuilder graphIndexQueryBuilder;
+ protected Integer nextOffset;
- OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, INDEX_SEARCH_PREFIX + "\"%s\": (*%s*)");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.CONTAINS, getContainsPredicateGenerator());
+ protected SearchProcessor(SearchContext context) {
+ this.context = context;
+ this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context);
+ }
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_CONTAINS, getNotContainsPredicateGenerator());
+ protected static Iterator<AtlasIndexQuery.Result> executeIndexQuery(SearchContext context, AtlasIndexQuery indexQuery, int qryOffset, int limit) {
+ String sortBy = getSortByAttribute(context);
- // TODO: Add contains any, contains all mappings here
+ if (sortBy != null && !sortBy.isEmpty()) {
+ Order sortOrder = getSortOrderAttribute(context);
- OPERATOR_MAP.put(SearchParameters.Operator.IS_NULL, "(*:* NOT " + INDEX_SEARCH_PREFIX + "\"%s\":[* TO *])");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.IS_NULL, getIsNullPredicateGenerator());
+ return indexQuery.vertices(qryOffset, limit, sortBy, sortOrder);
+ }
- OPERATOR_MAP.put(SearchParameters.Operator.NOT_NULL, INDEX_SEARCH_PREFIX + "\"%s\":[* TO *]");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_NULL, getNotNullPredicateGenerator());
+ return indexQuery.vertices(qryOffset, limit);
+ }
- OPERATOR_MAP.put(SearchParameters.Operator.NOT_EMPTY, INDEX_SEARCH_PREFIX + "\"%s\":[* TO *]");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_EMPTY, getNotEmptyPredicateGenerator());
+ protected static Iterator<AtlasIndexQuery.Result> executeIndexQueryForEdge(SearchContext context, AtlasIndexQuery indexQuery, int qryOffset, int limit) {
+ final AtlasRelationshipType relationshipType = context.getRelationshipTypes().iterator().next();
+ AtlasStructType.AtlasAttribute sortByAttribute = relationshipType.getAttribute(context.getSearchParameters().getSortBy());
- OPERATOR_MAP.put(SearchParameters.Operator.TIME_RANGE, INDEX_SEARCH_PREFIX + "\"%s\": [%s TO %s]");
- OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.TIME_RANGE, getInRangePredicateGenerator());
- }
+ if (sortByAttribute != null && StringUtils.isNotEmpty(sortByAttribute.getVertexPropertyName())) {
+ Order sortOrder = getSortOrderAttribute(context);
- protected final SearchContext context;
- protected SearchProcessor nextProcessor;
- protected Predicate inMemoryPredicate;
- protected GraphIndexQueryBuilder graphIndexQueryBuilder;
- protected Integer nextOffset;
+ return indexQuery.edges(qryOffset, limit, sortByAttribute.getVertexPropertyName(), sortOrder);
+ }
- protected SearchProcessor(SearchContext context) {
- this.context = context;
- this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context);
+ return indexQuery.edges(qryOffset, limit);
}
public void addProcessor(SearchProcessor processor) {
@@ -159,13 +173,144 @@ public abstract class SearchProcessor {
}
public abstract List<AtlasVertex> execute();
+
public abstract long getResultCount();
+ public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap) {
+ if (nextProcessor != null && MapUtils.isNotEmpty(offsetEntityVertexMap)) {
+ return nextProcessor.filter(offsetEntityVertexMap);
+ }
+
+ return offsetEntityVertexMap;
+ }
+
+ public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap, Predicate predicate) {
+ if (predicate != null) {
+ offsetEntityVertexMap = offsetEntityVertexMap.entrySet()
+ .stream()
+ .filter(x -> predicate.evaluate(x.getValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (x, y) -> y, LinkedHashMap::new));
+ }
+
+ return offsetEntityVertexMap;
+ }
+
+ @VisibleForTesting
+ public FilterCriteria processDateRange(FilterCriteria criteria) {
+ String attrName = criteria.getAttributeName();
+ SearchParameters.Operator op = criteria.getOperator();
+ String attrVal = criteria.getAttributeValue();
+ FilterCriteria ret = new FilterCriteria();
+ final LocalDateTime now = LocalDateTime.now();
+ final LocalDateTime startTime;
+ final LocalDateTime endTime;
+
+ switch (attrVal) {
+ case "LAST_7_DAYS":
+ startTime = now.minusDays(6).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusDays(7).minusNanos(1);
+ break;
+
+ case "LAST_30_DAYS":
+ startTime = now.minusDays(29).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusDays(30).minusNanos(1);
+ break;
+
+ case "LAST_MONTH":
+ startTime = now.minusMonths(1).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(1).minusNanos(1);
+ break;
+
+ case "THIS_MONTH":
+ startTime = now.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(1).minusNanos(1);
+ break;
+
+ case "TODAY":
+ startTime = now.withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusDays(1).minusNanos(1);
+ break;
+
+ case "YESTERDAY":
+ startTime = now.minusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusDays(1).minusNanos(1);
+ break;
+
+ case "THIS_YEAR":
+ startTime = now.withDayOfYear(1).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusYears(1).minusNanos(1);
+ break;
+
+ case "LAST_YEAR":
+ startTime = now.minusYears(1).withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusYears(1).minusNanos(1);
+ break;
+
+ case "THIS_QUARTER":
+ startTime = now.minusMonths(1).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(3).minusNanos(1);
+ break;
+
+ case "LAST_QUARTER":
+ startTime = now.minusMonths(4).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(3).minusNanos(1);
+ break;
+
+ case "LAST_3_MONTHS":
+ startTime = now.minusMonths(3).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(3).minusNanos(1);
+ break;
+
+ case "LAST_6_MONTHS":
+ startTime = now.minusMonths(6).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(6).minusNanos(1);
+ break;
+
+ case "LAST_12_MONTHS":
+ startTime = now.minusMonths(12).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
+ endTime = startTime.plusMonths(12).minusNanos(1);
+ break;
+
+ default:
+ startTime = null;
+ endTime = null;
+ break;
+ }
+
+ if (startTime == null || endTime == null) {
+ String[] rangeAttr = attrVal.split(ATTRIBUTE_VALUE_DELIMITER);
+
+ if (rangeAttr.length != 2) {
+ LOG.error("Separator invalid");
+ } else {
+ try {
+ Long.parseLong(String.valueOf(rangeAttr[0]));
+ Long.parseLong(String.valueOf(rangeAttr[1]));
+ } catch (NumberFormatException e) {
+ boolean numeric = false;
+
+ if (!numeric) {
+ LOG.error("Attributes passed need to be LONG", e);
+ }
+ }
+ }
+ } else {
+ attrVal = Timestamp.valueOf(startTime).getTime() + ATTRIBUTE_VALUE_DELIMITER + Timestamp.valueOf(endTime).getTime();
+ }
+
+ ret.setAttributeName(attrName);
+ ret.setOperator(op);
+ ret.setAttributeValue(attrVal);
+
+ return ret;
+ }
+
protected boolean isEntityRootType() {
//always size will be one if in case of _ALL_ENTITY_TYPES
if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
return context.getEntityTypes().iterator().next() == SearchContext.MATCH_ALL_ENTITY_TYPES;
}
+
return false;
}
@@ -174,6 +319,7 @@ public abstract class SearchProcessor {
if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
return context.getClassificationTypes().iterator().next() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
}
+
return false;
}
@@ -183,29 +329,31 @@ public abstract class SearchProcessor {
protected boolean isPipeSeparatedSystemAttribute(String attrName) {
return StringUtils.equals(attrName, CLASSIFICATION_NAMES_KEY) ||
- StringUtils.equals(attrName, PROPAGATED_CLASSIFICATION_NAMES_KEY) ||
- StringUtils.equals(attrName, LABELS_PROPERTY_KEY) ||
- StringUtils.equals(attrName, CUSTOM_ATTRIBUTES_PROPERTY_KEY);
+ StringUtils.equals(attrName, PROPAGATED_CLASSIFICATION_NAMES_KEY) ||
+ StringUtils.equals(attrName, LABELS_PROPERTY_KEY) ||
+ StringUtils.equals(attrName, CUSTOM_ATTRIBUTES_PROPERTY_KEY);
}
protected int collectResultVertices(final List<AtlasVertex> ret, final int startIdx, final int limit, int resultIdx, final Map<Integer, AtlasVertex> offsetEntityVertexMap, Integer marker) {
- int lastOffset = resultIdx;
+ int lastOffset = resultIdx;
- for (Map.Entry<Integer, AtlasVertex> offsetToEntity : offsetEntityVertexMap.entrySet()) {
- resultIdx++;
+ for (Map.Entry<Integer, AtlasVertex> offsetToEntity : offsetEntityVertexMap.entrySet()) {
+ resultIdx++;
- if (resultIdx <= startIdx) {
- continue;
- }
+ if (resultIdx <= startIdx) {
+ continue;
+ }
- lastOffset = offsetToEntity.getKey();
- ret.add(offsetToEntity.getValue());
+ lastOffset = offsetToEntity.getKey();
- if (ret.size() == limit) {
- break;
- }
+ ret.add(offsetToEntity.getValue());
+
+ if (ret.size() == limit) {
+ break;
}
- return marker == null ? resultIdx : lastOffset;
+ }
+
+ return marker == null ? resultIdx : lastOffset;
}
protected int collectResultEdges(final List<AtlasEdge> ret, final int startIdx, final int limit, int resultIdx, final Map<Integer, AtlasEdge> offsetEdgeMap, Integer marker) {
@@ -219,53 +367,40 @@ public abstract class SearchProcessor {
}
lastOffset = offsetToEdge.getKey();
+
ret.add(offsetToEdge.getValue());
if (ret.size() == limit) {
break;
}
}
- return marker == null ? resultIdx : lastOffset;
- }
-
- public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap) {
- if (nextProcessor != null && MapUtils.isNotEmpty(offsetEntityVertexMap)) {
- return nextProcessor.filter(offsetEntityVertexMap);
- }
- return offsetEntityVertexMap;
- }
- public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap, Predicate predicate) {
- if (predicate != null) {
- offsetEntityVertexMap = offsetEntityVertexMap.entrySet()
- .stream()
- .filter(x -> predicate.evaluate(x.getValue()))
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (x, y) -> y, LinkedHashMap::new));
- }
- return offsetEntityVertexMap;
+ return marker == null ? resultIdx : lastOffset;
}
protected Predicate buildTraitPredict(Set<AtlasClassificationType> classificationTypes) {
- Predicate traitPredicate;
+ Predicate traitPredicate;
AtlasClassificationType classificationType = null;
+
if (CollectionUtils.isNotEmpty(classificationTypes)) {
classificationType = classificationTypes.iterator().next();
}
if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (context.isWildCardSearch()) {
//For wildcard search __classificationNames which of String type is taken instead of _traitNames which is of Array type
- Set<String> classificationNames = context.getClassificationNames();
- List<Predicate> predicates = new ArrayList<>();
+ Set<String> classificationNames = context.getClassificationNames();
+ List<Predicate> predicates = new ArrayList<>();
classificationNames.forEach(classificationName -> {
//No need to escape, as classification Names only support letters,numbers,space and underscore
String regexString = getRegexString("\\|" + classificationName + "\\|");
+
predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class));
predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class));
});
@@ -273,12 +408,12 @@ public abstract class SearchProcessor {
traitPredicate = PredicateUtils.anyPredicate(predicates);
} else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class),
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class));
+ SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class));
}
+
return traitPredicate;
}
-
protected void processSearchAttributes(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
if (CollectionUtils.isEmpty(structTypes) || filterCriteria == null) {
return;
@@ -299,7 +434,6 @@ public abstract class SearchProcessor {
// vertex property IS_INCOMPLETE_PROPERTY_KEY will be set to INCOMPLETE_ENTITY_VALUE
// when entity is not incomplete (i.e. not a shell entity):
// vertex property IS_INCOMPLETE_PROPERTY_KEY will not be set
-
String attributeValue = filterCriteria.getAttributeValue();
switch (filterCriteria.getOperator()) {
@@ -310,7 +444,7 @@ public abstract class SearchProcessor {
filterCriteria.setOperator(SearchParameters.Operator.EQ);
filterCriteria.setAttributeValue(Constants.INCOMPLETE_ENTITY_VALUE.toString());
}
- break;
+ break;
case NEQ:
if (attributeValue == null || StringUtils.equals(attributeValue, "0") || StringUtils.equalsIgnoreCase(attributeValue, "false")) {
@@ -319,18 +453,19 @@ public abstract class SearchProcessor {
} else {
filterCriteria.setOperator(SearchParameters.Operator.IS_NULL);
}
- break;
+ break;
case NOT_NULL:
filterCriteria.setOperator(SearchParameters.Operator.EQ);
filterCriteria.setAttributeValue(Constants.INCOMPLETE_ENTITY_VALUE.toString());
- break;
+ break;
}
}
try {
for (AtlasStructType structType : structTypes) {
String qualifiedName = structType.getVertexPropertyName(attributeName);
+
if (isIndexSearchable(filterCriteria, structType)) {
indexFiltered.add(qualifiedName);
} else {
@@ -375,8 +510,6 @@ public abstract class SearchProcessor {
boolean ret = true;
Condition filterCondition = filterCriteria.getCondition();
List<FilterCriteria> criterion = filterCriteria.getCriterion();
- Set<String> indexedKeys = context.getIndexedKeys();
-
if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
insideOrCondition = insideOrCondition || filterCondition == Condition.OR;
@@ -405,10 +538,10 @@ public abstract class SearchProcessor {
return ret;
}
- protected LinkedHashMap<Integer,AtlasVertex> filterWhiteSpaceClassification(LinkedHashMap<Integer,AtlasVertex> offsetEntityVertexMap) {
+ protected LinkedHashMap<Integer, AtlasVertex> filterWhiteSpaceClassification(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap) {
if (offsetEntityVertexMap != null) {
- final Iterator<Map.Entry<Integer, AtlasVertex>> it = offsetEntityVertexMap.entrySet().iterator();
- final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
+ final Iterator<Map.Entry<Integer, AtlasVertex>> it = offsetEntityVertexMap.entrySet().iterator();
+ final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
while (it.hasNext()) {
AtlasVertex entityVertex = it.next().getValue();
@@ -431,14 +564,13 @@ public abstract class SearchProcessor {
it.remove();
}
}
+
return offsetEntityVertexMap;
}
protected void constructFilterQuery(StringBuilder indexQuery, Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
if (filterCriteria != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing Filters");
- }
+ LOG.debug("Processing Filters");
String filterQuery = toIndexQuery(structTypes, filterCriteria, indexAttributes, 0);
@@ -454,13 +586,13 @@ public abstract class SearchProcessor {
protected Predicate constructInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
Predicate ret = null;
+
if (filterCriteria != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing Filters");
- }
+ LOG.debug("Processing Filters");
ret = toInMemoryPredicate(structTypes, filterCriteria, indexAttributes);
}
+
return ret;
}
@@ -469,8 +601,7 @@ public abstract class SearchProcessor {
FilterCriteria.Condition condition = filterCriteria.getCondition();
if (condition != null) {
- StringBuilder orQuery = new StringBuilder();
-
+ StringBuilder orQuery = new StringBuilder();
List<FilterCriteria> criterion = filterCriteria.getCriterion();
for (int i = 0; i < criterion.size(); i++) {
@@ -502,23 +633,222 @@ public abstract class SearchProcessor {
} else {
LOG.warn("Ignoring unknown attribute {}.{}", structType.getTypeName(), attributeName);
}
-
}
}
}
- private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType structType) throws AtlasBaseException {
- String attributeName = filterCriteria.getAttributeName();
- String attributeValue = filterCriteria.getAttributeValue();
- AtlasType attributeType = structType.getAttributeType(attributeName);
- String typeName = attributeType.getTypeName();
- String qualifiedName = structType.getVertexPropertyName(attributeName);
- Set<String> indexedKeys = context.getIndexedKeys();
- Set<String> edgeIndexedKeys = context.getEdgeIndexKeys();
- boolean ret = (indexedKeys != null && (indexedKeys.contains(qualifiedName)) || (edgeIndexedKeys != null && edgeIndexedKeys.contains(qualifiedName)));
+ protected AtlasGraphQuery toGraphFilterQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
+ Set<String> filterAttributes = new HashSet<>(graphAttributes);
- SearchParameters.Operator operator = filterCriteria.getOperator();
- AtlasStructDef.AtlasAttributeDef.IndexType indexType = structType.getAttributeDef(attributeName).getIndexType();
+ if (criteria != null) {
+ if (criteria.getCondition() != null) {
+ if (criteria.getCondition() == Condition.AND) {
+ for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+ AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
+
+ query.addConditionsFrom(nestedQuery);
+ }
+ } else {
+ List<AtlasGraphQuery> orConditions = new LinkedList<>();
+
+ for (FilterCriteria filterCriteria : criteria.getCriterion()) {
+ AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
+
+ orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
+ }
+
+ if (!orConditions.isEmpty()) {
+ query.or(orConditions);
+ }
+ }
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ List<AtlasGraphQuery> queries = new ArrayList<>();
+
+ for (AtlasStructType structType : structTypes) {
+ String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName());
+
+ if (filterAttributes.contains(qualifiedName)) {
+ String attrName = criteria.getAttributeName();
+ String attrValue = criteria.getAttributeValue();
+ SearchParameters.Operator operator = criteria.getOperator();
+
+ //process attribute value and attribute operator for pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+
+ attrValue = processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
+ }
+
+ AtlasGraphQuery innerQry = context.getGraph().query().createChildQuery();
+
+ switch (operator) {
+ case LT:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
+ break;
+ case LTE:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
+ break;
+ case GT:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
+ break;
+ case GTE:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
+ break;
+ case EQ:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
+ break;
+ case NEQ:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
+ break;
+ case LIKE:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
+ break;
+ case CONTAINS:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
+ break;
+ case STARTS_WITH:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
+ break;
+ case ENDS_WITH:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
+ break;
+ case IS_NULL:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
+ break;
+ case NOT_NULL:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
+ break;
+ case NOT_CONTAINS:
+ break;
+ default:
+ LOG.warn("{}: unsupported operator. Ignored", operator);
+ break;
+ }
+
+ queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry));
+
+ filterAttributes.remove(qualifiedName);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(queries)) {
+ if (queries.size() > 1) {
+ return context.getGraph().query().createChildQuery().or(queries);
+ } else {
+ return queries.iterator().next();
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
+ }
+ }
+
+ return query;
+ }
+
+ protected Collection<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, Collection<AtlasVertex> vertices) {
+ if (idxQueryResult != null) {
+ while (idxQueryResult.hasNext()) {
+ AtlasVertex vertex = idxQueryResult.next().getVertex();
+
+ vertices.add(vertex);
+ }
+ }
+
+ return vertices;
+ }
+
+ protected LinkedHashMap<Integer, AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap, int qryOffset) {
+ if (idxQueryResult != null) {
+ while (idxQueryResult.hasNext()) {
+ AtlasVertex vertex = idxQueryResult.next().getVertex();
+
+ offsetEntityVertexMap.put(qryOffset++, vertex);
+ }
+ }
+
+ return offsetEntityVertexMap;
+ }
+
+ protected LinkedHashMap<Integer, AtlasEdge> getEdgesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap, int qryOffset) {
+ if (idxQueryResult != null) {
+ while (idxQueryResult.hasNext()) {
+ AtlasEdge edge = idxQueryResult.next().getEdge();
+
+ offsetEdgeMap.put(qryOffset++, edge);
+ }
+ }
+
+ return offsetEdgeMap;
+ }
+
+ protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, Collection<AtlasVertex> vertices) {
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ AtlasVertex vertex = iterator.next();
+
+ vertices.add(vertex);
+ }
+ }
+
+ return vertices;
+ }
+
+ protected LinkedHashMap<Integer, AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap, int qryOffset) {
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ AtlasVertex vertex = iterator.next();
+
+ offsetEntityVertexMap.put(qryOffset++, vertex);
+ }
+ }
+
+ return offsetEntityVertexMap;
+ }
+
+ protected LinkedHashMap<Integer, AtlasEdge> getEdges(Iterator<AtlasEdge> iterator, LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap, int qryOffset) {
+ if (iterator != null) {
+ while (iterator.hasNext()) {
+ AtlasEdge edge = iterator.next();
+
+ offsetEdgeMap.put(qryOffset++, edge);
+ }
+ }
+
+ return offsetEdgeMap;
+ }
+
+ protected Set<String> getGuids(List<AtlasVertex> vertices) {
+ Set<String> ret = new HashSet<>();
+
+ if (vertices != null) {
+ for (AtlasVertex vertex : vertices) {
+ String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex);
+
+ if (StringUtils.isNotEmpty(guid)) {
+ ret.add(guid);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType structType) throws AtlasBaseException {
+ String attributeName = filterCriteria.getAttributeName();
+ String attributeValue = filterCriteria.getAttributeValue();
+ AtlasType attributeType = structType.getAttributeType(attributeName);
+ String typeName = attributeType.getTypeName();
+ String qualifiedName = structType.getVertexPropertyName(attributeName);
+ Set<String> indexedKeys = context.getIndexedKeys();
+ Set<String> edgeIndexedKeys = context.getEdgeIndexKeys();
+ boolean ret = (indexedKeys != null && (indexedKeys.contains(qualifiedName)) || (edgeIndexedKeys != null && edgeIndexedKeys.contains(qualifiedName)));
+
+ SearchParameters.Operator operator = filterCriteria.getOperator();
+ AtlasStructDef.AtlasAttributeDef.IndexType indexType = structType.getAttributeDef(attributeName).getIndexType();
if (ret) { // index exists
// for string type attributes, don't use index query in the following cases:
@@ -526,25 +856,19 @@ public abstract class SearchProcessor {
// - value-to-compare has special characters
if (AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(typeName)) {
if (operator == SearchParameters.Operator.NEQ || operator == SearchParameters.Operator.NOT_CONTAINS) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("{} operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", operator, qualifiedName);
- }
+ LOG.debug("{} operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", operator, qualifiedName);
ret = false;
} else if (operator == SearchParameters.Operator.CONTAINS && AtlasAttribute.hastokenizeChar(attributeValue) && indexType == null) { // indexType = TEXT
- if (LOG.isDebugEnabled()) {
- LOG.debug("{} operator found for string (TEXT) attribute {} and special characters found in filter value {}, deferring to in-memory or graph query (might cause poor performance)", operator, qualifiedName, attributeValue);
- }
+ LOG.debug("{} operator found for string (TEXT) attribute {} and special characters found in filter value {}, deferring to in-memory or graph query (might cause poor performance)", operator, qualifiedName, attributeValue);
ret = false;
}
}
}
- if (LOG.isDebugEnabled()) {
- if (!ret) {
- LOG.debug("Not using index query for: attribute='{}', operator='{}', value='{}'", qualifiedName, operator, attributeValue);
- }
+ if (!ret) {
+ LOG.debug("Not using index query for: attribute='{}', operator='{}', value='{}'", qualifiedName, operator, attributeValue);
}
return ret;
@@ -555,10 +879,9 @@ public abstract class SearchProcessor {
}
private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) {
- Set<String> filterAttributes = new HashSet<>();
- filterAttributes.addAll(indexAttributes);
+ Set<String> filterAttributes = new HashSet<>(indexAttributes);
+ Condition condition = criteria.getCondition();
- Condition condition = criteria.getCondition();
if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder();
@@ -569,16 +892,18 @@ public abstract class SearchProcessor {
if (nestedExpression.length() > 0) {
nestedExpression.append(SPACE_STRING).append(condition).append(SPACE_STRING);
}
+
nestedExpression.append(nestedQuery);
}
}
boolean needSurroundingBraces = level != 0 || (condition == Condition.OR && criteria.getCriterion().size() > 1);
+
if (nestedExpression.length() > 0) {
return sb.append(needSurroundingBraces ? BRACE_OPEN_STR : EMPTY_STRING)
- .append(nestedExpression)
- .append(needSurroundingBraces ? BRACE_CLOSE_STR : EMPTY_STRING)
- .toString();
+ .append(nestedExpression)
+ .append(needSurroundingBraces ? BRACE_CLOSE_STR : EMPTY_STRING)
+ .toString();
} else {
return EMPTY_STRING;
}
@@ -587,12 +912,15 @@ public abstract class SearchProcessor {
if (criteria.getOperator() == SearchParameters.Operator.TIME_RANGE) {
criteria = processDateRange(criteria);
}
+
ArrayList<String> orExpQuery = new ArrayList<>();
+
for (AtlasStructType structType : structTypes) {
String name = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(name)) {
String nestedQuery = toIndexExpression(structType, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
+
orExpQuery.add(nestedQuery);
filterAttributes.remove(name);
}
@@ -600,7 +928,8 @@ public abstract class SearchProcessor {
if (CollectionUtils.isNotEmpty(orExpQuery)) {
if (orExpQuery.size() > 1) {
- String orExpStr = StringUtils.join(orExpQuery, " "+Condition.OR.name()+" ");
+ String orExpStr = StringUtils.join(orExpQuery, " " + Condition.OR.name() + " ");
+
return BRACE_OPEN_STR + " " + orExpStr + " " + BRACE_CLOSE_STR;
} else {
return orExpQuery.iterator().next();
@@ -612,12 +941,12 @@ public abstract class SearchProcessor {
LOG.warn(e.getMessage());
}
}
+
return EMPTY_STRING;
}
private Predicate toInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes) {
- Set<String> filterAttributes = new HashSet<>();
- filterAttributes.addAll(indexAttributes);
+ Set<String> filterAttributes = new HashSet<>(indexAttributes);
if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
List<Predicate> predicates = new ArrayList<>();
@@ -640,23 +969,26 @@ public abstract class SearchProcessor {
} else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
try {
ArrayList<Predicate> predicates = new ArrayList<>();
+
for (AtlasStructType structType : structTypes) {
String name = structType.getVertexPropertyName(criteria.getAttributeName());
if (filterAttributes.contains(name)) {
- String attrName = criteria.getAttributeName();
- String attrValue = criteria.getAttributeValue();
- SearchParameters.Operator operator = criteria.getOperator();
+ String attrName = criteria.getAttributeName();
+ String attrValue = criteria.getAttributeValue();
+ SearchParameters.Operator operator = criteria.getOperator();
if (operator == SearchParameters.Operator.TIME_RANGE) {
FilterCriteria processedRangeCriteria = processDateRange(criteria);
- attrValue = processedRangeCriteria.getAttributeValue();
+
+ attrValue = processedRangeCriteria.getAttributeValue();
}
//process attribute value and attribute operator for pipeSeperated fields
if (isPipeSeparatedSystemAttribute(attrName)) {
FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
- attrValue = processedCriteria.getAttributeValue();
- operator = processedCriteria.getOperator();
+
+ attrValue = processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
}
predicates.add(toInMemoryPredicate(structType, attrName, operator, attrValue));
@@ -675,120 +1007,11 @@ public abstract class SearchProcessor {
LOG.warn(e.getMessage());
}
}
- return null;
- }
-
- @VisibleForTesting
- public FilterCriteria processDateRange(FilterCriteria criteria) {
- String attrName = criteria.getAttributeName();
- SearchParameters.Operator op = criteria.getOperator();
- String attrVal = criteria.getAttributeValue();
- FilterCriteria ret = new FilterCriteria();
- final LocalDateTime now = LocalDateTime.now();
- final LocalDateTime startTime;
- final LocalDateTime endTime;
-
- switch (attrVal) {
- case "LAST_7_DAYS":
- startTime = now.minusDays(6).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusDays(7).minusNanos(1);
- break;
-
- case "LAST_30_DAYS":
- startTime = now.minusDays(29).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusDays(30).minusNanos(1);
- break;
-
- case "LAST_MONTH":
- startTime = now.minusMonths(1).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(1).minusNanos(1);
- break;
-
- case "THIS_MONTH":
- startTime = now.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(1).minusNanos(1);
- break;
-
- case "TODAY":
- startTime = now.withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusDays(1).minusNanos(1);
- break;
-
- case "YESTERDAY":
- startTime = now.minusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusDays(1).minusNanos(1);
- break;
-
- case "THIS_YEAR":
- startTime = now.withDayOfYear(1).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusYears(1).minusNanos(1);
- break;
-
- case "LAST_YEAR":
- startTime = now.minusYears(1).withDayOfYear(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusYears(1).minusNanos(1);
- break;
-
- case "THIS_QUARTER":
- startTime = now.minusMonths(1).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(3).minusNanos(1);
- break;
-
- case "LAST_QUARTER":
- startTime = now.minusMonths(4).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(3).minusNanos(1);
- break;
-
- case "LAST_3_MONTHS":
- startTime = now.minusMonths(3).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(3).minusNanos(1);
- break;
-
- case "LAST_6_MONTHS":
- startTime = now.minusMonths(6).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(6).minusNanos(1);
- break;
-
- case "LAST_12_MONTHS":
- startTime = now.minusMonths(12).withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
- endTime = startTime.plusMonths(12).minusNanos(1);
- break;
-
- default:
- startTime = null;
- endTime = null;
- break;
- }
-
- if (startTime == null || endTime == null) {
- String[] rangeAttr = attrVal.split(ATTRIBUTE_VALUE_DELIMITER);
- boolean numeric = true;
- if (rangeAttr.length != 2) {
- LOG.error("Separator invalid");
- } else {
- try {
- Long parsestartTime = Long.parseLong(String.valueOf(rangeAttr[0]));
- Long parseendTime = Long.parseLong(String.valueOf(rangeAttr[1]));
- } catch (NumberFormatException e) {
- numeric = false;
- if (!numeric) {
- LOG.error("Attributes passed need to be LONG");
- }
- }
- }
- }else {
- attrVal = Timestamp.valueOf(startTime).getTime() + ATTRIBUTE_VALUE_DELIMITER + Timestamp.valueOf(endTime).getTime();
- }
- ret.setAttributeName(attrName);
- ret.setOperator(op);
- ret.setAttributeValue(attrVal);
-
- return ret;
+ return null;
}
private FilterCriteria processPipeSeperatedSystemAttribute(String attrName, SearchParameters.Operator op, String attrVal) {
-
FilterCriteria ret = new FilterCriteria();
if (op != null && attrVal != null) {
@@ -825,25 +1048,27 @@ public abstract class SearchProcessor {
return ret;
}
- private String toIndexExpression(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) throws AtlasBaseException{
+ private String toIndexExpression(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) throws AtlasBaseException {
String ret = EMPTY_STRING;
try {
if (OPERATOR_MAP.get(op) != null) {
- String rangeStart = "";
- String rangeEnd = "";
String qualifiedName = type.getVertexPropertyName(attrName);
+
if (op == SearchParameters.Operator.TIME_RANGE) {
String[] parts = attrVal.split(ATTRIBUTE_VALUE_DELIMITER);
+
if (parts.length == 2) {
- rangeStart = parts[0];
- rangeEnd = parts[1];
+ String rangeStart = parts[0];
+ String rangeEnd = parts[1];
+
String rangeStartIndexQueryValue = AtlasAttribute.escapeIndexQueryValue(rangeStart);
- String rangeEndIndexQueryValue = AtlasAttribute.escapeIndexQueryValue(rangeEnd);
+ String rangeEndIndexQueryValue = AtlasAttribute.escapeIndexQueryValue(rangeEnd);
+
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, rangeStartIndexQueryValue, rangeEndIndexQueryValue);
}
} else {
- String escapeIndexQueryValue;
+ String escapeIndexQueryValue;
boolean replaceWildcardChar = false;
AtlasStructDef.AtlasAttributeDef def = type.getAttributeDef(attrName);
@@ -853,17 +1078,17 @@ public abstract class SearchProcessor {
if (!isPipeSeparatedSystemAttribute(attrName)
&& (op == SearchParameters.Operator.CONTAINS || op == SearchParameters.Operator.STARTS_WITH || op == SearchParameters.Operator.ENDS_WITH)
&& def.getTypeName().equalsIgnoreCase(AtlasBaseTypeDef.ATLAS_TYPE_STRING)) {
-
if (def.getIndexType() == null && AtlasAttribute.hastokenizeChar(attrVal)) {
replaceWildcardChar = true;
}
- escapeIndexQueryValue = AtlasAttribute.escapeIndexQueryValue(attrVal, false, false);
+ escapeIndexQueryValue = AtlasAttribute.escapeIndexQueryValue(attrVal, false, false);
} else {
escapeIndexQueryValue = AtlasAttribute.escapeIndexQueryValue(attrVal);
}
String operatorStr = OPERATOR_MAP.get(op);
+
if (replaceWildcardChar) {
operatorStr = operatorStr.replace("*", "");
}
@@ -906,15 +1131,15 @@ public abstract class SearchProcessor {
}
private Predicate toInMemoryPredicate(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) {
- Predicate ret = null;
- AtlasAttribute attribute = type.getAttribute(attrName);
+ Predicate ret = null;
+ AtlasAttribute attribute = type.getAttribute(attrName);
ElementAttributePredicateGenerator predicate = OPERATOR_PREDICATE_MAP.get(op);
if (attribute != null && predicate != null) {
- final AtlasType attrType = attribute.getAttributeType();
- final Class attrClass;
+ final AtlasType attrType = attribute.getAttributeType();
+ final Class<?> attrClass;
final Object attrValue;
- Object attrValue2 = null;
+ Object attrValue2 = null;
// Some operators support null comparison, thus the parsing has to be conditional
switch (attrType.getTypeName()) {
@@ -945,15 +1170,19 @@ public abstract class SearchProcessor {
case AtlasBaseTypeDef.ATLAS_TYPE_LONG:
case AtlasBaseTypeDef.ATLAS_TYPE_DATE:
attrClass = Long.class;
+
String rangeStart = "";
- String rangeEnd = "";
+ String rangeEnd = "";
+
if (op == SearchParameters.Operator.TIME_RANGE) {
String[] parts = attrVal.split(ATTRIBUTE_VALUE_DELIMITER);
+
if (parts.length == 2) {
rangeStart = parts[0];
rangeEnd = parts[1];
}
}
+
if (StringUtils.isNotEmpty(rangeStart) && StringUtils.isNotEmpty(rangeEnd)) {
attrValue = Long.parseLong(rangeStart);
attrValue2 = Long.parseLong(rangeEnd);
@@ -987,133 +1216,24 @@ public abstract class SearchProcessor {
}
String vertexPropertyName = attribute.getVertexPropertyName();
+
if (attrValue != null && attrValue2 != null) {
- ret = predicate.generatePredicate(StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName,
- attrValue, attrValue2, attrClass);
+ ret = predicate.generatePredicate(StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName, attrValue, attrValue2, attrClass);
} else {
- ret = predicate.generatePredicate(
- StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName,
- attrValue, attrClass);
+ ret = predicate.generatePredicate(StringUtils.isEmpty(vertexPropertyName) ? attribute.getQualifiedName() : vertexPropertyName, attrValue, attrClass);
}
}
return ret;
}
- protected AtlasGraphQuery toGraphFilterQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
- Set<String> filterAttributes = new HashSet<>();
- filterAttributes.addAll(graphAttributes);
-
- if (criteria != null) {
- if (criteria.getCondition() != null) {
- if (criteria.getCondition() == Condition.AND) {
- for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
-
- query.addConditionsFrom(nestedQuery);
- }
- } else {
- List<AtlasGraphQuery> orConditions = new LinkedList<>();
-
- for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
-
- orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
- }
-
- if (!orConditions.isEmpty()) {
- query.or(orConditions);
- }
- }
- } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
- try {
- ArrayList<AtlasGraphQuery> queries = new ArrayList<>();
- for (AtlasStructType structType : structTypes) {
- String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName());
- if (filterAttributes.contains(qualifiedName)) {
-
- String attrName = criteria.getAttributeName();
- String attrValue = criteria.getAttributeValue();
- SearchParameters.Operator operator = criteria.getOperator();
-
- //process attribute value and attribute operator for pipeSeperated fields
- if (isPipeSeparatedSystemAttribute(attrName)) {
- FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
- attrValue = processedCriteria.getAttributeValue();
- operator = processedCriteria.getOperator();
- }
-
- AtlasGraphQuery innerQry = context.getGraph().query().createChildQuery();
- switch (operator) {
- case LT:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
- break;
- case LTE:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
- break;
- case GT:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
- break;
- case GTE:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
- break;
- case EQ:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
- break;
- case NEQ:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
- break;
- case LIKE:
- innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
- break;
- case CONTAINS:
- innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
- break;
- case STARTS_WITH:
- innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
- break;
- case ENDS_WITH:
- innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
- break;
- case IS_NULL:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
- break;
- case NOT_NULL:
- innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
- break;
- case NOT_CONTAINS:
- break;
- default:
- LOG.warn("{}: unsupported operator. Ignored", operator);
- break;
- }
- queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry));
- filterAttributes.remove(qualifiedName);
- }
- }
-
- if (CollectionUtils.isNotEmpty(queries)) {
- if (queries.size() > 1) {
- return context.getGraph().query().createChildQuery().or(queries);
- } else {
- return queries.iterator().next();
- }
- }
- } catch (AtlasBaseException e) {
- LOG.warn(e.getMessage());
- }
- }
- }
-
- return query;
- }
-
private String toGremlinComparisonQuery(AtlasAttribute attribute, SearchParameters.Operator operator, String attrValue, Map<String, Object> queryBindings) {
String bindName = "__bind_" + queryBindings.size();
Object bindValue = attribute.getAttributeType().getNormalizedValue(attrValue);
- AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
- String queryTemplate = null;
+ AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.getInstance();
+ String queryTemplate = null;
+
switch (operator) {
case LT:
queryTemplate = queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.COMPARE_LT);
@@ -1155,7 +1275,7 @@ public abstract class SearchProcessor {
if (org.apache.commons.lang3.StringUtils.isNotEmpty(queryTemplate)) {
if (bindValue instanceof Date) {
- bindValue = ((Date)bindValue).getTime();
+ bindValue = ((Date) bindValue).getTime();
}
queryBindings.put(bindName, bindValue);
@@ -1239,140 +1359,88 @@ public abstract class SearchProcessor {
return false;
}
- protected Collection<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, Collection<AtlasVertex> vertices) {
- if (idxQueryResult != null) {
- while (idxQueryResult.hasNext()) {
- AtlasVertex vertex = idxQueryResult.next().getVertex();
-
- vertices.add(vertex);
- }
- }
-
- return vertices;
- }
-
- protected LinkedHashMap<Integer, AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap, int qryOffset) {
- if (idxQueryResult != null) {
- while (idxQueryResult.hasNext()) {
- AtlasVertex vertex = idxQueryResult.next().getVertex();
-
- offsetEntityVertexMap.put(qryOffset++, vertex);
- }
+ private static int getApplicationProperty(String propertyName, int defaultValue) {
+ try {
+ return ApplicationProperties.get().getInt(propertyName, defaultValue);
+ } catch (AtlasException excp) {
+ // ignore
}
- return offsetEntityVertexMap;
+ return defaultValue;
}
- protected LinkedHashMap<Integer, AtlasEdge> getEdgesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap, int qryOffset) {
- if (idxQueryResult != null) {
- while (idxQueryResult.hasNext()) {
- AtlasEdge edge = idxQueryResult.next().getEdge();
+ private static String getSortByAttribute(SearchContext context) {
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
+ String sortBy = context.getSearchParameters().getSortBy();
+ AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
- offsetEdgeMap.put(qryOffset++, edge);
+ if (sortByAttribute != null) {
+ return sortByAttribute.getVertexPropertyName();
}
}
- return offsetEdgeMap;
+ return null;
}
- protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, Collection<AtlasVertex> vertices) {
- if (iterator != null) {
- while (iterator.hasNext()) {
- AtlasVertex vertex = iterator.next();
+ private static Order getSortOrderAttribute(SearchContext context) {
+ SortOrder sortOrder = context.getSearchParameters().getSortOrder();
- vertices.add(vertex);
- }
+ if (sortOrder == null) {
+ sortOrder = ASCENDING;
}
- return vertices;
+ return sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
}
- protected LinkedHashMap<Integer, AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap, int qryOffset) {
- if (iterator != null) {
- while (iterator.hasNext()) {
- AtlasVertex vertex = iterator.next();
-
- offsetEntityVertexMap.put(qryOffset++, vertex);
- }
- }
+ static {
+ OPERATOR_MAP.put(SearchParameters.Operator.LT, INDEX_SEARCH_PREFIX + "\"%s\": [* TO %s}");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LT, getLTPredicateGenerator());
- return offsetEntityVertexMap;
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.GT, INDEX_SEARCH_PREFIX + "\"%s\": {%s TO *]");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.GT, getGTPredicateGenerator());
- protected LinkedHashMap<Integer, AtlasEdge> getEdges(Iterator<AtlasEdge> iterator, LinkedHashMap<Integer, AtlasEdge> offsetEdgeMap, int qryOffset) {
- if (iterator != null) {
- while (iterator.hasNext()) {
- AtlasEdge edge = iterator.next();
+ OPERATOR_MAP.put(SearchParameters.Operator.LTE, INDEX_SEARCH_PREFIX + "\"%s\": [* TO %s]");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LTE, getLTEPredicateGenerator());
- offsetEdgeMap.put(qryOffset++, edge);
- }
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.GTE, INDEX_SEARCH_PREFIX + "\"%s\": [%s TO *]");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.GTE, getGTEPredicateGenerator());
- return offsetEdgeMap;
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.EQ, INDEX_SEARCH_PREFIX + "\"%s\": %s");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.EQ, getEQPredicateGenerator());
- protected Set<String> getGuids(List<AtlasVertex> vertices) {
- Set<String> ret = new HashSet<>();
+ OPERATOR_MAP.put(SearchParameters.Operator.NEQ, "(*:* -" + INDEX_SEARCH_PREFIX + "\"%s\": %s)");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NEQ, getNEQPredicateGenerator());
- if (vertices != null) {
- for(AtlasVertex vertex : vertices) {
- String guid = AtlasGraphUtilsV2.getIdFromVertex(vertex);
+ OPERATOR_MAP.put(SearchParameters.Operator.IN, INDEX_SEARCH_PREFIX + "\"%s\": (%s)"); // this should be a list of quoted strings
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.IN, getINPredicateGenerator()); // this should be a list of quoted strings
- if (StringUtils.isNotEmpty(guid)) {
- ret.add(guid);
- }
- }
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.LIKE, INDEX_SEARCH_PREFIX + "\"%s\": (%s)"); // this should be regex pattern
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.LIKE, getLIKEPredicateGenerator()); // this should be regex pattern
- return ret;
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.STARTS_WITH, INDEX_SEARCH_PREFIX + "\"%s\": (%s*)");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.STARTS_WITH, getStartsWithPredicateGenerator());
- private static int getApplicationProperty(String propertyName, int defaultValue) {
- try {
- return ApplicationProperties.get().getInt(propertyName, defaultValue);
- } catch (AtlasException excp) {
- // ignore
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.ENDS_WITH, INDEX_SEARCH_PREFIX + "\"%s\": (*%s)");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.ENDS_WITH, getEndsWithPredicateGenerator());
- return defaultValue;
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, INDEX_SEARCH_PREFIX + "\"%s\": (*%s*)");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.CONTAINS, getContainsPredicateGenerator());
- private static String getSortByAttribute(SearchContext context) {
- if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
- final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
- String sortBy = context.getSearchParameters().getSortBy();
- AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
- if (sortByAttribute != null) {
- return sortByAttribute.getVertexPropertyName();
- }
- }
- return null;
- }
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_CONTAINS, getNotContainsPredicateGenerator());
- private static Order getSortOrderAttribute(SearchContext context) {
- SortOrder sortOrder = context.getSearchParameters().getSortOrder();
- if (sortOrder == null) sortOrder = ASCENDING;
+ // TODO: Add contains any, contains all mappings here
- return sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.IS_NULL, "(*:* NOT " + INDEX_SEARCH_PREFIX + "\"%s\":[* TO *])");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.IS_NULL, getIsNullPredicateGenerator());
- protected static Iterator<AtlasIndexQuery.Result> executeIndexQuery(SearchContext context, AtlasIndexQuery indexQuery, int qryOffset, int limit) {
- String sortBy = getSortByAttribute(context);
- if (sortBy != null && !sortBy.isEmpty()) {
- Order sortOrder = getSortOrderAttribute(context);
- return indexQuery.vertices(qryOffset, limit, sortBy, sortOrder);
- }
- return indexQuery.vertices(qryOffset, limit);
- }
+ OPERATOR_MAP.put(SearchParameters.Operator.NOT_NULL, INDEX_SEARCH_PREFIX + "\"%s\":[* TO *]");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_NULL, getNotNullPredicateGenerator());
- protected static Iterator<AtlasIndexQuery.Result> executeIndexQueryForEdge(SearchContext context, AtlasIndexQuery indexQuery, int qryOffset, int limit) {
- final AtlasRelationshipType relationshipType = context.getRelationshipTypes().iterator().next();
- AtlasStructType.AtlasAttribute sortByAttribute = relationshipType.getAttribute(context.getSearchParameters().getSortBy());
+ OPERATOR_MAP.put(SearchParameters.Operator.NOT_EMPTY, INDEX_SEARCH_PREFIX + "\"%s\":[* TO *]");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_EMPTY, getNotEmptyPredicateGenerator());
- if (sortByAttribute != null && StringUtils.isNotEmpty(sortByAttribute.getVertexPropertyName())) {
- Order sortOrder = getSortOrderAttribute(context);
- return indexQuery.edges(qryOffset, limit, sortByAttribute.getVertexPropertyName(), sortOrder);
- }
- return indexQuery.edges(qryOffset, limit);
+ OPERATOR_MAP.put(SearchParameters.Operator.TIME_RANGE, INDEX_SEARCH_PREFIX + "\"%s\": [%s TO %s]");
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.TIME_RANGE, getInRangePredicateGenerator());
}
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SuggestionsProviderImpl.java b/repository/src/main/java/org/apache/atlas/discovery/SuggestionsProviderImpl.java
index d7824aed9..834a963be 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SuggestionsProviderImpl.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SuggestionsProviderImpl.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,7 +27,6 @@ import org.slf4j.LoggerFactory;
import java.util.Collections;
-
public class SuggestionsProviderImpl implements SuggestionsProvider {
private static final Logger LOG = LoggerFactory.getLogger(SuggestionsProviderImpl.class);
@@ -51,7 +50,7 @@ public class SuggestionsProviderImpl implements SuggestionsProvider {
} catch (AtlasException e) {
LOG.error("Error encountered in performing quick suggestions. Will return no suggestions.", e);
- result.setSuggestions(Collections.EMPTY_LIST);
+ result.setSuggestions(Collections.emptyList());
}
return result;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java
index b8a507eac..13c6cacb6 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/TermSearchProcessor.java
@@ -30,7 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-
public class TermSearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(TermSearchProcessor.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("TermSearchProcessor");
@@ -50,15 +49,13 @@ public class TermSearchProcessor extends SearchProcessor {
@Override
public List<AtlasVertex> execute() {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> TermSearchProcessor.execute({})", context);
- }
+ LOG.debug("==> TermSearchProcessor.execute({})", context);
List<AtlasVertex> ret = new ArrayList<>();
AtlasPerfTracer perf = null;
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
- perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TermSearchProcessor.execute(" + context + ")");
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TermSearchProcessor.execute(" + context + ")");
}
//marker functionality will not work when there is need to fetch Term vertices and get entities from it
@@ -82,35 +79,35 @@ public class TermSearchProcessor extends SearchProcessor {
AtlasPerfTracer.log(perf);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== TermSearchProcessor.execute({}): ret.size()={}", context, ret.size());
- }
+ LOG.debug("<== TermSearchProcessor.execute({}): ret.size()={}", context, ret.size());
return ret;
}
+ @Override
+ public long getResultCount() {
+ return assignedEntities != null ? assignedEntities.size() : -1;
+ }
+
//this filter is never used
@Override
public LinkedHashMap<Integer, AtlasVertex> filter(LinkedHashMap<Integer, AtlasVertex> offsetEntityVertexMap) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> TermSearchProcessor.filter({})", offsetEntityVertexMap.size());
- }
+ LOG.debug("==> TermSearchProcessor.filter({})", offsetEntityVertexMap.size());
if (MapUtils.isNotEmpty(offsetEntityVertexMap)) {
if (CollectionUtils.isEmpty(assignedEntities)) {
offsetEntityVertexMap.clear();
} else {
- offsetEntityVertexMap.entrySet().stream().
- filter(o -> {
- if (o instanceof AtlasVertex) {
- AtlasVertex entityVertex = (AtlasVertex) o;
-
- for (AtlasVertex assignedEntity : assignedEntities) {
- if (assignedEntity.getId().equals(entityVertex.getId())) {
- return true;
- }
- }
+ offsetEntityVertexMap = offsetEntityVertexMap.entrySet().stream().filter(o -> {
+ if (o instanceof AtlasVertex) {
+ AtlasVertex entityVertex = (AtlasVertex) o;
+
+ for (AtlasVertex assignedEntity : assignedEntities) {
+ if (assignedEntity.getId().equals(entityVertex.getId())) {
+ return true;
}
+ }
+ }
return false;
}).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (x, y) -> y, LinkedHashMap::new));
@@ -119,15 +116,8 @@ public class TermSearchProcessor extends SearchProcessor {
offsetEntityVertexMap = super.filter(offsetEntityVertexMap);
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== TermSearchProcessor.filter(): ret.size()={}", offsetEntityVertexMap.size());
- }
+ LOG.debug("<== TermSearchProcessor.filter(): ret.size()={}", offsetEntityVertexMap.size());
return offsetEntityVertexMap;
}
-
- @Override
- public long getResultCount() {
- return assignedEntities != null ? assignedEntities.size() : -1;
- }
}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/Action.java b/repository/src/main/java/org/apache/atlas/entitytransform/Action.java
index 0c6102db9..6e79c365c 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.entitytransform;
+import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
@@ -25,15 +26,12 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
-import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collections;
-
-
public abstract class Action {
private static final Logger LOG = LoggerFactory.getLogger(Action.class);
@@ -47,24 +45,12 @@ public abstract class Action {
protected final EntityAttribute attribute;
-
protected Action(EntityAttribute attribute) {
this.attribute = attribute;
}
- public EntityAttribute getAttribute() { return attribute; }
-
- public boolean isValid() {
- return true;
- }
-
- public abstract void apply(AtlasTransformableEntity entity);
-
-
public static Action createAction(String key, String value, TransformerContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> Action.createAction(key={}, value={})", key, value);
- }
+ LOG.debug("==> Action.createAction(key={}, value={})", key, value);
final Action ret;
@@ -81,40 +67,47 @@ public abstract class Action {
switch (actionName.toUpperCase()) {
case ACTION_ADD_CLASSIFICATION:
ret = new AddClassificationAction(attribute, actionValue, context);
- break;
+ break;
case ACTION_NAME_REPLACE_PREFIX:
ret = new PrefixReplaceAction(attribute, actionValue);
- break;
+ break;
case ACTION_NAME_TO_LOWER:
ret = new ToLowerCaseAction(attribute);
- break;
+ break;
case ACTION_NAME_TO_UPPER:
ret = new ToUpperCaseAction(attribute);
- break;
+ break;
case ACTION_NAME_SET:
ret = new SetAction(attribute, actionValue);
- break;
+ break;
case ACTION_NAME_CLEAR:
ret = new ClearAction(attribute);
- break;
+ break;
default:
ret = new SetAction(attribute, value); // treat unspecified/unknown action as 'SET'
- break;
+ break;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== Action.createAction(key={}, value={}): actionName={}, actionValue={}, ret={}", key, value, actionName, actionValue, ret);
- }
+ LOG.debug("<== Action.createAction(key={}, value={}): actionName={}, actionValue={}, ret={}", key, value, actionName, actionValue, ret);
return ret;
}
+ public EntityAttribute getAttribute() {
+ return attribute;
+ }
+
+ public boolean isValid() {
+ return true;
+ }
+
+ public abstract void apply(AtlasTransformableEntity entity);
public static class SetAction extends Action {
private final String attributeValue;
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java b/repository/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
index 27a57a61e..dacece61b 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
@@ -25,12 +25,10 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-
public class AtlasEntityTransformer {
private final List<Condition> conditions;
private final List<Action> actions;
-
public AtlasEntityTransformer(AttributeTransform attributeTransform, TransformerContext context) {
this(attributeTransform.getConditions(), attributeTransform.getAction(), context);
}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java b/repository/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
index 975e4ddd6..230ad64c3 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
@@ -28,45 +28,17 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
-
public class BaseEntityHandler {
private static final Logger LOG = LoggerFactory.getLogger(BaseEntityHandler.class);
protected final List<AtlasEntityTransformer> transformers;
-
public BaseEntityHandler(List<AtlasEntityTransformer> transformers) {
this.transformers = transformers;
}
- public AtlasEntity transform(AtlasEntity entity) {
- if (CollectionUtils.isEmpty(transformers)) {
- return entity;
- }
-
- AtlasTransformableEntity transformableEntity = getTransformableEntity(entity);
-
- if (transformableEntity == null) {
- return entity;
- }
-
- for (AtlasEntityTransformer transformer : transformers) {
- transformer.transform(transformableEntity);
- }
-
- transformableEntity.transformComplete();
-
- return entity;
- }
-
- public AtlasTransformableEntity getTransformableEntity(AtlasEntity entity) {
- return new AtlasTransformableEntity(entity);
- }
-
public static List<BaseEntityHandler> createEntityHandlers(List<AttributeTransform> transforms, TransformerContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> BaseEntityHandler.createEntityHandlers(transforms={})", transforms);
- }
+ LOG.debug("==> BaseEntityHandler.createEntityHandlers(transforms={})", transforms);
List<BaseEntityHandler> ret = new ArrayList<>();
@@ -102,13 +74,73 @@ public class BaseEntityHandler {
}
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== BaseEntityHandler.createEntityHandlers(transforms={}): ret.size={}", transforms, ret.size());
- }
+ LOG.debug("<== BaseEntityHandler.createEntityHandlers(transforms={}): ret.size={}", transforms, ret.size());
return ret;
}
+ public static List<BaseEntityHandler> fromJson(String transformersString, TransformerContext context) {
+ if (StringUtils.isEmpty(transformersString)) {
+ return null;
+ }
+
+ Object transformersObj = AtlasType.fromJson(transformersString, Object.class);
+ List<?> transformers = (transformersObj instanceof List) ? (List<?>) transformersObj : null;
+
+ List<AttributeTransform> attributeTransforms = new ArrayList<>();
+
+ if (CollectionUtils.isEmpty(transformers)) {
+ return null;
+ }
+
+ for (Object transformer : transformers) {
+ String transformerStr = AtlasType.toJson(transformer);
+ AttributeTransform attributeTransform = AtlasType.fromJson(transformerStr, AttributeTransform.class);
+
+ if (attributeTransform == null) {
+ continue;
+ }
+
+ attributeTransforms.add(attributeTransform);
+ }
+
+ if (CollectionUtils.isEmpty(attributeTransforms)) {
+ return null;
+ }
+
+ List<BaseEntityHandler> entityHandlers = createEntityHandlers(attributeTransforms, context);
+
+ if (CollectionUtils.isEmpty(entityHandlers)) {
+ return null;
+ }
+
+ return entityHandlers;
+ }
+
+ public AtlasEntity transform(AtlasEntity entity) {
+ if (CollectionUtils.isEmpty(transformers)) {
+ return entity;
+ }
+
+ AtlasTransformableEntity transformableEntity = getTransformableEntity(entity);
+
+ if (transformableEntity == null) {
+ return entity;
+ }
+
+ for (AtlasEntityTransformer transformer : transformers) {
+ transformer.transform(transformableEntity);
+ }
+
+ transformableEntity.transformComplete();
+
+ return entity;
+ }
+
+ public AtlasTransformableEntity getTransformableEntity(AtlasEntity entity) {
+ return new AtlasTransformableEntity(entity);
+ }
+
private static boolean hasTransformerForAnyAttribute(List<AtlasEntityTransformer> transformers, List<String> attributes) {
if (CollectionUtils.isNotEmpty(transformers) && CollectionUtils.isNotEmpty(attributes)) {
for (AtlasEntityTransformer transformer : transformers) {
@@ -160,41 +192,4 @@ public class BaseEntityHandler {
// implementations can override to set value of computed-attributes
}
}
-
- public static List<BaseEntityHandler> fromJson(String transformersString, TransformerContext context) {
- if (StringUtils.isEmpty(transformersString)) {
- return null;
- }
-
- Object transformersObj = AtlasType.fromJson(transformersString, Object.class);
- List transformers = (transformersObj != null && transformersObj instanceof List) ? (List) transformersObj : null;
-
- List<AttributeTransform> attributeTransforms = new ArrayList<>();
-
- if (CollectionUtils.isEmpty(transformers)) {
- return null;
- }
-
- for (Object transformer : transformers) {
- String transformerStr = AtlasType.toJson(transformer);
- AttributeTransform attributeTransform = AtlasType.fromJson(transformerStr, AttributeTransform.class);
-
- if (attributeTransform == null) {
- continue;
- }
-
- attributeTransforms.add(attributeTransform);
- }
-
- if (CollectionUtils.isEmpty(attributeTransforms)) {
- return null;
- }
-
- List<BaseEntityHandler> entityHandlers = createEntityHandlers(attributeTransforms, context);
- if (CollectionUtils.isEmpty(entityHandlers)) {
- return null;
- }
-
- return entityHandlers;
- }
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/Condition.java b/repository/src/main/java/org/apache/atlas/entitytransform/Condition.java
index b834f4614..ae1b7962b 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -30,8 +30,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
-
-
public abstract class Condition {
private static final Logger LOG = LoggerFactory.getLogger(Condition.class);
@@ -47,20 +45,12 @@ public abstract class Condition {
protected final EntityAttribute attribute;
-
protected Condition(EntityAttribute attribute) {
this.attribute = attribute;
}
- public EntityAttribute getAttribute() { return attribute; }
-
- public abstract boolean matches(AtlasTransformableEntity entity);
-
-
public static Condition createCondition(String key, String value, TransformerContext context) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("==> Condition.createCondition(key={}, value={})", key, value);
- }
+ LOG.debug("==> Condition.createCondition(key={}, value={})", key, value);
final Condition ret;
@@ -89,19 +79,19 @@ public abstract class Condition {
case CONDITION_NAME_EQUALS:
ret = new EqualsCondition(attribute, conditionValue);
- break;
+ break;
case CONDITION_NAME_EQUALS_IGNORE_CASE:
ret = new EqualsIgnoreCaseCondition(attribute, conditionValue);
- break;
+ break;
case CONDITION_NAME_STARTS_WITH:
ret = new StartsWithCondition(attribute, conditionValue);
- break;
+ break;
case CONDITION_NAME_STARTS_WITH_IGNORE_CASE:
ret = new StartsWithIgnoreCaseCondition(attribute, conditionValue);
- break;
+ break;
case CONDITION_NAME_HAS_VALUE:
ret = new HasValueCondition(attribute);
@@ -109,16 +99,19 @@ public abstract class Condition {
default:
ret = new EqualsCondition(attribute, value); // treat unspecified/unknown condition as 'EQUALS'
- break;
+ break;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== Condition.createCondition(key={}, value={}): actionName={}, actionValue={}, ret={}", key, value, conditionName, conditionValue, ret);
- }
+ LOG.debug("<== Condition.createCondition(key={}, value={}): actionName={}, actionValue={}, ret={}", key, value, conditionName, conditionValue, ret);
return ret;
}
+ public EntityAttribute getAttribute() {
+ return attribute;
+ }
+
+ public abstract boolean matches(AtlasTransformableEntity entity);
public static class EqualsCondition extends Condition {
protected final String attributeValue;
@@ -137,7 +130,6 @@ public abstract class Condition {
}
}
-
public static class EqualsIgnoreCaseCondition extends Condition {
protected final String attributeValue;
@@ -155,7 +147,6 @@ public abstract class Condition {
}
}
-
public static class StartsWithCondition extends Condition {
protected final String prefix;
@@ -173,7 +164,6 @@ public abstract class Condition {
}
}
-
public static class StartsWithIgnoreCaseCondition extends Condition {
protected final String prefix;
@@ -204,7 +194,7 @@ public abstract class Condition {
if (!isMatchAll && context != null && context.getExportRequest() != null) {
AtlasExportRequest request = context.getExportRequest();
- for(AtlasObjectId objectId : request.getItemsToExport()) {
+ for (AtlasObjectId objectId : request.getItemsToExport()) {
addObjectId(objectId);
}
}
@@ -250,7 +240,6 @@ public abstract class Condition {
}
}
-
public static class HasValueCondition extends Condition {
public HasValueCondition(EntityAttribute attribute) {
super(attribute);
@@ -260,7 +249,7 @@ public abstract class Condition {
public boolean matches(AtlasTransformableEntity entity) {
Object attributeValue = entity != null ? entity.getAttribute(attribute) : null;
- return attributeValue != null ? StringUtils.isNotEmpty(attributeValue.toString()) : false;
+ return attributeValue != null && StringUtils.isNotEmpty(attributeValue.toString());
}
}
}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java b/repository/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
index 2df98a67b..1eb767d8a 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
@@ -27,13 +27,12 @@ import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_D
import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_NAME_ATTRIBUTE;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_CLUSTER_NAME_ATTRIBUTE;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH;
-import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH_PATH_ATTRIBUTE;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH_PATH_ATTRIBUTE;
import static org.apache.atlas.entitytransform.TransformationConstants.NAME_ATTRIBUTE;
import static org.apache.atlas.entitytransform.TransformationConstants.PATH_ATTRIBUTE;
import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE;
-
public class HdfsPathEntityHandler extends BaseEntityHandler {
static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HDFS_PATH_NAME_ATTRIBUTE, HDFS_PATH_PATH_ATTRIBUTE, HDFS_CLUSTER_NAME_ATTRIBUTE);
@@ -50,15 +49,13 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
return StringUtils.equals(entity.getTypeName(), HDFS_PATH);
}
-
public static class HdfsPathEntity extends AtlasTransformableEntity {
private String clusterName;
private String path;
private String name;
private String pathPrefix;
- private boolean isPathUpdated = false;
- private boolean isCustomAttributeUpdated = false;
-
+ private boolean isPathUpdated;
+ private boolean isCustomAttributeUpdated;
public HdfsPathEntity(AtlasEntity entity) {
super(entity);
@@ -115,24 +112,24 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
clusterName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HDFS_PATH_NAME_ATTRIBUTE:
name = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HDFS_PATH_PATH_ATTRIBUTE:
path = attributeValue;
- isPathUpdated = true;
+ isPathUpdated = true;
isCustomAttributeUpdated = true;
- break;
+ break;
default:
super.setAttribute(attribute, attributeValue);
- break;
+ break;
}
}
@@ -146,7 +143,6 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
}
}
-
private String toQualifiedName() {
return StringUtils.isEmpty(clusterName) ? toPath() : String.format("%s@%s", toPath(), clusterName);
}
@@ -167,4 +163,4 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
return ret;
}
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java b/repository/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
index 686e11c40..a0216a3f6 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
@@ -23,8 +23,15 @@ import org.apache.commons.lang.StringUtils;
import java.util.Arrays;
import java.util.List;
-import static org.apache.atlas.entitytransform.TransformationConstants.*;
-
+import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.DATABASE_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_COLUMN;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_COLUMN_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_CLUSTER_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE;
public class HiveColumnEntityHandler extends BaseEntityHandler {
static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_COLUMN_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
@@ -42,13 +49,12 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
return StringUtils.equals(entity.getTypeName(), HIVE_COLUMN);
}
-
public static class HiveColumnEntity extends AtlasTransformableEntity {
private String databaseName;
private String tableName;
private String columnName;
private String clusterName;
- private boolean isCustomAttributeUpdated = false;
+ private boolean isCustomAttributeUpdated;
public HiveColumnEntity(AtlasEntity entity) {
super(entity);
@@ -59,12 +65,12 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
if (qualifiedName != null) {
int databaseSeparatorIdx = qualifiedName.indexOf(DATABASE_DELIMITER);
- int tableSeparatorIdx = databaseSeparatorIdx != -1 ? qualifiedName.indexOf(DATABASE_DELIMITER, databaseSeparatorIdx + 1) : - 1;
+ int tableSeparatorIdx = databaseSeparatorIdx != -1 ? qualifiedName.indexOf(DATABASE_DELIMITER, databaseSeparatorIdx + 1) : -1;
int clusterSeparatorIdx = qualifiedName.lastIndexOf(CLUSTER_DELIMITER);
this.databaseName = (databaseSeparatorIdx != -1) ? qualifiedName.substring(0, databaseSeparatorIdx).trim() : "";
- this.tableName = (tableSeparatorIdx != -1) ? qualifiedName.substring(databaseSeparatorIdx + 1, tableSeparatorIdx).trim() : "";
- this.clusterName = (clusterSeparatorIdx != -1) ? qualifiedName.substring(clusterSeparatorIdx + 1).trim() : "";
+ this.tableName = (tableSeparatorIdx != -1) ? qualifiedName.substring(databaseSeparatorIdx + 1, tableSeparatorIdx).trim() : "";
+ this.clusterName = (clusterSeparatorIdx != -1) ? qualifiedName.substring(clusterSeparatorIdx + 1).trim() : "";
} else {
this.databaseName = "";
this.tableName = "";
@@ -98,29 +104,29 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
databaseName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_TABLE_NAME_ATTRIBUTE:
tableName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_COLUMN_NAME_ATTRIBUTE:
columnName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
default:
super.setAttribute(attribute, attributeValue);
- break;
+ break;
}
}
@@ -136,4 +142,4 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
return String.format("%s.%s.%s@%s", databaseName, tableName, columnName, clusterName);
}
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java b/repository/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
index b8032aa09..c4f3b6f97 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
@@ -23,7 +23,13 @@ import org.apache.commons.lang.StringUtils;
import java.util.Arrays;
import java.util.List;
-import static org.apache.atlas.entitytransform.TransformationConstants.*;
+import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DATABASE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_CLUSTER_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE;
public class HiveDatabaseEntityHandler extends BaseEntityHandler {
static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
@@ -37,7 +43,6 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
return isHiveDatabaseEntity(entity) ? new HiveDatabaseEntity(entity) : null;
}
-
private boolean isHiveDatabaseEntity(AtlasEntity entity) {
return StringUtils.equals(entity.getTypeName(), HIVE_DATABASE);
}
@@ -45,7 +50,7 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
private static class HiveDatabaseEntity extends AtlasTransformableEntity {
private String databaseName;
private String clusterName;
- private boolean isCustomAttributeUpdated = false;
+ private boolean isCustomAttributeUpdated;
public HiveDatabaseEntity(AtlasEntity entity) {
super(entity);
@@ -83,17 +88,17 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
databaseName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
default:
super.setAttribute(attribute, attributeValue);
- break;
+ break;
}
}
@@ -110,4 +115,4 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
return String.format("%s@%s", databaseName, clusterName);
}
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java b/repository/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
index 38de206d1..e0fafaa25 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
@@ -23,12 +23,21 @@ import org.apache.commons.lang.StringUtils;
import java.util.Arrays;
import java.util.List;
-import static org.apache.atlas.entitytransform.TransformationConstants.*;
+import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.DATABASE_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_CLUSTER_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_STORAGEDESC_SUFFIX;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_STORAGE_DESCRIPTOR;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.LOCATION_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.PATH_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE;
public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE, HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE);
-
public HiveStorageDescriptorEntityHandler(List<AtlasEntityTransformer> transformers) {
super(transformers);
}
@@ -47,8 +56,7 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
private String tableName;
private String clusterName;
private String location;
- private boolean isCustomAttributeUpdated = false;
-
+ private boolean isCustomAttributeUpdated;
public HiveStorageDescriptorEntity(AtlasEntity entity) {
super(entity);
@@ -105,27 +113,27 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
databaseName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_TABLE_NAME_ATTRIBUTE:
tableName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
location = attributeValue;
- break;
+ break;
default:
super.setAttribute(attribute, attributeValue);
- break;
+ break;
}
}
@@ -147,4 +155,4 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
return String.format("%s.%s@%s_storage", databaseName, tableName, clusterName);
}
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java b/repository/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
index caf08dd52..b49d2f1bc 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
@@ -23,12 +23,18 @@ import org.apache.commons.lang.StringUtils;
import java.util.Arrays;
import java.util.List;
-import static org.apache.atlas.entitytransform.TransformationConstants.*;
+import static org.apache.atlas.entitytransform.TransformationConstants.CLUSTER_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.DATABASE_DELIMITER;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_CLUSTER_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DB_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE_NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.NAME_ATTRIBUTE;
+import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED_NAME_ATTRIBUTE;
public class HiveTableEntityHandler extends BaseEntityHandler {
static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
-
public HiveTableEntityHandler(List<AtlasEntityTransformer> transformers) {
super(transformers);
}
@@ -43,14 +49,13 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
}
private static class HiveTableEntity extends AtlasTransformableEntity {
- private final String qualifiedName;
- private String databaseName;
- private String tableName;
- private String clusterName;
- private boolean isCustomAttributeUpdated = false;
-
+ private final String qualifiedName;
private final boolean tableNameFromQualifiedNameDifferent;
private final String tableNameFromQualifiedName;
+ private String databaseName;
+ private String tableName;
+ private String clusterName;
+ private boolean isCustomAttributeUpdated;
public HiveTableEntity(AtlasEntity entity) {
super(entity);
@@ -63,14 +68,14 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
int databaseSeparatorIdx = qualifiedName.indexOf(DATABASE_DELIMITER);
int clusterSeparatorIdx = qualifiedName.lastIndexOf(CLUSTER_DELIMITER);
- this.databaseName = databaseSeparatorIdx != -1 ? qualifiedName.substring(0, databaseSeparatorIdx) : "";
- this.clusterName = clusterSeparatorIdx != -1 ? qualifiedName.substring(clusterSeparatorIdx + 1) : "";
- this.tableNameFromQualifiedName = clusterSeparatorIdx != -1 ? this.qualifiedName.substring(databaseSeparatorIdx + 1, clusterSeparatorIdx) : "";
+ this.databaseName = databaseSeparatorIdx != -1 ? qualifiedName.substring(0, databaseSeparatorIdx) : "";
+ this.clusterName = clusterSeparatorIdx != -1 ? qualifiedName.substring(clusterSeparatorIdx + 1) : "";
+ this.tableNameFromQualifiedName = clusterSeparatorIdx != -1 ? this.qualifiedName.substring(databaseSeparatorIdx + 1, clusterSeparatorIdx) : "";
this.tableNameFromQualifiedNameDifferent = !this.tableNameFromQualifiedName.equals(this.tableName);
} else {
- this.databaseName = "";
- this.clusterName = "";
- this.tableNameFromQualifiedName = "";
+ this.databaseName = "";
+ this.clusterName = "";
+ this.tableNameFromQualifiedName = "";
this.tableNameFromQualifiedNameDifferent = false;
}
}
@@ -98,23 +103,23 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
tableName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_DB_NAME_ATTRIBUTE:
databaseName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
isCustomAttributeUpdated = true;
- break;
+ break;
default:
super.setAttribute(attribute, attributeValue);
- break;
+ break;
}
}
@@ -126,11 +131,10 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
}
}
-
private String toQualifiedName() {
String tableName = tableNameFromQualifiedNameDifferent ? this.tableNameFromQualifiedName : this.tableName;
return String.format("%s.%s@%s", databaseName, tableName, clusterName);
}
}
-}
\ No newline at end of file
+}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java b/repository/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
index 247de7320..8bffac9e3 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
@@ -30,13 +30,13 @@ public final class TransformationConstants {
public static final String LOCATION_ATTRIBUTE = "location";
public static final String PATH_ATTRIBUTE = "path";
- public static final String HIVE_DB_NAME_ATTRIBUTE = "hive_db.name";
- public static final String HIVE_DB_CLUSTER_NAME_ATTRIBUTE = "hive_db.clusterName";
- public static final String HIVE_TABLE_NAME_ATTRIBUTE = "hive_table.name";
- public static final String HIVE_COLUMN_NAME_ATTRIBUTE = "hive_column.name";
- public static final String HDFS_PATH_NAME_ATTRIBUTE = "hdfs_path.name";
- public static final String HDFS_PATH_PATH_ATTRIBUTE = "hdfs_path.path";
- public static final String HDFS_CLUSTER_NAME_ATTRIBUTE = "hdfs_path.clusterName";
+ public static final String HIVE_DB_NAME_ATTRIBUTE = "hive_db.name";
+ public static final String HIVE_DB_CLUSTER_NAME_ATTRIBUTE = "hive_db.clusterName";
+ public static final String HIVE_TABLE_NAME_ATTRIBUTE = "hive_table.name";
+ public static final String HIVE_COLUMN_NAME_ATTRIBUTE = "hive_column.name";
+ public static final String HDFS_PATH_NAME_ATTRIBUTE = "hdfs_path.name";
+ public static final String HDFS_PATH_PATH_ATTRIBUTE = "hdfs_path.path";
+ public static final String HDFS_CLUSTER_NAME_ATTRIBUTE = "hdfs_path.clusterName";
public static final String HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE = "hive_storagedesc.location";
public static final char TYPE_NAME_ATTRIBUTE_NAME_SEP = '.';
@@ -45,5 +45,7 @@ public final class TransformationConstants {
public static final char PATH_DELIMITER = '/';
public static final String HIVE_STORAGEDESC_SUFFIX = "_storage";
-
+ private TransformationConstants() {
+ // to block instantiation
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java b/repository/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
index 4b2ece61f..fa098f2df 100644
--- a/repository/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
+++ b/repository/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
index e218c2905..2a1a05cd6 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
@@ -46,17 +46,15 @@ import java.util.Set;
import java.util.stream.Collectors;
public class GlossaryCategoryUtils extends GlossaryUtils {
- private static final Logger LOG = LoggerFactory.getLogger(GlossaryCategoryUtils.class);
- private static final boolean DEBUG_ENABLED = LOG.isDebugEnabled();
+ private static final Logger LOG = LoggerFactory.getLogger(GlossaryCategoryUtils.class);
protected GlossaryCategoryUtils(AtlasRelationshipStore relationshipStore, AtlasTypeRegistry typeRegistry, DataAccess dataAccess) {
super(relationshipStore, typeRegistry, dataAccess);
}
public Map<String, AtlasGlossaryCategory> processCategoryRelations(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory, RelationshipOperation op) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryCategoryUtils.processCategoryRelations({}, {}, {})", storeObject, updatedCategory, op);
- }
+ LOG.debug("==> GlossaryCategoryUtils.processCategoryRelations({}, {}, {})", storeObject, updatedCategory, op);
+
Map<String, AtlasGlossaryCategory> impactedCategories = new HashMap<>();
processCategoryAnchor(storeObject, updatedCategory, op, impactedCategories);
@@ -64,15 +62,12 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
processCategoryChildren(storeObject, updatedCategory, op, impactedCategories);
processAssociatedTerms(storeObject, updatedCategory, op);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryCategoryUtils.processCategoryRelations(): {}", impactedCategories);
- }
+ LOG.debug("<== GlossaryCategoryUtils.processCategoryRelations(): {}", impactedCategories);
return impactedCategories;
}
- private void processCategoryAnchor(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory,
- RelationshipOperation op, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
+ private void processCategoryAnchor(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory, RelationshipOperation op, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
if (Objects.isNull(updatedCategory.getAnchor()) && op != RelationshipOperation.DELETE) {
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ANCHOR);
}
@@ -85,11 +80,10 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
if (StringUtils.isEmpty(updatedCategoryAnchor.getGlossaryGuid())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_NEW_ANCHOR_GUID);
} else {
- if (DEBUG_ENABLED) {
- LOG.debug("Creating new category anchor, category = {}, glossary = {}", storeObject.getGuid(), updatedCategory.getAnchor().getGlossaryGuid());
- }
-
String anchorGlossaryGuid = updatedCategory.getAnchor().getGlossaryGuid();
+
+ LOG.debug("Creating new category anchor, category = {}, glossary = {}", storeObject.getGuid(), anchorGlossaryGuid);
+
createRelationship(defineCategoryAnchorRelation(anchorGlossaryGuid, storeObject.getGuid()));
}
break;
@@ -99,22 +93,17 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
throw new AtlasBaseException(AtlasErrorCode.INVALID_NEW_ANCHOR_GUID);
}
- if (DEBUG_ENABLED) {
- LOG.debug("Updating category anchor, currAnchor = {}, newAnchor = {} and category = {}",
- storeObject.getAnchor().getGlossaryGuid(),
- updatedCategory.getAnchor().getGlossaryGuid(),
- storeObject.getGuid());
- }
+ LOG.debug("Updating category anchor, currAnchor = {}, newAnchor = {} and category = {}", storeObject.getAnchor().getGlossaryGuid(), updatedCategory.getAnchor().getGlossaryGuid(), storeObject.getGuid());
+
relationshipStore.deleteById(storeObject.getAnchor().getRelationGuid(), true);
// Derive the qualifiedName when anchor changes
String anchorGlossaryGuid = updatedCategory.getAnchor().getGlossaryGuid();
AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid));
- storeObject.setQualifiedName(storeObject.getName()+ "@" + glossary.getQualifiedName());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Derived qualifiedName = {}", storeObject.getQualifiedName());
- }
+ storeObject.setQualifiedName(storeObject.getName() + "@" + glossary.getQualifiedName());
+
+ LOG.debug("Derived qualifiedName = {}", storeObject.getQualifiedName());
createRelationship(defineCategoryAnchorRelation(updatedCategory.getAnchor().getGlossaryGuid(), storeObject.getGuid()));
@@ -124,17 +113,15 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
break;
case DELETE:
if (Objects.nonNull(storeObject.getAnchor())) {
- if (DEBUG_ENABLED) {
- LOG.debug("Deleting category anchor");
- }
+ LOG.debug("Deleting category anchor");
+
relationshipStore.deleteById(storeObject.getAnchor().getRelationGuid(), true);
}
break;
}
}
- private void processParentCategory(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory,
- RelationshipOperation op, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
+ private void processParentCategory(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory, RelationshipOperation op, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
AtlasRelatedCategoryHeader newParent = updatedCategory.getParentCategory();
AtlasRelatedCategoryHeader existingParent = storeObject.getParentCategory();
@@ -146,9 +133,8 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
break;
case UPDATE:
if (Objects.equals(newParent, existingParent)) {
- if (DEBUG_ENABLED) {
- LOG.debug("No change to parent");
- }
+ LOG.debug("No change to parent");
+
break;
}
@@ -157,10 +143,10 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
} else if (Objects.isNull(newParent)) {
processParentRemoval(storeObject, updatedCategory, existingParent, impactedCategories);
} else {
- if (DEBUG_ENABLED) {
- LOG.debug("Updating category parent, category = {}, currParent = {}, newParent = {}", storeObject.getGuid(), existingParent.getDisplayText(), newParent.getDisplayText());
- }
+ LOG.debug("Updating category parent, category = {}, currParent = {}, newParent = {}", storeObject.getGuid(), existingParent.getDisplayText(), newParent.getDisplayText());
+
AtlasRelationship parentRelationship = relationshipStore.getById(existingParent.getRelationGuid());
+
if (existingParent.getCategoryGuid().equals(newParent.getCategoryGuid())) {
updateRelationshipAttributes(parentRelationship, newParent);
relationshipStore.update(parentRelationship);
@@ -180,27 +166,24 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
}
private void processNewParent(AtlasGlossaryCategory storeObject, AtlasRelatedCategoryHeader newParent, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("Creating new parent, category = {}, parent = {}", storeObject.getGuid(), newParent.getDisplayText());
- }
+ LOG.debug("Creating new parent, category = {}, parent = {}", storeObject.getGuid(), newParent.getDisplayText());
+
createRelationship(defineCategoryHierarchyLink(newParent, storeObject.getGuid()));
// New parent added, qualifiedName needs recomputation
// Derive the qualifiedName of the Glossary
AtlasGlossaryCategory parentCategory = dataAccess.load(getAtlasGlossaryCategorySkeleton(newParent.getCategoryGuid()));
+
storeObject.setQualifiedName(storeObject.getName() + "." + parentCategory.getQualifiedName());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Derived qualifiedName = {}", storeObject.getQualifiedName());
- }
+ LOG.debug("Derived qualifiedName = {}", storeObject.getQualifiedName());
updateChildCategories(storeObject, storeObject.getChildrenCategories(), impactedCategories, false);
}
private void processParentRemoval(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory, AtlasRelatedCategoryHeader existingParent, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("Removing category parent, category = {}, parent = {}", storeObject.getGuid(), existingParent.getDisplayText());
- }
+ LOG.debug("Removing category parent, category = {}, parent = {}", storeObject.getGuid(), existingParent.getDisplayText());
+
relationshipStore.deleteById(existingParent.getRelationGuid(), true);
// Parent deleted, qualifiedName needs recomputation
@@ -210,9 +193,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid));
storeObject.setQualifiedName(storeObject.getName() + "@" + glossary.getQualifiedName());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Derived qualifiedName = {}", storeObject.getQualifiedName());
- }
+ LOG.debug("Derived qualifiedName = {}", storeObject.getQualifiedName());
updateChildCategories(storeObject, storeObject.getChildrenCategories(), impactedCategories, false);
}
@@ -223,49 +204,47 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
switch (op) {
case CREATE:
- if (DEBUG_ENABLED) {
- LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getName(),
- Objects.nonNull(newTerms) ? newTerms.size() : "none");
- }
+ LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getName(), Objects.nonNull(newTerms) ? newTerms.size() : "none");
+
createTermCategorizationRelationships(storeObject, newTerms.values());
break;
case UPDATE:
if (MapUtils.isEmpty(existingTerms)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getName(),
- Objects.nonNull(newTerms) ? newTerms.size() : "none");
- }
+ LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getName(), Objects.nonNull(newTerms) ? newTerms.size() : "none");
+
createTermCategorizationRelationships(storeObject, newTerms.values());
break;
}
if (MapUtils.isEmpty(newTerms)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Deleting term relation with category = {}, terms = {}", storeObject.getName(), existingTerms.size());
- }
+ LOG.debug("Deleting term relation with category = {}, terms = {}", storeObject.getName(), existingTerms.size());
+
deleteTermCategorizationRelationships(storeObject, existingTerms.values());
break;
}
Set<AtlasRelatedTermHeader> toCreate = newTerms
- .values()
- .stream()
- .filter(t -> !existingTerms.containsKey(t.getTermGuid()))
- .collect(Collectors.toSet());
+ .values()
+ .stream()
+ .filter(t -> !existingTerms.containsKey(t.getTermGuid()))
+ .collect(Collectors.toSet());
+
createTermCategorizationRelationships(storeObject, toCreate);
Set<AtlasRelatedTermHeader> toUpdate = newTerms
- .values()
- .stream()
- .filter(t -> updatedExistingTermRelation(existingTerms, t))
- .collect(Collectors.toSet());
+ .values()
+ .stream()
+ .filter(t -> updatedExistingTermRelation(existingTerms, t))
+ .collect(Collectors.toSet());
+
updateTermCategorizationRelationships(storeObject, toUpdate);
Set<AtlasRelatedTermHeader> toDelete = existingTerms
- .values()
- .stream()
- .filter(t -> !newTerms.containsKey(t.getTermGuid()))
- .collect(Collectors.toSet());
+ .values()
+ .stream()
+ .filter(t -> !newTerms.containsKey(t.getTermGuid()))
+ .collect(Collectors.toSet());
+
deleteTermCategorizationRelationships(storeObject, toDelete);
break;
case DELETE:
@@ -281,8 +260,10 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private Map<String, AtlasRelatedTermHeader> getTerms(AtlasGlossaryCategory category) {
if (Objects.nonNull(category.getTerms())) {
Map<String, AtlasRelatedTermHeader> map = new HashMap<>();
+
for (AtlasRelatedTermHeader t : category.getTerms()) {
AtlasRelatedTermHeader header = map.get(t.getTermGuid());
+
if (header == null) {
map.put(t.getTermGuid(), t);
} else if (StringUtils.isEmpty(header.getRelationGuid()) && StringUtils.isNotEmpty(t.getRelationGuid())) {
@@ -290,26 +271,27 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
}
}
return map;
+ } else {
+ return Collections.emptyMap();
}
- else return Collections.emptyMap();
}
private void createTermCategorizationRelationships(AtlasGlossaryCategory storeObject, Collection<AtlasRelatedTermHeader> terms) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(terms)) {
Set<AtlasRelatedTermHeader> existingTerms = storeObject.getTerms();
+
for (AtlasRelatedTermHeader term : terms) {
if (Objects.isNull(term.getTermGuid())) {
throw new AtlasBaseException(AtlasErrorCode.MISSING_TERM_ID_FOR_CATEGORIZATION);
} else {
if (Objects.nonNull(existingTerms) && existingTerms.contains(term)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Skipping existing term guid={}", term.getTermGuid());
- }
+ LOG.debug("Skipping existing term guid={}", term.getTermGuid());
+
continue;
}
- if (DEBUG_ENABLED) {
- LOG.debug("Creating relation between category = {} and term = {}", storeObject.getGuid(), term.getDisplayText());
- }
+
+ LOG.debug("Creating relation between category = {} and term = {}", storeObject.getGuid(), term.getDisplayText());
+
createRelationship(defineCategorizedTerm(storeObject.getGuid(), term));
}
}
@@ -319,11 +301,12 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private void updateTermCategorizationRelationships(AtlasGlossaryCategory storeObject, Collection<AtlasRelatedTermHeader> terms) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(terms)) {
for (AtlasRelatedTermHeader term : terms) {
- if (DEBUG_ENABLED) {
- LOG.debug("Updating term relation with category = {}, term = {}", storeObject.getName(), term.getDisplayText());
- }
+ LOG.debug("Updating term relation with category = {}, term = {}", storeObject.getName(), term.getDisplayText());
+
AtlasRelationship relationship = relationshipStore.getById(term.getRelationGuid());
+
updateRelationshipAttributes(relationship, term);
+
relationshipStore.update(relationship);
}
}
@@ -332,9 +315,8 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private void deleteTermCategorizationRelationships(AtlasGlossaryCategory storeObject, Collection<AtlasRelatedTermHeader> terms) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(terms)) {
for (AtlasRelatedTermHeader term : terms) {
- if (DEBUG_ENABLED) {
- LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getName(), term.getDisplayText());
- }
+ LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getName(), term.getDisplayText());
+
relationshipStore.deleteById(term.getRelationGuid(), true);
}
}
@@ -343,12 +325,11 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private void processCategoryChildren(AtlasGlossaryCategory storeObject, AtlasGlossaryCategory updatedCategory, RelationshipOperation op, Map<String, AtlasGlossaryCategory> impactedCategories) throws AtlasBaseException {
Map<String, AtlasRelatedCategoryHeader> newChildren = getChildren(updatedCategory);
Map<String, AtlasRelatedCategoryHeader> existingChildren = getChildren(storeObject);
+
switch (op) {
case CREATE:
- if (DEBUG_ENABLED) {
- LOG.debug("Creating new children, category = {}, children = {}", storeObject.getName(),
- Objects.nonNull(newChildren) ? newChildren.size() : "none");
- }
+ LOG.debug("Creating new children, category = {}, children = {}", storeObject.getName(), Objects.nonNull(newChildren) ? newChildren.size() : "none");
+
createCategoryRelationships(storeObject, newChildren.values());
// New children added, qualifiedName needs recomputation
@@ -357,21 +338,19 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
case UPDATE:
// Create new children
if (MapUtils.isEmpty(existingChildren)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Creating new children, category = {}, children = {}", storeObject.getName(),
- Objects.nonNull(newChildren) ? newChildren.size() : "none");
- }
+ LOG.debug("Creating new children, category = {}, children = {}", storeObject.getName(), Objects.nonNull(newChildren) ? newChildren.size() : "none");
+
createCategoryRelationships(storeObject, newChildren.values());
// New children added, qualifiedName needs recomputation
updateChildCategories(storeObject, newChildren.values(), impactedCategories, false);
break;
}
+
// Delete current children
if (MapUtils.isEmpty(newChildren)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Deleting children, category = {}, children = {}", storeObject.getName(), existingChildren.size());
- }
+ LOG.debug("Deleting children, category = {}, children = {}", storeObject.getName(), existingChildren.size());
+
deleteCategoryRelationships(storeObject, existingChildren.values());
// Re-compute the qualifiedName for all children
@@ -380,26 +359,30 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
}
Set<AtlasRelatedCategoryHeader> toCreate = newChildren
- .values()
- .stream()
- .filter(c -> !existingChildren.containsKey(c.getCategoryGuid()))
- .collect(Collectors.toSet());
+ .values()
+ .stream()
+ .filter(c -> !existingChildren.containsKey(c.getCategoryGuid()))
+ .collect(Collectors.toSet());
+
createCategoryRelationships(storeObject, toCreate);
+
// New children added, qualifiedName needs recomputation
updateChildCategories(storeObject, toCreate, impactedCategories, false);
Set<AtlasRelatedCategoryHeader> toUpdate = newChildren
- .values()
- .stream()
- .filter(c -> updatedExistingCategoryRelation(existingChildren, c))
- .collect(Collectors.toSet());
+ .values()
+ .stream()
+ .filter(c -> updatedExistingCategoryRelation(existingChildren, c))
+ .collect(Collectors.toSet());
+
updateCategoryRelationships(storeObject, toUpdate);
Set<AtlasRelatedCategoryHeader> toDelete = existingChildren
- .values()
- .stream()
- .filter(c -> !newChildren.containsKey(c.getCategoryGuid()))
- .collect(Collectors.toSet());
+ .values()
+ .stream()
+ .filter(c -> !newChildren.containsKey(c.getCategoryGuid()))
+ .collect(Collectors.toSet());
+
deleteCategoryRelationships(storeObject, toDelete);
break;
case DELETE:
@@ -415,45 +398,47 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private Map<String, AtlasRelatedCategoryHeader> getChildren(AtlasGlossaryCategory category) {
if (Objects.nonNull(category.getChildrenCategories())) {
Map<String, AtlasRelatedCategoryHeader> map = new HashMap<>();
+
for (AtlasRelatedCategoryHeader c : category.getChildrenCategories()) {
AtlasRelatedCategoryHeader header = map.get(c.getCategoryGuid());
+
if (header == null || (StringUtils.isEmpty(header.getRelationGuid()) && StringUtils.isNotEmpty(c.getRelationGuid()))) {
map.put(c.getCategoryGuid(), c);
}
}
+
return map;
+ } else {
+ return Collections.emptyMap();
}
- else return Collections.emptyMap();
}
private void createCategoryRelationships(AtlasGlossaryCategory storeObject, Collection<AtlasRelatedCategoryHeader> newChildren) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(newChildren)) {
Set<AtlasRelatedCategoryHeader> existingChildren = storeObject.getChildrenCategories();
+
for (AtlasRelatedCategoryHeader child : newChildren) {
if (Objects.nonNull(existingChildren) && existingChildren.contains(child)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Skipping existing child relation for category guid = {}", child.getCategoryGuid());
- }
+ LOG.debug("Skipping existing child relation for category guid = {}", child.getCategoryGuid());
+
continue;
}
- if (DEBUG_ENABLED) {
- LOG.debug("Loading the child category to perform glossary check");
- }
+ LOG.debug("Loading the child category to perform glossary check");
AtlasGlossaryCategory childCategory = new AtlasGlossaryCategory();
+
childCategory.setGuid(child.getCategoryGuid());
+
childCategory = dataAccess.load(childCategory);
if (StringUtils.equals(storeObject.getAnchor().getGlossaryGuid(), childCategory.getAnchor().getGlossaryGuid())) {
- if (DEBUG_ENABLED) {
- LOG.debug("Creating new child, category = {}, child = {}", storeObject.getName(), child.getDisplayText());
- }
+ LOG.debug("Creating new child, category = {}, child = {}", storeObject.getName(), child.getDisplayText());
+
createRelationship(defineCategoryHierarchyLink(storeObject.getGuid(), child));
} else {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CHILD_CATEGORY_DIFFERENT_GLOSSARY, child.getCategoryGuid());
}
-
}
}
}
@@ -461,11 +446,12 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private void updateCategoryRelationships(AtlasGlossaryCategory storeObject, Collection<AtlasRelatedCategoryHeader> toUpdate) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(toUpdate)) {
for (AtlasRelatedCategoryHeader categoryHeader : toUpdate) {
- if (DEBUG_ENABLED) {
- LOG.debug("Updating child, category = {}, child = {}", storeObject.getName(), categoryHeader.getDisplayText());
- }
+ LOG.debug("Updating child, category = {}, child = {}", storeObject.getName(), categoryHeader.getDisplayText());
+
AtlasRelationship childRelationship = relationshipStore.getById(categoryHeader.getRelationGuid());
+
updateRelationshipAttributes(childRelationship, categoryHeader);
+
relationshipStore.update(childRelationship);
}
}
@@ -474,9 +460,8 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private void deleteCategoryRelationships(AtlasGlossaryCategory storeObject, Collection<AtlasRelatedCategoryHeader> existingChildren) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(existingChildren)) {
for (AtlasRelatedCategoryHeader child : existingChildren) {
- if (DEBUG_ENABLED) {
- LOG.debug("Deleting child, category = {}, child = {}", storeObject.getName(), child.getDisplayText());
- }
+ LOG.debug("Deleting child, category = {}, child = {}", storeObject.getName(), child.getDisplayText());
+
relationshipStore.deleteById(child.getRelationGuid(), true);
}
}
@@ -492,27 +477,30 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
private AtlasRelationship defineCategoryHierarchyLink(String parentCategoryGuid, AtlasRelatedCategoryHeader childCategory) {
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(CATEGORY_HIERARCHY);
AtlasStruct defaultAttrs = relationshipType.createDefaultValue();
+ AtlasRelationship relationship = new AtlasRelationship(CATEGORY_HIERARCHY, new AtlasObjectId(parentCategoryGuid), new AtlasObjectId(childCategory.getCategoryGuid()), defaultAttrs.getAttributes());
- AtlasRelationship relationship = new AtlasRelationship(CATEGORY_HIERARCHY, new AtlasObjectId(parentCategoryGuid), new AtlasObjectId(childCategory.getCategoryGuid()), defaultAttrs.getAttributes());
updateRelationshipAttributes(relationship, childCategory);
+
return relationship;
}
private AtlasRelationship defineCategoryHierarchyLink(AtlasRelatedCategoryHeader parentCategory, String childGuid) {
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(CATEGORY_HIERARCHY);
AtlasStruct defaultAttrs = relationshipType.createDefaultValue();
+ AtlasRelationship relationship = new AtlasRelationship(CATEGORY_HIERARCHY, new AtlasObjectId(parentCategory.getCategoryGuid()), new AtlasObjectId(childGuid), defaultAttrs.getAttributes());
- AtlasRelationship relationship = new AtlasRelationship(CATEGORY_HIERARCHY, new AtlasObjectId(parentCategory.getCategoryGuid()), new AtlasObjectId(childGuid), defaultAttrs.getAttributes());
updateRelationshipAttributes(relationship, parentCategory);
+
return relationship;
}
private AtlasRelationship defineCategorizedTerm(String categoryGuid, AtlasRelatedTermHeader relatedTermId) {
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(TERM_CATEGORIZATION);
AtlasStruct defaultAttrs = relationshipType.createDefaultValue();
+ AtlasRelationship relationship = new AtlasRelationship(TERM_CATEGORIZATION, new AtlasObjectId(categoryGuid), new AtlasObjectId(relatedTermId.getTermGuid()), defaultAttrs.getAttributes());
- AtlasRelationship relationship = new AtlasRelationship(TERM_CATEGORIZATION, new AtlasObjectId(categoryGuid), new AtlasObjectId(relatedTermId.getTermGuid()), defaultAttrs.getAttributes());
updateRelationshipAttributes(relationship, relatedTermId);
+
return relationship;
}
@@ -527,9 +515,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
return;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("Recomputing qualifiedName for {} children of category(guid={}, qualifiedName={})", childCategories.size(), parentCategory.getGuid(), parentCategory.getQualifiedName());
- }
+ LOG.debug("Recomputing qualifiedName for {} children of category(guid={}, qualifiedName={})", childCategories.size(), parentCategory.getGuid(), parentCategory.getQualifiedName());
String parentAnchorGuid = parentCategory.getAnchor().getGlossaryGuid();
@@ -537,25 +523,26 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
AtlasGlossaryCategory child = dataAccess.load(getAtlasGlossaryCategorySkeleton(childCategoryHeader.getCategoryGuid()));
String qualifiedName = child.getName() + ".";
String childAnchorGuid = child.getAnchor().getGlossaryGuid();
+
if (isParentRemoved) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Parent removed, deriving qualifiedName using Glossary");
- }
+ LOG.debug("Parent removed, deriving qualifiedName using Glossary");
+
AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(childAnchorGuid));
+
qualifiedName += glossary.getQualifiedName();
+
child.setParentCategory(null);
} else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using parent to derive qualifiedName");
- }
+ LOG.debug("Using parent to derive qualifiedName");
+
qualifiedName += parentCategory.getQualifiedName();
}
+
child.setQualifiedName(qualifiedName);
- if (!StringUtils.equals(childAnchorGuid, parentAnchorGuid)){
- if (LOG.isDebugEnabled()) {
- LOG.debug("Child anchor guid({}) doesn't match parent anchor guid({}). Updating child anchor", childAnchorGuid, parentAnchorGuid);
- }
+ if (!StringUtils.equals(childAnchorGuid, parentAnchorGuid)) {
+ LOG.debug("Child anchor guid({}) doesn't match parent anchor guid({}). Updating child anchor", childAnchorGuid, parentAnchorGuid);
+
// Remove old glossary relation
relationshipStore.deleteById(child.getAnchor().getRelationGuid(), true);
@@ -566,14 +553,11 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
child.getAnchor().setGlossaryGuid(parentAnchorGuid);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("Derived qualifiedName = {}", qualifiedName);
- }
+ LOG.debug("Derived qualifiedName = {}", qualifiedName);
// Collect all affected/impacted categories in this accumulator. All impacted categories will be saved as a part of post processing
impactedCategories.put(child.getGuid(), child);
updateChildCategories(child, child.getChildrenCategories(), impactedCategories, false);
}
}
-
}
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java
index 298fda359..5533d9292 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java
@@ -49,18 +49,20 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import javax.inject.Inject;
+
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED;
import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.SUCCESS;
import static org.apache.atlas.glossary.GlossaryUtils.getAtlasGlossaryCategorySkeleton;
@@ -69,17 +71,21 @@ import static org.apache.atlas.glossary.GlossaryUtils.getGlossarySkeleton;
@Service
public class GlossaryService {
- private static final Logger LOG = LoggerFactory.getLogger(GlossaryService.class);
- private static final boolean DEBUG_ENABLED = LOG.isDebugEnabled();
- private static final String ATLAS_GLOSSARY_TERM = "AtlasGlossaryTerm";
- private static final String NAME_ATTR = "name";
- private static final String QUALIFIED_NAME_ATTR = "qualifiedName";
- private static final String GLOSSARY_QUALIFIED_NAME_PROPERTY = "AtlasGlossary." + QUALIFIED_NAME_ATTR;
- private static final String GLOSSARY_CATEGORY_NAME_PROPERTY = "AtlasGlossaryCategory.name";
- private static final String GLOSSARY_TERM_NAME_PROPERTY = ATLAS_GLOSSARY_TERM + "." + NAME_ATTR;
- private static final String GLOSSARY_TERM_QUALIFIED_NAME_PROPERTY = ATLAS_GLOSSARY_TERM + "." + QUALIFIED_NAME_ATTR;
- private static final String TERM_UNIQUE_QUALIFIED_NAME_PROPERTY = ATLAS_GLOSSARY_TERM + ".__u_" + QUALIFIED_NAME_ATTR;
- private static final String GLOSSARY_TERM_ANCHOR_EDGE_LABEL = "r:AtlasGlossaryTermAnchor";
+ private static final Logger LOG = LoggerFactory.getLogger(GlossaryService.class);
+
+ private static final String ATLAS_GLOSSARY_TERM = "AtlasGlossaryTerm";
+ private static final String NAME_ATTR = "name";
+ private static final String QUALIFIED_NAME_ATTR = "qualifiedName";
+ private static final String GLOSSARY_QUALIFIED_NAME_PROPERTY = "AtlasGlossary." + QUALIFIED_NAME_ATTR;
+ private static final String GLOSSARY_CATEGORY_NAME_PROPERTY = "AtlasGlossaryCategory.name";
+ private static final String GLOSSARY_TERM_NAME_PROPERTY = ATLAS_GLOSSARY_TERM + "." + NAME_ATTR;
+ private static final String GLOSSARY_TERM_QUALIFIED_NAME_PROPERTY = ATLAS_GLOSSARY_TERM + "." + QUALIFIED_NAME_ATTR;
+ private static final String TERM_UNIQUE_QUALIFIED_NAME_PROPERTY = ATLAS_GLOSSARY_TERM + ".__u_" + QUALIFIED_NAME_ATTR;
+ private static final String GLOSSARY_TERM_ANCHOR_EDGE_LABEL = "r:AtlasGlossaryTermAnchor";
+ private static final char[] INVALID_NAME_CHARS = {'@', '.', '<', '>'};
+
+ private static final Map<String, String> GLOSSARY_GUID_QUALIFIED_NAME_CACHE = new HashMap<>();
+ private static final Map<String, String> CATEGORY_GUID_NAME_CACHE = new HashMap<>();
private final DataAccess dataAccess;
private final GlossaryTermUtils glossaryTermUtils;
@@ -88,14 +94,8 @@ public class GlossaryService {
private final AtlasEntityChangeNotifier entityChangeNotifier;
private final AtlasGlossaryDTO glossaryDTO;
- private static final char[] invalidNameChars = { '@', '.', '<', '>'};
-
- private static final Map<String, String> glossaryGuidQualifiedNameCache = new HashMap<>();
- private static final Map<String, String> categoryGuidNameCache = new HashMap<>();
-
@Inject
- public GlossaryService(DataAccess dataAccess, final AtlasRelationshipStore relationshipStore,
- final AtlasTypeRegistry typeRegistry, AtlasEntityChangeNotifier entityChangeNotifier, final AtlasGlossaryDTO glossaryDTO) {
+ public GlossaryService(DataAccess dataAccess, final AtlasRelationshipStore relationshipStore, final AtlasTypeRegistry typeRegistry, AtlasEntityChangeNotifier entityChangeNotifier, final AtlasGlossaryDTO glossaryDTO) {
this.dataAccess = dataAccess;
atlasTypeRegistry = typeRegistry;
glossaryTermUtils = new GlossaryTermUtils(relationshipStore, typeRegistry, dataAccess);
@@ -104,42 +104,45 @@ public class GlossaryService {
this.glossaryDTO = glossaryDTO;
}
+ public static boolean isNameInvalid(String name) {
+ return StringUtils.containsAny(name, INVALID_NAME_CHARS);
+ }
+
/**
* List all glossaries
*
- * @param limit page size - no paging by default
- * @param offset offset for pagination
+ * @param limit page size - no paging by default
+ * @param offset offset for pagination
* @param sortOrder ASC (default) or DESC
* @return List of all glossaries
* @throws AtlasBaseException
*/
@GraphTransaction
public List<AtlasGlossary> getGlossaries(int limit, int offset, SortOrder sortOrder) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossaries({}, {}, {})", limit, offset, sortOrder);
- }
+ LOG.debug("==> GlossaryService.getGlossaries({}, {}, {})", limit, offset, sortOrder);
- List<String> glossaryGuids = AtlasGraphUtilsV2.findEntityGUIDsByType(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, sortOrder);
- PaginationHelper paginationHelper = new PaginationHelper<>(glossaryGuids, offset, limit);
+ List<String> glossaryGuids = AtlasGraphUtilsV2.findEntityGUIDsByType(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME, sortOrder);
+ PaginationHelper<String> paginationHelper = new PaginationHelper<>(glossaryGuids, offset, limit);
- List<AtlasGlossary> ret;
- List<String> guidsToLoad = paginationHelper.getPaginatedList();
+ List<AtlasGlossary> ret;
+ List<String> guidsToLoad = paginationHelper.getPaginatedList();
AtlasEntity.AtlasEntitiesWithExtInfo glossaryEntities;
if (CollectionUtils.isNotEmpty(guidsToLoad)) {
glossaryEntities = dataAccess.getAtlasEntityStore().getByIds(guidsToLoad, true, false);
- ret = new ArrayList<>();
+ ret = new ArrayList<>();
+
for (AtlasEntity glossaryEntity : glossaryEntities.getEntities()) {
AtlasGlossary glossary = glossaryDTO.from(glossaryEntity);
+
ret.add(glossary);
}
} else {
ret = Collections.emptyList();
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossaries() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossaries() : {}", ret);
+
return ret;
}
@@ -152,11 +155,9 @@ public class GlossaryService {
*/
@GraphTransaction
public AtlasGlossary createGlossary(AtlasGlossary atlasGlossary) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.createGlossary({})", atlasGlossary);
- }
+ LOG.debug("==> GlossaryService.createGlossary({})", atlasGlossary);
- if (Objects.isNull(atlasGlossary)) {
+ if (isNull(atlasGlossary)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Glossary definition missing");
}
@@ -164,7 +165,8 @@ public class GlossaryService {
if (StringUtils.isEmpty(atlasGlossary.getName())) {
throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_QUALIFIED_NAME_CANT_BE_DERIVED);
}
- if (isNameInvalid(atlasGlossary.getName())){
+
+ if (isNameInvalid(atlasGlossary.getName())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME);
} else {
atlasGlossary.setQualifiedName(atlasGlossary.getName());
@@ -177,9 +179,7 @@ public class GlossaryService {
AtlasGlossary storeObject = dataAccess.save(atlasGlossary);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.createGlossary() : {}", storeObject);
- }
+ LOG.debug("<== GlossaryService.createGlossary() : {}", storeObject);
return storeObject;
}
@@ -193,11 +193,9 @@ public class GlossaryService {
*/
@GraphTransaction
public AtlasGlossary getGlossary(String glossaryGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossary({})", glossaryGuid);
- }
+ LOG.debug("==> GlossaryService.getGlossary({})", glossaryGuid);
- if (Objects.isNull(glossaryGuid)) {
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
@@ -206,9 +204,8 @@ public class GlossaryService {
setInfoForRelations(ret);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossary() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossary() : {}", ret);
+
return ret;
}
@@ -221,11 +218,9 @@ public class GlossaryService {
*/
@GraphTransaction
public AtlasGlossary.AtlasGlossaryExtInfo getDetailedGlossary(String glossaryGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossary({})", glossaryGuid);
- }
+ LOG.debug("==> GlossaryService.getGlossary({})", glossaryGuid);
- if (Objects.isNull(glossaryGuid)) {
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
@@ -237,36 +232,37 @@ public class GlossaryService {
// Load all linked terms
if (CollectionUtils.isNotEmpty(ret.getTerms())) {
List<AtlasGlossaryTerm> termsToLoad = ret.getTerms()
- .stream()
- .map(id -> getAtlasGlossaryTermSkeleton(id.getTermGuid()))
- .collect(Collectors.toList());
+ .stream()
+ .map(id -> getAtlasGlossaryTermSkeleton(id.getTermGuid()))
+ .collect(Collectors.toList());
+
Iterable<AtlasGlossaryTerm> glossaryTerms = dataAccess.load(termsToLoad);
+
glossaryTerms.forEach(ret::addTermInfo);
}
// Load all linked categories
if (CollectionUtils.isNotEmpty(ret.getCategories())) {
List<AtlasGlossaryCategory> categoriesToLoad = ret.getCategories()
- .stream()
- .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid()))
- .collect(Collectors.toList());
+ .stream()
+ .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid()))
+ .collect(Collectors.toList());
+
Iterable<AtlasGlossaryCategory> glossaryCategories = dataAccess.load(categoriesToLoad);
+
glossaryCategories.forEach(ret::addCategoryInfo);
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossary() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossary() : {}", ret);
+
return ret;
}
@GraphTransaction
public AtlasGlossary updateGlossary(AtlasGlossary atlasGlossary) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.updateGlossary({})", atlasGlossary);
- }
+ LOG.debug("==> GlossaryService.updateGlossary({})", atlasGlossary);
- if (Objects.isNull(atlasGlossary)) {
+ if (isNull(atlasGlossary)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Glossary is null/empty");
}
@@ -285,38 +281,37 @@ public class GlossaryService {
atlasGlossary.setQualifiedName(storeObject.getQualifiedName());
storeObject = dataAccess.save(atlasGlossary);
+
setInfoForRelations(storeObject);
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.updateGlossary() : {}", storeObject);
- }
+ LOG.debug("<== GlossaryService.updateGlossary() : {}", storeObject);
+
return storeObject;
}
@GraphTransaction
public void deleteGlossary(String glossaryGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.deleteGlossary({})", glossaryGuid);
- }
- if (Objects.isNull(glossaryGuid)) {
+ LOG.debug("==> GlossaryService.deleteGlossary({})", glossaryGuid);
+
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
AtlasGlossary storeObject = dataAccess.load(getGlossarySkeleton(glossaryGuid));
Set<AtlasRelatedTermHeader> terms = storeObject.getTerms();
+
deleteTerms(storeObject, terms);
Set<AtlasRelatedCategoryHeader> categories = storeObject.getCategories();
+
deleteCategories(storeObject, categories);
// Once all relations are deleted, then delete the Glossary
dataAccess.delete(glossaryGuid);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.deleteGlossary()");
- }
+ LOG.debug("<== GlossaryService.deleteGlossary()");
}
/*
@@ -324,36 +319,31 @@ public class GlossaryService {
* */
@GraphTransaction
public AtlasGlossaryTerm getTerm(String termGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getTerm({})", termGuid);
- }
- if (Objects.isNull(termGuid)) {
+ LOG.debug("==> GlossaryService.getTerm({})", termGuid);
+
+ if (isNull(termGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "termGuid is null/empty");
}
-
AtlasGlossaryTerm atlasGlossary = getAtlasGlossaryTermSkeleton(termGuid);
AtlasGlossaryTerm ret = dataAccess.load(atlasGlossary);
setInfoForRelations(ret);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getTerm() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getTerm() : {}", ret);
+
return ret;
}
@GraphTransaction
public AtlasGlossaryTerm createTerm(AtlasGlossaryTerm term) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.create({})", term);
- }
+ LOG.debug("==> GlossaryService.create({})", term);
- if (Objects.isNull(term)) {
+ if (isNull(term)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "GlossaryTerm definition missing");
}
- if (Objects.isNull(term.getAnchor())) {
+ if (isNull(term.getAnchor())) {
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ANCHOR);
}
@@ -361,7 +351,7 @@ public class GlossaryService {
throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_TERM_QUALIFIED_NAME_CANT_BE_DERIVED);
}
- if (isNameInvalid(term.getName())){
+ if (isNameInvalid(term.getName())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME);
} else {
// Derive the qualifiedName
@@ -374,9 +364,7 @@ public class GlossaryService {
term.setQualifiedName(term.getName() + "@" + glossaryQualifiedName);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Derived qualifiedName = {}", term.getQualifiedName());
- }
+ LOG.debug("Derived qualifiedName = {}", term.getQualifiedName());
}
// This might fail for the case when the term's qualifiedName has been updated and the duplicate request comes in with old name
@@ -392,34 +380,28 @@ public class GlossaryService {
storeGlossaryTerm = dataAccess.load(term);
setInfoForRelations(storeGlossaryTerm);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.create() : {}", storeGlossaryTerm);
- }
+ LOG.debug("<== GlossaryService.create() : {}", storeGlossaryTerm);
return storeGlossaryTerm;
}
@GraphTransaction
public List<AtlasGlossaryTerm> createTerms(List<AtlasGlossaryTerm> glossaryTerms) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.create({})", glossaryTerms);
- }
+ LOG.debug("==> GlossaryService.create({})", glossaryTerms);
- if (Objects.isNull(glossaryTerms)) {
+ if (isNull(glossaryTerms)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryTerm(s) is null/empty");
}
-
List<AtlasGlossaryTerm> ret = new ArrayList<>();
+
for (AtlasGlossaryTerm glossaryTerm : glossaryTerms) {
AtlasGlossaryTerm term = createTerm(glossaryTerm);
ret.add(term);
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("<== GlossaryService.createTerms() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.createTerms() : {}", ret);
return ret;
}
@@ -430,11 +412,9 @@ public class GlossaryService {
@GraphTransaction
public AtlasGlossaryTerm updateTerm(AtlasGlossaryTerm atlasGlossaryTerm, boolean ignoreUpdateIfTermExists) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.updateTerm({})", atlasGlossaryTerm);
- }
+ LOG.debug("==> GlossaryService.updateTerm({})", atlasGlossaryTerm);
- if (Objects.isNull(atlasGlossaryTerm)) {
+ if (isNull(atlasGlossaryTerm)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "atlasGlossaryTerm is null/empty");
}
@@ -455,6 +435,7 @@ public class GlossaryService {
}
AtlasGlossaryTerm storeObject = dataAccess.load(atlasGlossaryTerm);
+
if (!storeObject.equals(atlasGlossaryTerm)) {
atlasGlossaryTerm.setGuid(storeObject.getGuid());
atlasGlossaryTerm.setQualifiedName(storeObject.getQualifiedName());
@@ -479,19 +460,16 @@ public class GlossaryService {
setInfoForRelations(storeObject);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.updateTerm() : {}", storeObject);
- }
+ LOG.debug("<== GlossaryService.updateTerm() : {}", storeObject);
return storeObject;
}
@GraphTransaction
public void deleteTerm(String termGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.deleteTerm({})", termGuid);
- }
- if (Objects.isNull(termGuid)) {
+ LOG.debug("==> GlossaryService.deleteTerm({})", termGuid);
+
+ if (isNull(termGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "termGuid is null/empty");
}
@@ -505,20 +483,15 @@ public class GlossaryService {
// Remove term from Glossary
glossaryTermUtils.processTermRelations(storeObject, storeObject, GlossaryUtils.RelationshipOperation.DELETE);
-
// Now delete the term
dataAccess.delete(termGuid);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.deleteTerm()");
- }
+ LOG.debug("<== GlossaryService.deleteTerm()");
}
@GraphTransaction
public void assignTermToEntities(String termGuid, List<AtlasRelatedObjectId> relatedObjectIds) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.assignTermToEntities({}, {})", termGuid, relatedObjectIds);
- }
+ LOG.debug("==> GlossaryService.assignTermToEntities({}, {})", termGuid, relatedObjectIds);
AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid));
@@ -526,17 +499,12 @@ public class GlossaryService {
entityChangeNotifier.onTermAddedToEntities(glossaryTerm, relatedObjectIds);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.assignTermToEntities()");
- }
-
+ LOG.debug("<== GlossaryService.assignTermToEntities()");
}
@GraphTransaction
public void removeTermFromEntities(String termGuid, List<AtlasRelatedObjectId> relatedObjectIds) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.removeTermFromEntities({}, {})", termGuid, relatedObjectIds);
- }
+ LOG.debug("==> GlossaryService.removeTermFromEntities({}, {})", termGuid, relatedObjectIds);
AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid));
@@ -544,9 +512,7 @@ public class GlossaryService {
entityChangeNotifier.onTermDeletedFromEntities(glossaryTerm, relatedObjectIds);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.removeTermFromEntities()");
- }
+ LOG.debug("<== GlossaryService.removeTermFromEntities()");
}
/*
@@ -554,10 +520,9 @@ public class GlossaryService {
* */
@GraphTransaction
public AtlasGlossaryCategory getCategory(String categoryGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getCategory({})", categoryGuid);
- }
- if (Objects.isNull(categoryGuid)) {
+ LOG.debug("==> GlossaryService.getCategory({})", categoryGuid);
+
+ if (isNull(categoryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "categoryGuid is null/empty");
}
@@ -566,40 +531,37 @@ public class GlossaryService {
setInfoForRelations(ret);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getCategory() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getCategory() : {}", ret);
+
return ret;
}
@GraphTransaction
public AtlasGlossaryCategory createCategory(AtlasGlossaryCategory glossaryCategory) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.createCategory({})", glossaryCategory);
- }
+ LOG.debug("==> GlossaryService.createCategory({})", glossaryCategory);
- if (Objects.isNull(glossaryCategory)) {
+ if (isNull(glossaryCategory)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "GlossaryCategory definition missing");
}
- if (Objects.isNull(glossaryCategory.getAnchor())) {
+
+ if (isNull(glossaryCategory.getAnchor())) {
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ANCHOR);
}
+
if (StringUtils.isEmpty(glossaryCategory.getName())) {
throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_CATEGORY_QUALIFIED_NAME_CANT_BE_DERIVED);
}
- if (isNameInvalid(glossaryCategory.getName())){
+
+ if (isNameInvalid(glossaryCategory.getName())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME);
} else {
// Derive the qualifiedName
- String anchorGlossaryGuid = glossaryCategory.getAnchor().getGlossaryGuid();
- AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid));
- glossaryCategory.setQualifiedName(glossaryCategory.getName()+ "@" + glossary.getQualifiedName());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Derived qualifiedName = {}", glossaryCategory.getQualifiedName());
- }
+ String anchorGlossaryGuid = glossaryCategory.getAnchor().getGlossaryGuid();
+ AtlasGlossary glossary = dataAccess.load(getGlossarySkeleton(anchorGlossaryGuid));
+ glossaryCategory.setQualifiedName(glossaryCategory.getName() + "@" + glossary.getQualifiedName());
+ LOG.debug("Derived qualifiedName = {}", glossaryCategory.getQualifiedName());
}
// This might fail for the case when the category's qualifiedName has been updated during a hierarchy change
@@ -631,39 +593,35 @@ public class GlossaryService {
setInfoForRelations(storeObject);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.createCategory() : {}", storeObject);
- }
+ LOG.debug("<== GlossaryService.createCategory() : {}", storeObject);
return storeObject;
}
@GraphTransaction
public List<AtlasGlossaryCategory> createCategories(List<AtlasGlossaryCategory> glossaryCategory) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.createCategories({})", glossaryCategory);
- }
- if (Objects.isNull(glossaryCategory)) {
+ LOG.debug("==> GlossaryService.createCategories({})", glossaryCategory);
+
+ if (isNull(glossaryCategory)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryCategory is null/empty");
}
List<AtlasGlossaryCategory> ret = new ArrayList<>();
+
for (AtlasGlossaryCategory category : glossaryCategory) {
ret.add(createCategory(category));
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.createCategories() : {}", ret);
- }
+
+ LOG.debug("<== GlossaryService.createCategories() : {}", ret);
+
return ret;
}
@GraphTransaction
public AtlasGlossaryCategory updateCategory(AtlasGlossaryCategory glossaryCategory) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.updateCategory({})", glossaryCategory);
- }
+ LOG.debug("==> GlossaryService.updateCategory({})", glossaryCategory);
- if (Objects.isNull(glossaryCategory)) {
+ if (isNull(glossaryCategory)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "GlossaryCategory is null/empty");
}
@@ -705,9 +663,7 @@ public class GlossaryService {
dataAccess.save(impactedCategories.values());
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.updateCategory() : {}", storeObject);
- }
+ LOG.debug("<== GlossaryService.updateCategory() : {}", storeObject);
setInfoForRelations(storeObject);
@@ -716,10 +672,9 @@ public class GlossaryService {
@GraphTransaction
public void deleteCategory(String categoryGuid) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.deleteCategory({})", categoryGuid);
- }
- if (Objects.isNull(categoryGuid)) {
+ LOG.debug("==> GlossaryService.deleteCategory({})", categoryGuid);
+
+ if (isNull(categoryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Category guid is null/empty");
}
@@ -731,228 +686,208 @@ public class GlossaryService {
// Now delete the category
dataAccess.delete(categoryGuid);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.deleteCategory()");
- }
+ LOG.debug("<== GlossaryService.deleteCategory()");
}
@GraphTransaction
public List<AtlasRelatedTermHeader> getGlossaryTermsHeaders(String glossaryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(glossaryGuid)) {
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossaryTermsHeaders({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
- }
+ LOG.debug("==> GlossaryService.getGlossaryTermsHeaders({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
AtlasGlossary glossary = getGlossary(glossaryGuid);
List<AtlasRelatedTermHeader> ret;
if (CollectionUtils.isNotEmpty(glossary.getTerms())) {
List<AtlasRelatedTermHeader> terms = new ArrayList<>(glossary.getTerms());
+
if (sortOrder != null) {
terms.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ?
- o1.getDisplayText().compareTo(o2.getDisplayText()) :
- o2.getDisplayText().compareTo(o1.getDisplayText()));
+ o1.getDisplayText().compareTo(o2.getDisplayText()) :
+ o2.getDisplayText().compareTo(o1.getDisplayText()));
}
+
ret = new PaginationHelper<>(terms, offset, limit).getPaginatedList();
} else {
ret = Collections.emptyList();
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossaryTermsHeaders() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossaryTermsHeaders() : {}", ret);
return ret;
}
@GraphTransaction
public List<AtlasGlossaryTerm> getGlossaryTerms(String glossaryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(glossaryGuid)) {
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossaryTerms({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
- }
-
- List<AtlasGlossaryTerm> ret = new ArrayList<>();
+ LOG.debug("==> GlossaryService.getGlossaryTerms({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
+ List<AtlasGlossaryTerm> ret = new ArrayList<>();
List<AtlasRelatedTermHeader> termHeaders = getGlossaryTermsHeaders(glossaryGuid, offset, limit, sortOrder);
+
for (AtlasRelatedTermHeader header : termHeaders) {
ret.add(dataAccess.load(getAtlasGlossaryTermSkeleton(header.getTermGuid())));
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossaryTerms() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossaryTerms() : {}", ret);
return ret;
}
@GraphTransaction
public List<AtlasRelatedCategoryHeader> getGlossaryCategoriesHeaders(String glossaryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(glossaryGuid)) {
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossaryCategoriesHeaders({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
- }
+ LOG.debug("==> GlossaryService.getGlossaryCategoriesHeaders({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
List<AtlasRelatedCategoryHeader> ret;
-
- AtlasGlossary glossary = getGlossary(glossaryGuid);
+ AtlasGlossary glossary = getGlossary(glossaryGuid);
if (CollectionUtils.isNotEmpty(glossary.getCategories())) {
List<AtlasRelatedCategoryHeader> categories = new ArrayList<>(glossary.getCategories());
+
if (sortOrder != null) {
categories.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ?
- o1.getDisplayText().compareTo(o2.getDisplayText()) :
- o2.getDisplayText().compareTo(o1.getDisplayText()));
+ o1.getDisplayText().compareTo(o2.getDisplayText()) :
+ o2.getDisplayText().compareTo(o1.getDisplayText()));
}
+
ret = new PaginationHelper<>(categories, offset, limit).getPaginatedList();
} else {
ret = Collections.emptyList();
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossaryCategoriesHeaders() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossaryCategoriesHeaders() : {}", ret);
return ret;
}
@GraphTransaction
public List<AtlasGlossaryCategory> getGlossaryCategories(String glossaryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(glossaryGuid)) {
+ if (isNull(glossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "glossaryGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getGlossaryCategories({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
- }
-
- List<AtlasGlossaryCategory> ret = new ArrayList<>();
+ LOG.debug("==> GlossaryService.getGlossaryCategories({}, {}, {}, {})", glossaryGuid, offset, limit, sortOrder);
+ List<AtlasGlossaryCategory> ret = new ArrayList<>();
List<AtlasRelatedCategoryHeader> categoryHeaders = getGlossaryCategoriesHeaders(glossaryGuid, offset, limit, sortOrder);
+
for (AtlasRelatedCategoryHeader header : categoryHeaders) {
ret.add(dataAccess.load(getAtlasGlossaryCategorySkeleton(header.getCategoryGuid())));
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getGlossaryCategories() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getGlossaryCategories() : {}", ret);
return ret;
}
@GraphTransaction
public List<AtlasRelatedTermHeader> getCategoryTerms(String categoryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(categoryGuid)) {
+ if (isNull(categoryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "categoryGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getCategoryTerms({}, {}, {}, {})", categoryGuid, offset, limit, sortOrder);
- }
+ LOG.debug("==> GlossaryService.getCategoryTerms({}, {}, {}, {})", categoryGuid, offset, limit, sortOrder);
List<AtlasRelatedTermHeader> ret;
-
- AtlasGlossaryCategory glossaryCategory = getCategory(categoryGuid);
+ AtlasGlossaryCategory glossaryCategory = getCategory(categoryGuid);
if (CollectionUtils.isNotEmpty(glossaryCategory.getTerms())) {
List<AtlasRelatedTermHeader> terms = new ArrayList<>(glossaryCategory.getTerms());
+
if (sortOrder != null) {
terms.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ?
- o1.getDisplayText().compareTo(o2.getDisplayText()) :
- o2.getDisplayText().compareTo(o1.getDisplayText()));
+ o1.getDisplayText().compareTo(o2.getDisplayText()) :
+ o2.getDisplayText().compareTo(o1.getDisplayText()));
}
+
ret = new PaginationHelper<>(terms, offset, limit).getPaginatedList();
} else {
ret = Collections.emptyList();
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getCategoryTerms() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getCategoryTerms() : {}", ret);
+
return ret;
}
@GraphTransaction
public Map<AtlasGlossaryTerm.Relation, Set<AtlasRelatedTermHeader>> getRelatedTerms(String termGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(termGuid)) {
+ if (isNull(termGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "termGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getRelatedTerms({}, {}, {}, {})", termGuid, offset, limit, sortOrder);
- }
+ LOG.debug("==> GlossaryService.getRelatedTerms({}, {}, {}, {})", termGuid, offset, limit, sortOrder);
AtlasGlossaryTerm glossaryTerm = getTerm(termGuid);
Map<AtlasGlossaryTerm.Relation, Set<AtlasRelatedTermHeader>> ret;
+
if (glossaryTerm.hasTerms()) {
ret = glossaryTerm.getRelatedTerms();
} else {
ret = Collections.emptyMap();
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getRelatedTerms() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getRelatedTerms() : {}", ret);
return ret;
}
@GraphTransaction
public Map<String, List<AtlasRelatedCategoryHeader>> getRelatedCategories(String categoryGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(categoryGuid)) {
+ if (isNull(categoryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "categoryGuid is null/empty");
}
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryService.getRelatedCategories({}, {}, {}, {})", categoryGuid, offset, limit, sortOrder);
- }
+ LOG.debug("==> GlossaryService.getRelatedCategories({}, {}, {}, {})", categoryGuid, offset, limit, sortOrder);
- AtlasGlossaryCategory glossaryCategory = getCategory(categoryGuid);
+ Map<String, List<AtlasRelatedCategoryHeader>> ret = new HashMap<>();
+ AtlasGlossaryCategory glossaryCategory = getCategory(categoryGuid);
- Map<String, List<AtlasRelatedCategoryHeader>> ret = new HashMap<>();
if (glossaryCategory.getParentCategory() != null) {
- ret.put("parent", new ArrayList<AtlasRelatedCategoryHeader>() {{
- add(glossaryCategory.getParentCategory());
- }});
+ List<AtlasRelatedCategoryHeader> headers = new ArrayList<>();
+
+ headers.add(glossaryCategory.getParentCategory());
+
+ ret.put("parent", headers);
}
+
if (CollectionUtils.isNotEmpty(glossaryCategory.getChildrenCategories())) {
ret.put("children", new ArrayList<>(glossaryCategory.getChildrenCategories()));
}
-
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryService.getRelatedCategories() : {}", ret);
- }
+ LOG.debug("<== GlossaryService.getRelatedCategories() : {}", ret);
return ret;
}
@GraphTransaction
public List<AtlasRelatedObjectId> getAssignedEntities(final String termGuid, int offset, int limit, SortOrder sortOrder) throws AtlasBaseException {
- if (Objects.isNull(termGuid)) {
+ if (isNull(termGuid)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "termGuid is null/empty");
}
- AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid));
- Set<AtlasRelatedObjectId> assignedEntities = glossaryTerm.getAssignedEntities();
-
+ AtlasGlossaryTerm glossaryTerm = dataAccess.load(getAtlasGlossaryTermSkeleton(termGuid));
+ Set<AtlasRelatedObjectId> assignedEntities = glossaryTerm.getAssignedEntities();
List<AtlasRelatedObjectId> ret;
+
if (CollectionUtils.isNotEmpty(assignedEntities)) {
ret = new ArrayList<>(assignedEntities);
+
if (sortOrder != null) {
ret.sort((o1, o2) -> sortOrder == SortOrder.ASCENDING ?
- o1.getDisplayText().compareTo(o2.getDisplayText()) :
- o2.getDisplayText().compareTo(o1.getDisplayText()));
+ o1.getDisplayText().compareTo(o2.getDisplayText()) :
+ o2.getDisplayText().compareTo(o1.getDisplayText()));
}
+
ret = new PaginationHelper<>(assignedEntities, offset, limit).getPaginatedList();
} else {
ret = Collections.emptyList();
@@ -961,15 +896,42 @@ public class GlossaryService {
return ret;
}
+ @GraphTransaction
+ public BulkImportResponse importGlossaryData(InputStream inputStream, String fileName) throws AtlasBaseException {
+ BulkImportResponse ret = new BulkImportResponse();
+
+ if (StringUtils.isBlank(fileName)) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName);
+ }
+
+ try {
+ List<String[]> fileData = FileUtils.readFileData(fileName, inputStream);
+ List<AtlasGlossaryTerm> glossaryTermsWithoutRelations = glossaryTermUtils.getGlossaryTermDataWithoutRelations(fileData, ret);
+
+ createGlossaryTerms(glossaryTermsWithoutRelations, ret);
+
+ List<AtlasGlossaryTerm> glossaryTermsWithRelations = glossaryTermUtils.getGlossaryTermDataWithRelations(fileData, ret);
+
+ updateGlossaryTermsRelation(glossaryTermsWithRelations, ret);
+ } finally {
+ glossaryTermUtils.clearImportCache();
+ }
+
+ return ret;
+ }
+
private boolean glossaryExists(AtlasGlossary atlasGlossary) {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(atlasTypeRegistry.getEntityTypeByName(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME), new HashMap<String, Object>() {{
- put(QUALIFIED_NAME_ATTR, atlasGlossary.getQualifiedName());
- }});
- return Objects.nonNull(vertex);
+ Map<String, Object> uniqAttr = new HashMap<>();
+
+ uniqAttr.put(QUALIFIED_NAME_ATTR, atlasGlossary.getQualifiedName());
+
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(atlasTypeRegistry.getEntityTypeByName(GlossaryUtils.ATLAS_GLOSSARY_TYPENAME), uniqAttr);
+
+ return nonNull(vertex);
}
private String getGlossaryQualifiedName(String glossaryGuid) {
- String ret = glossaryGuidQualifiedNameCache.get(glossaryGuid);
+ String ret = GLOSSARY_GUID_QUALIFIED_NAME_CACHE.get(glossaryGuid);
if (StringUtils.isEmpty(ret)) {
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(glossaryGuid);
@@ -977,7 +939,7 @@ public class GlossaryService {
if (vertex != null) {
ret = vertex.getProperty(GLOSSARY_QUALIFIED_NAME_PROPERTY, String.class);
- glossaryGuidQualifiedNameCache.put(glossaryGuid, ret);
+ GLOSSARY_GUID_QUALIFIED_NAME_CACHE.put(glossaryGuid, ret);
}
}
@@ -985,7 +947,7 @@ public class GlossaryService {
}
private String getGlossaryCategoryName(String glossaryCategoryGuid) {
- String ret = categoryGuidNameCache.get(glossaryCategoryGuid);
+ String ret = CATEGORY_GUID_NAME_CACHE.get(glossaryCategoryGuid);
if (StringUtils.isEmpty(ret)) {
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(glossaryCategoryGuid);
@@ -993,7 +955,7 @@ public class GlossaryService {
if (vertex != null) {
ret = vertex.getProperty(GLOSSARY_CATEGORY_NAME_PROPERTY, String.class);
- categoryGuidNameCache.put(glossaryCategoryGuid, ret);
+ CATEGORY_GUID_NAME_CACHE.put(glossaryCategoryGuid, ret);
}
}
@@ -1001,11 +963,13 @@ public class GlossaryService {
}
private boolean termExists(AtlasGlossaryTerm term) {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(atlasTypeRegistry.getEntityTypeByName(GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME), new HashMap<String, Object>() {{
- put(QUALIFIED_NAME_ATTR, term.getQualifiedName());
- }});
+ Map<String, Object> uniqAttr = new HashMap<>();
- return Objects.nonNull(vertex);
+ uniqAttr.put(QUALIFIED_NAME_ATTR, term.getQualifiedName());
+
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(atlasTypeRegistry.getEntityTypeByName(GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME), uniqAttr);
+
+ return nonNull(vertex);
}
private boolean termExists2(AtlasGlossaryTerm term) {
@@ -1013,8 +977,8 @@ public class GlossaryService {
AtlasVertex glossaryVertex = AtlasGraphUtilsV2.findByGuid(term.getAnchor().getGlossaryGuid());
Iterable<AtlasEdge> glossaryTermEdges = glossaryVertex.getEdges(AtlasEdgeDirection.OUT, GLOSSARY_TERM_ANCHOR_EDGE_LABEL);
- for (Iterator<AtlasEdge> iter = glossaryTermEdges.iterator(); iter.hasNext(); ) {
- AtlasVertex termVertex = iter.next().getInVertex();
+ for (AtlasEdge glossaryTermEdge : glossaryTermEdges) {
+ AtlasVertex termVertex = glossaryTermEdge.getInVertex();
String termQualifiedName = termVertex.getProperty(TERM_UNIQUE_QUALIFIED_NAME_PROPERTY, String.class);
if (StringUtils.equals(termQualifiedName, term.getQualifiedName())) {
@@ -1027,17 +991,19 @@ public class GlossaryService {
}
private boolean categoryExists(AtlasGlossaryCategory category) {
- AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(atlasTypeRegistry.getEntityTypeByName(GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME), new HashMap<String, Object>() {{
- put(QUALIFIED_NAME_ATTR, category.getQualifiedName());
- }});
- return Objects.nonNull(vertex);
+ Map<String, Object> uniqAttr = new HashMap<>();
+
+ uniqAttr.put(QUALIFIED_NAME_ATTR, category.getQualifiedName());
+
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByUniqueAttributes(atlasTypeRegistry.getEntityTypeByName(GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME), uniqAttr);
+
+ return nonNull(vertex);
}
private void deleteCategories(final AtlasGlossary storeObject, final Collection<AtlasRelatedCategoryHeader> categories) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(categories)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Deleting categories within glossary guid = {}", storeObject.getGuid());
- }
+ LOG.debug("Deleting categories within glossary guid = {}", storeObject.getGuid());
+
for (AtlasRelatedCategoryHeader category : categories) {
// Delete category
deleteCategory(category.getCategoryGuid());
@@ -1047,9 +1013,8 @@ public class GlossaryService {
private void deleteTerms(final AtlasGlossary storeObject, final Collection<AtlasRelatedTermHeader> terms) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(terms)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Deleting terms within glossary guid = {}", storeObject.getGuid());
- }
+ LOG.debug("Deleting terms within glossary guid = {}", storeObject.getGuid());
+
for (AtlasRelatedTermHeader term : terms) {
// Delete the term
deleteTerm(term.getTermGuid());
@@ -1058,19 +1023,19 @@ public class GlossaryService {
}
private void setInfoForRelations(final AtlasGlossary ret) throws AtlasBaseException {
- if (Objects.nonNull(ret.getTerms())) {
+ if (nonNull(ret.getTerms())) {
setDisplayNameForTerms(ret.getTerms());
}
- if (Objects.nonNull(ret.getCategories())) {
+ if (nonNull(ret.getCategories())) {
setInfoForRelatedCategories(ret.getCategories());
}
}
- private void setInfoForRelations(final AtlasGlossaryTerm glossaryTerm) throws AtlasBaseException {
+ private void setInfoForRelations(final AtlasGlossaryTerm glossaryTerm) {
setDisplayNameForCategories(glossaryTerm.getCategories());
- if (Objects.nonNull(glossaryTerm.getRelatedTerms())) {
+ if (nonNull(glossaryTerm.getRelatedTerms())) {
for (Map.Entry<AtlasGlossaryTerm.Relation, Set<AtlasRelatedTermHeader>> entry : glossaryTerm.getRelatedTerms().entrySet()) {
setDisplayNameForTerms(entry.getValue());
setQualifiedNameForTerms(entry.getValue());
@@ -1079,15 +1044,16 @@ public class GlossaryService {
}
private void setInfoForRelations(final AtlasGlossaryCategory glossaryCategory) throws AtlasBaseException {
- if (Objects.nonNull(glossaryCategory.getChildrenCategories())) {
+ if (nonNull(glossaryCategory.getChildrenCategories())) {
setInfoForRelatedCategories(glossaryCategory.getChildrenCategories());
}
- if (Objects.nonNull(glossaryCategory.getTerms())) {
+
+ if (nonNull(glossaryCategory.getTerms())) {
setDisplayNameForTerms(glossaryCategory.getTerms());
}
}
- private void setDisplayNameForCategories(final Set<AtlasTermCategorizationHeader> categorizationHeaders) throws AtlasBaseException {
+ private void setDisplayNameForCategories(final Set<AtlasTermCategorizationHeader> categorizationHeaders) {
if (CollectionUtils.isEmpty(categorizationHeaders)) {
return;
}
@@ -1104,50 +1070,52 @@ public class GlossaryService {
private void setInfoForRelatedCategories(final Collection<AtlasRelatedCategoryHeader> categoryHeaders) throws AtlasBaseException {
List<AtlasGlossaryCategory> categories = categoryHeaders
- .stream()
- .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid()))
- .collect(Collectors.toList());
+ .stream()
+ .map(id -> getAtlasGlossaryCategorySkeleton(id.getCategoryGuid()))
+ .collect(Collectors.toList());
+
Map<String, AtlasGlossaryCategory> categoryMap = new HashMap<>();
+
dataAccess.load(categories).forEach(c -> categoryMap.put(c.getGuid(), c));
+
for (AtlasRelatedCategoryHeader c : categoryHeaders) {
AtlasGlossaryCategory category = categoryMap.get(c.getCategoryGuid());
+
c.setDisplayText(category.getName());
- if (Objects.nonNull(category.getParentCategory())) {
+
+ if (nonNull(category.getParentCategory())) {
c.setParentCategoryGuid(category.getParentCategory().getCategoryGuid());
}
}
}
- private void setDisplayNameForTerms(final Collection<AtlasRelatedTermHeader> termHeaders) throws AtlasBaseException {
-
+ private void setDisplayNameForTerms(final Collection<AtlasRelatedTermHeader> termHeaders) {
for (AtlasRelatedTermHeader termHeader : termHeaders) {
String termGuid = termHeader.getTermGuid();
AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(termGuid);
String termDisplayText = termVertex.getProperty(GLOSSARY_TERM_NAME_PROPERTY, String.class);
+
if (StringUtils.isNotEmpty(termDisplayText)) {
termHeader.setDisplayText(termDisplayText);
}
}
}
- private void setQualifiedNameForTerms(final Collection<AtlasRelatedTermHeader> termHeaders) throws AtlasBaseException {
-
+ private void setQualifiedNameForTerms(final Collection<AtlasRelatedTermHeader> termHeaders) {
for (AtlasRelatedTermHeader termHeader : termHeaders) {
- String termGuid = termHeader.getTermGuid();
- AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(termGuid);
+ String termGuid = termHeader.getTermGuid();
+ AtlasVertex termVertex = AtlasGraphUtilsV2.findByGuid(termGuid);
String termQualifiedName = termVertex.getProperty(GLOSSARY_TERM_QUALIFIED_NAME_PROPERTY, String.class);
+
if (StringUtils.isNotEmpty(termQualifiedName)) {
termHeader.setQualifiedName(termQualifiedName);
}
}
}
- public static boolean isNameInvalid(String name) {
- return StringUtils.containsAny(name, invalidNameChars);
- }
-
private String getDuplicateGlossaryRelatedTerm(AtlasGlossaryTerm atlasGlossaryTerm) throws AtlasBaseException {
Map<AtlasGlossaryTerm.Relation, Set<AtlasRelatedTermHeader>> relatedTermsMap = atlasGlossaryTerm.getRelatedTerms();
+
for (Map.Entry<AtlasGlossaryTerm.Relation, Set<AtlasRelatedTermHeader>> relatedTermsMapEntry : relatedTermsMap.entrySet()) {
Set<AtlasRelatedTermHeader> termHeaders = relatedTermsMapEntry.getValue();
@@ -1159,13 +1127,15 @@ public class GlossaryService {
.flatMap(Collection::stream)
.collect(Collectors.toList());
- if (CollectionUtils.isNotEmpty(duplicateTermHeaders) && duplicateTermHeaders.size() > 0) {
- String dupTermGuid = duplicateTermHeaders.get(0).getTermGuid();
+ if (CollectionUtils.isNotEmpty(duplicateTermHeaders) && !duplicateTermHeaders.isEmpty()) {
+ String dupTermGuid = duplicateTermHeaders.get(0).getTermGuid();
AtlasGlossaryTerm glossaryTerm = getTerm(dupTermGuid);
+
return glossaryTerm.getQualifiedName();
}
}
}
+
return StringUtils.EMPTY;
}
@@ -1173,72 +1143,6 @@ public class GlossaryService {
return term != null ? term.getName() : null;
}
- static class PaginationHelper<T> {
- private int pageStart;
- private int pageEnd;
- private int maxSize;
- private List<T> items;
-
- PaginationHelper(Collection<T> items, int offset, int limit) {
- Objects.requireNonNull(items, "items can't be empty/null");
- this.items = new ArrayList<>(items);
- this.maxSize = items.size();
-
- // If limit is negative then limit is effectively the maxSize, else the smaller one out of limit and maxSize
- int adjustedLimit = limit < 0 ? maxSize : Integer.min(maxSize, limit);
- // Page starting can only be between zero and adjustedLimit
- pageStart = Integer.max(0, offset);
- // Page end can't exceed the maxSize
- pageEnd = Integer.min(adjustedLimit + pageStart, maxSize);
- }
-
- List<T> getPaginatedList() {
- List<T> ret;
- if (isValidOffset()) {
- if (isPagingNeeded()) {
- ret = items.subList(pageStart, pageEnd);
- } else {
- ret = items;
- }
- } else {
- ret = Collections.emptyList();
- }
-
- return ret;
- }
-
- private boolean isPagingNeeded() {
- return !(pageStart == 0 && pageEnd == maxSize) && pageStart <= pageEnd;
- }
-
- private boolean isValidOffset() {
- return pageStart <= maxSize;
- }
- }
-
- @GraphTransaction
- public BulkImportResponse importGlossaryData(InputStream inputStream, String fileName) throws AtlasBaseException {
- BulkImportResponse ret = new BulkImportResponse();
-
- if (StringUtils.isBlank(fileName)) {
- throw new AtlasBaseException(AtlasErrorCode.INVALID_FILE_TYPE, fileName);
- }
-
- try {
- List<String[]> fileData = FileUtils.readFileData(fileName, inputStream);
-
- List<AtlasGlossaryTerm> glossaryTermsWithoutRelations = glossaryTermUtils.getGlossaryTermDataWithoutRelations(fileData, ret);
- createGlossaryTerms(glossaryTermsWithoutRelations, ret);
-
- List<AtlasGlossaryTerm> glossaryTermsWithRelations = glossaryTermUtils.getGlossaryTermDataWithRelations(fileData, ret);
- updateGlossaryTermsRelation(glossaryTermsWithRelations, ret);
- } finally {
- glossaryTermUtils.clearImportCache();
- }
-
- return ret;
- }
-
private void createGlossaryTerms(List<AtlasGlossaryTerm> glossaryTerms, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
for (AtlasGlossaryTerm glossaryTerm : glossaryTerms) {
String glossaryTermName = glossaryTerm.getName();
@@ -1281,7 +1185,7 @@ public class GlossaryService {
String ret = "";
String glossaryTermQName = glossaryTerm.getQualifiedName();
- if (StringUtils.isNotBlank(glossaryTermQName)){
+ if (StringUtils.isNotBlank(glossaryTermQName)) {
String[] glossaryQnameSplit = glossaryTermQName.split("@");
ret = (glossaryQnameSplit.length == 2) ? glossaryQnameSplit[1] : "";
@@ -1295,4 +1199,50 @@ public class GlossaryService {
throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_IMPORT_FAILED);
}
}
+
+ static class PaginationHelper<T> {
+ private final int pageStart;
+ private final int pageEnd;
+ private final int maxSize;
+ private final List<T> items;
+
+ PaginationHelper(Collection<T> items, int offset, int limit) {
+ requireNonNull(items, "items can't be empty/null");
+
+ this.items = new ArrayList<>(items);
+ this.maxSize = items.size();
+
+ // If limit is negative then limit is effectively the maxSize, else the smaller one out of limit and maxSize
+ int adjustedLimit = limit < 0 ? maxSize : Integer.min(maxSize, limit);
+
+ // Page starting can only be between zero and adjustedLimit
+ pageStart = Integer.max(0, offset);
+ // Page end can't exceed the maxSize
+ pageEnd = Integer.min(adjustedLimit + pageStart, maxSize);
+ }
+
+ List<T> getPaginatedList() {
+ List<T> ret;
+
+ if (isValidOffset()) {
+ if (isPagingNeeded()) {
+ ret = items.subList(pageStart, pageEnd);
+ } else {
+ ret = items;
+ }
+ } else {
+ ret = Collections.emptyList();
+ }
+
+ return ret;
+ }
+
+ private boolean isPagingNeeded() {
+ return !(pageStart == 0 && pageEnd == maxSize) && pageStart <= pageEnd;
+ }
+
+ private boolean isValidOffset() {
+ return pageStart <= maxSize;
+ }
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
index 8a161e420..601d4ac1e 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
@@ -18,7 +18,6 @@
package org.apache.atlas.glossary;
import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.RequestContext;
import org.apache.atlas.bulkimport.BulkImportResponse;
import org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo;
import org.apache.atlas.exception.AtlasBaseException;
@@ -38,7 +37,6 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.util.FileUtils;
-import org.apache.atlas.utils.AtlasPerfMetrics;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
@@ -51,80 +49,101 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
+import static java.util.Objects.isNull;
+import static java.util.Objects.nonNull;
+import static java.util.Objects.requireNonNull;
import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED;
public class GlossaryTermUtils extends GlossaryUtils {
- private static final Logger LOG = LoggerFactory.getLogger(GlossaryTermUtils.class);
- private static final boolean DEBUG_ENABLED = LOG.isDebugEnabled();
+ private static final Logger LOG = LoggerFactory.getLogger(GlossaryTermUtils.class);
private static final int INDEX_FOR_GLOSSARY_AT_RECORD = 0;
private static final int INDEX_FOR_TERM_AT_RECORD = 1;
- private static final ThreadLocal<Map<String, String>> glossaryNameGuidCache = ThreadLocal.withInitial(() -> new LinkedHashMap<>());
- private static final ThreadLocal<Map<String, Integer>> glossaryTermOrderCache = ThreadLocal.withInitial(() -> new HashMap<>());
- private static final ThreadLocal<Map<String, String>> glossaryTermQNameGuidCache = ThreadLocal.withInitial(() -> new HashMap<>());
+ private static final ThreadLocal<Map<String, String>> GLOSSARY_NAME_GUID_CACHE = ThreadLocal.withInitial(LinkedHashMap::new);
+ private static final ThreadLocal<Map<String, Integer>> GLOSSARY_TERM_ORDER_CACHE = ThreadLocal.withInitial(HashMap::new);
+ private static final ThreadLocal<Map<String, String>> GLOSSARY_TERM_Q_NAME_GUID_CACHE = ThreadLocal.withInitial(HashMap::new);
protected GlossaryTermUtils(AtlasRelationshipStore relationshipStore, AtlasTypeRegistry typeRegistry, DataAccess dataAccess) {
super(relationshipStore, typeRegistry, dataAccess);
}
+ public static String getGlossaryTermHeaders() {
+ List<String> ret = new ArrayList<>();
+
+ ret.add("GlossaryName");
+ ret.add("TermName");
+ ret.add("ShortDescription");
+ ret.add("LongDescription");
+ ret.add("Examples");
+ ret.add("Abbreviation");
+ ret.add("Usage");
+ ret.add("AdditionalAttributes");
+ ret.add("TranslationTerms");
+ ret.add("ValidValuesFor");
+ ret.add("Synonyms");
+ ret.add("ReplacedBy");
+ ret.add("ValidValues");
+ ret.add("ReplacementTerms");
+ ret.add("SeeAlso");
+ ret.add("TranslatedTerms");
+ ret.add("IsA");
+ ret.add("Antonyms");
+ ret.add("Classifies");
+ ret.add("PreferredToTerms");
+ ret.add("PreferredTerms");
+
+ return String.join(", ", ret);
+ }
+
public void processTermRelations(AtlasGlossaryTerm storeObject, AtlasGlossaryTerm updatedTerm, RelationshipOperation op) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryTermUtils.processTermRelations({}, {}, {})", storeObject, updatedTerm, op);
- }
+ LOG.debug("==> GlossaryTermUtils.processTermRelations({}, {}, {})", storeObject, updatedTerm, op);
processTermAnchor(storeObject, updatedTerm, op);
processRelatedTerms(storeObject, updatedTerm, op);
processAssociatedCategories(storeObject, updatedTerm, op);
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryTermUtils.processTermRelations()");
- }
+ LOG.debug("<== GlossaryTermUtils.processTermRelations()");
}
public void processTermAssignments(AtlasGlossaryTerm glossaryTerm, Collection<AtlasRelatedObjectId> relatedObjectIds) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryTermUtils.processTermAssignments({}, {})", glossaryTerm, relatedObjectIds);
- }
+ LOG.debug("==> GlossaryTermUtils.processTermAssignments({}, {})", glossaryTerm, relatedObjectIds);
- Objects.requireNonNull(glossaryTerm);
+ requireNonNull(glossaryTerm);
Set<AtlasRelatedObjectId> assignedEntities = glossaryTerm.getAssignedEntities();
+
for (AtlasRelatedObjectId objectId : relatedObjectIds) {
if (CollectionUtils.isNotEmpty(assignedEntities) && assignedEntities.contains(objectId)) {
- if (DEBUG_ENABLED) {
- LOG.debug("Skipping already assigned entity {}", objectId);
- continue;
- }
- }
+ LOG.debug("Skipping already assigned entity {}", objectId);
- if (DEBUG_ENABLED) {
- LOG.debug("Assigning term guid={}, to entity guid = {}", glossaryTerm.getGuid(), objectId.getGuid());
+ continue;
}
+
+ LOG.debug("Assigning term guid={}, to entity guid = {}", glossaryTerm.getGuid(), objectId.getGuid());
+
createRelationship(defineTermAssignment(glossaryTerm.getGuid(), objectId));
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryTermUtils.processTermAssignments()");
- }
+ LOG.debug("<== GlossaryTermUtils.processTermAssignments()");
}
public void processTermDissociation(AtlasGlossaryTerm glossaryTerm, Collection<AtlasRelatedObjectId> relatedObjectIds) throws AtlasBaseException {
- if (DEBUG_ENABLED) {
- LOG.debug("==> GlossaryTermUtils.processTermDissociation({}, {}, {})", glossaryTerm.getGuid(), relatedObjectIds, glossaryTerm);
- }
+ LOG.debug("==> GlossaryTermUtils.processTermDissociation({}, {}, {})", glossaryTerm.getGuid(), relatedObjectIds, glossaryTerm);
- Objects.requireNonNull(glossaryTerm);
- Set<AtlasRelatedObjectId> assignedEntities = glossaryTerm.getAssignedEntities();
+ requireNonNull(glossaryTerm);
+
+ Set<AtlasRelatedObjectId> assignedEntities = glossaryTerm.getAssignedEntities();
Map<String, AtlasRelatedObjectId> assignedEntityMap = new HashMap<>();
+
if (CollectionUtils.isNotEmpty(assignedEntities)) {
for (AtlasRelatedObjectId relatedObjectId : assignedEntities) {
assignedEntityMap.put(relatedObjectId.getGuid(), relatedObjectId);
@@ -133,13 +152,14 @@ public class GlossaryTermUtils extends GlossaryUtils {
if (CollectionUtils.isNotEmpty(relatedObjectIds)) {
for (AtlasRelatedObjectId relatedObjectId : relatedObjectIds) {
- if (DEBUG_ENABLED) {
- LOG.debug("Removing term guid={}, from entity guid = {}", glossaryTerm.getGuid(), relatedObjectId.getGuid());
- }
- if (Objects.isNull(relatedObjectId.getRelationshipGuid())) {
+ LOG.debug("Removing term guid={}, from entity guid = {}", glossaryTerm.getGuid(), relatedObjectId.getGuid());
+
+ if (isNull(relatedObjectId.getRelationshipGuid())) {
throw new AtlasBaseException(AtlasErrorCode.TERM_DISSOCIATION_MISSING_RELATION_GUID);
}
+
AtlasRelatedObjectId existingTermRelation = assignedEntityMap.get(relatedObjectId.getGuid());
+
if (CollectionUtils.isNotEmpty(assignedEntities) && isRelationshipGuidSame(existingTermRelation, relatedObjectId)) {
relationshipStore.deleteById(relatedObjectId.getRelationshipGuid(), true);
} else {
@@ -148,15 +168,248 @@ public class GlossaryTermUtils extends GlossaryUtils {
}
}
- if (DEBUG_ENABLED) {
- LOG.debug("<== GlossaryTermUtils.processTermDissociation()");
- }
+ LOG.debug("<== GlossaryTermUtils.processTermDissociation()");
}
public void clearImportCache() {
- glossaryNameGuidCache.get().clear();
- glossaryTermOrderCache.get().clear();
- glossaryTermQNameGuidCache.get().clear();
+ GLOSSARY_NAME_GUID_CACHE.get().clear();
+ GLOSSARY_TERM_ORDER_CACHE.get().clear();
+ GLOSSARY_TERM_Q_NAME_GUID_CACHE.get().clear();
+ }
+
+ public void updateGlossaryTermRelations(AtlasGlossaryTerm updatedGlossaryTerm) {
+ if (GLOSSARY_TERM_Q_NAME_GUID_CACHE.get().containsKey(updatedGlossaryTerm.getQualifiedName())) {
+ try {
+ AtlasGlossaryTerm glossaryTermFromDB = getGlossaryTem(GLOSSARY_TERM_Q_NAME_GUID_CACHE.get().get(updatedGlossaryTerm.getQualifiedName()));
+
+ copyRelations(updatedGlossaryTerm, glossaryTermFromDB);
+ } catch (AtlasBaseException e) {
+ LOG.debug("Error occurred while loading glossary Term", e);
+ }
+ }
+ }
+
+ protected List<AtlasGlossaryTerm> getGlossaryTermDataWithoutRelations(List<String[]> fileData, BulkImportResponse bulkImportResponse) throws AtlasBaseException {
+ List<AtlasGlossaryTerm> glossaryTerms = new ArrayList<>();
+ int rowCount = 1;
+
+ for (String[] record : fileData) {
+ List<String> failedTermMsgs = new ArrayList<>();
+ AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm();
+ String glossaryName = StringUtils.EMPTY;
+
+ if (ArrayUtils.isEmpty(record) || StringUtils.isBlank(record[INDEX_FOR_GLOSSARY_AT_RECORD])) {
+ LOG.error("The GlossaryName is blank for the record : {}", Arrays.toString(record));
+
+ failedTermMsgs.add("The GlossaryName is blank for the record : " + Arrays.toString(record));
+ } else {
+ glossaryName = record[INDEX_FOR_GLOSSARY_AT_RECORD];
+
+ String glossaryGuid = GLOSSARY_NAME_GUID_CACHE.get().get(glossaryName);
+
+ if (StringUtils.isEmpty(glossaryGuid)) {
+ glossaryGuid = getGlossaryGUIDFromGraphDB(glossaryName);
+
+ if (StringUtils.isEmpty(glossaryGuid)) {
+ glossaryGuid = createGlossary(glossaryName, failedTermMsgs);
+ }
+
+ GLOSSARY_NAME_GUID_CACHE.get().put(glossaryName, glossaryGuid);
+ }
+
+ if (StringUtils.isNotEmpty(glossaryGuid)) {
+ glossaryTerm = populateGlossaryTermObject(failedTermMsgs, record, glossaryGuid, false);
+
+ glossaryTerm.setQualifiedName(getGlossaryTermQualifiedName(glossaryTerm.getName(), glossaryName));
+
+ GLOSSARY_TERM_ORDER_CACHE.get().put(glossaryTerm.getQualifiedName(), rowCount);
+
+ glossaryTerms.add(glossaryTerm);
+ }
+ }
+
+ if (!failedTermMsgs.isEmpty()) {
+ String failedTermMsg = StringUtils.join(failedTermMsgs, System.lineSeparator());
+ String glossaryTermName = glossaryTerm.getName();
+
+ bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, failedTermMsg, rowCount));
+ }
+
+ rowCount++;
+ }
+
+ return glossaryTerms;
+ }
+
+ protected List<AtlasGlossaryTerm> getGlossaryTermDataWithRelations(List<String[]> fileData, BulkImportResponse bulkImportResponse) {
+ List<AtlasGlossaryTerm> glossaryTerms = new ArrayList<>();
+ int rowCount = 1;
+
+ for (String[] record : fileData) {
+ List<String> failedTermMsgs = new ArrayList<>();
+
+ if (ArrayUtils.isNotEmpty(record) && StringUtils.isNotBlank(record[INDEX_FOR_GLOSSARY_AT_RECORD])) {
+ AtlasGlossaryTerm glossaryTerm = new AtlasGlossaryTerm();
+ String glossaryName = record[INDEX_FOR_GLOSSARY_AT_RECORD];
+ String glossaryGuid = GLOSSARY_NAME_GUID_CACHE.get().get(glossaryName);
+
+ if (StringUtils.isNotEmpty(glossaryGuid)) {
+ glossaryTerm = populateGlossaryTermObject(failedTermMsgs, record, glossaryGuid, true);
+
+ glossaryTerm.setQualifiedName(getGlossaryTermQualifiedName(glossaryTerm.getName(), glossaryName));
+
+ glossaryTerms.add(glossaryTerm);
+ }
+
+ if (!failedTermMsgs.isEmpty()) {
+ String failedTermMsg = StringUtils.join(failedTermMsgs, System.lineSeparator());
+ String glossaryTermName = glossaryTerm.getName();
+
+ bulkImportResponse.addToFailedImportInfoList(new ImportInfo(glossaryName, glossaryTermName, FAILED, failedTermMsg, rowCount));
+ }
+ }
+
+ rowCount++;
+ }
+
+ return glossaryTerms;
+ }
+
+ protected Map<String, Object> getMapValue(String csvRecord, List<String> failedTermMsgs, boolean populateRelations) {
+ Map<String, Object> ret = null;
+
+ if (StringUtils.isNotBlank(csvRecord)) {
+ ret = new HashMap<>();
+
+ String[] csvRecordArray = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER);
+
+ for (String record : csvRecordArray) {
+ String[] recordArray = record.split(FileUtils.COLON_CHARACTER);
+
+ if ((recordArray.length % 2) == 0) {
+ ret.put(recordArray[0], recordArray[1]);
+ } else if (!populateRelations) {
+ failedTermMsgs.add("The Data in the uploaded file is incorrectly specified : " + csvRecord + System.lineSeparator() + "AdditionalAttributes needs to be a key:value pair");
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ protected List<String> getListValue(String csvRecord) {
+ List<String> ret = null;
+
+ if (StringUtils.isNotBlank(csvRecord)) {
+ ret = Arrays.asList(csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER));
+ }
+
+ return ret;
+ }
+
+ protected Set<AtlasRelatedTermHeader> getAtlasRelatedTermHeaderSet(String csvRecord, String termName, String glossaryName, List<String> failedTermMsgs) {
+ Set<AtlasRelatedTermHeader> ret = null;
+
+ if (StringUtils.isNotBlank(csvRecord)) {
+ ret = new HashSet<>();
+
+ String[] csvRecordArray = csvRecord.split(FileUtils.ESCAPE_CHARACTER + FileUtils.PIPE_CHARACTER);
+
+ AtlasRelatedTermHeader relatedTermHeader;
+
+ for (String data : csvRecordArray) {
+ String[] dataArray = data.split(FileUtils.ESCAPE_CHARACTER + FileUtils.COLON_CHARACTER);
+
+ if (dataArray.length == 2) {
+ String relatedTermQualifiedName = dataArray[1] + INVALID_NAME_CHARS[0] + dataArray[0];
+ String currTermQualifiedName = termName + INVALID_NAME_CHARS[0] + glossaryName;
+
+ if (relatedTermQualifiedName.equalsIgnoreCase(currTermQualifiedName)) {
+ failedTermMsgs.add("Invalid relationship specified for Term. Term cannot have a relationship with self");
+ } else {
+ AtlasVertex vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME, GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME + INVALID_NAME_CHARS[1] + QUALIFIED_NAME_ATTR, relatedTermQualifiedName);
+
+ if (vertex != null) {
+ String glossaryTermGuid = AtlasGraphUtilsV2.getIdFromVertex(vertex);
+
+ relatedTermHeader = new AtlasRelatedTermHeader();
+
+ relatedTermHeader.setTermGuid(glossaryTermGuid);
+
+ cacheRelatedTermQNameGuid(currTermQualifiedName, relatedTermQualifiedName, glossaryTermGuid);
+
+ ret.add(relatedTermHeader);
+ } else {
+ failedTermMsgs.add("The provided Reference " + dataArray[1] + "@" + dataArray[0] + " does not exist at Atlas referred at record with TermName : " + termName + " and GlossaryName : " + glossaryName);
+ }
+ }
+ } else {
+ failedTermMsgs.add("Incorrect relation data specified for the term : " + termName + "@" + glossaryName);
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ protected AtlasGlossaryTerm populateGlossaryTermObject(List<String> failedTermMsgList, String[] record, String glossaryGuid, boolean populateRelations) {
+ int length = record.length;
+ int i = INDEX_FOR_TERM_AT_RECORD;
+ AtlasGlossaryTerm ret = new AtlasGlossaryTerm();
+
+ if (length > i) {
+ ret.setName(record[i]);
+ }
+
+ if (StringUtils.isBlank(ret.getName())) {
+ if (!populateRelations) {
+ failedTermMsgList.add("The TermName is blank for provided record: " + Arrays.toString(record));
+ }
+ } else {
+ ret.setAnchor(new AtlasGlossaryHeader(glossaryGuid));
+
+ ret.setShortDescription((length > ++i) ? record[i] : null);
+
+ ret.setLongDescription((length > ++i) ? record[i] : null);
+
+ ret.setExamples((length > ++i) ? getListValue(record[i]) : null);
+
+ ret.setAbbreviation((length > ++i) ? record[i] : null);
+
+ ret.setUsage((length > ++i) ? record[i] : null);
+
+ ret.setAdditionalAttributes(((length > ++i) ? getMapValue(record[i], failedTermMsgList, populateRelations) : null));
+
+ if (populateRelations) {
+ ret.setTranslationTerms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setValidValuesFor((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setSynonyms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setReplacedBy((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setValidValues((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setReplacementTerms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setSeeAlso((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setTranslatedTerms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setIsA((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setAntonyms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setClassifies((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setPreferredToTerms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+
+ ret.setPreferredTerms((length > ++i) ? getAtlasRelatedTermHeaderSet(record[i], ret.getName(), record[INDEX_FOR_GLOSSARY_AT_RECORD], failedTermMsgList) : null);
+ }
+ }
+
+ return ret;
}
private boolean isRelationshipGuidSame(AtlasRelatedObjectId storeObject, AtlasRelatedObjectId relatedObjectId) {
@@ -164,7 +417,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
}
private void processTermAnchor(AtlasGlossaryTerm currentTerm, AtlasGlossaryTerm updatedTerm, RelationshipOperation op) throws AtlasBaseException {
- if (Objects.isNull(updatedTerm.getAnchor()) && op != RelationshipOperation.DELETE) {
+ if (isNull(updatedTerm.getAnchor()) && op != RelationshipOperation.DELETE) {
throw new AtlasBaseException(AtlasErrorCode.MISSING_MANDATORY_ANCHOR);
}
@@ -175,12 +428,10 @@ public class GlossaryTermUtils extends GlossaryUtils {
switch (op) {
case CREATE:
- if (Objects.isNull(updatedTermGlossaryGuid)) {
+ if (isNull(updatedTermGlossaryGuid)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_NEW_ANCHOR_GUID);
} else {
... 76058 lines suppressed ...