You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2020/07/31 00:19:00 UTC
[atlas] branch branch-2.0 updated (addb04e -> 2012745)
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from addb04e ATLAS-3902: Import Service: UpdateVertexGuid Now Makes Updates to AtlasEntityWithExtInfo
new 7086a4a ATLAS-3907: Java Patch Handler: Set index consistency.
new 2012745 ATLAS-3398: introduced consitency-lock configuration to avoid multiple entities with same unique attribute value
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../repository/graphdb/AtlasGraphManagement.java | 5 ++
.../graphdb/janus/AtlasJanusGraphManagement.java | 60 +++++++++++++++++-----
.../java/org/apache/atlas/AtlasConfiguration.java | 3 +-
.../repository/patches/AtlasPatchManager.java | 3 +-
.../repository/patches/IndexConsistencyPatch.java | 56 ++++++++++++++++++++
.../discovery/FreeTextSearchProcessorTest.java | 4 +-
.../resources/solr/core-template/solrconfig.xml | 2 +-
7 files changed, 115 insertions(+), 18 deletions(-)
create mode 100644 repository/src/main/java/org/apache/atlas/repository/patches/IndexConsistencyPatch.java
[atlas] 01/02: ATLAS-3907: Java Patch Handler: Set index
consistency.
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 7086a4ac3ebdda56c7f125edfd0014a3c2ad3f58
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Thu Jul 30 15:50:21 2020 -0700
ATLAS-3907: Java Patch Handler: Set index consistency.
(cherry picked from commit 05eba3fdb9fedf3483cdb1cef938f7a2cbf7933e)
---
.../repository/graphdb/AtlasGraphManagement.java | 5 ++
.../graphdb/janus/AtlasJanusGraphManagement.java | 35 ++++++++++++++
.../repository/patches/AtlasPatchManager.java | 3 +-
.../repository/patches/IndexConsistencyPatch.java | 56 ++++++++++++++++++++++
4 files changed, 98 insertions(+), 1 deletion(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
index fca7890..f7d2e27 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraphManagement.java
@@ -167,4 +167,9 @@ public interface AtlasGraphManagement {
* @return the encoded name for the index
*/
String getIndexFieldName(String indexName, AtlasPropertyKey propertyKey, boolean isStringField);
+
+ /**
+ * Set consistency to ConsistencyModifier.LOCK for all vertex and edge indexes.
+ */
+ void updateUniqueIndexesForConsistencyLock();
}
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index 6ef9cb7..d0cda71 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -20,6 +20,7 @@ package org.apache.atlas.repository.graphdb.janus;
import com.google.common.base.Preconditions;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Element;
import org.janusgraph.core.Cardinality;
import org.janusgraph.core.EdgeLabel;
import org.janusgraph.core.PropertyKey;
@@ -274,4 +275,38 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
indexBuilder.buildCompositeIndex();
}
+
+ @Override
+ public void updateUniqueIndexesForConsistencyLock() {
+ try {
+ setConsistency(this.management, Vertex.class);
+ setConsistency(this.management, Edge.class);
+ } finally {
+ commit();
+ }
+ }
+
+ private static void setConsistency(JanusGraphManagement mgmt, Class<? extends Element> elementType) {
+ LOG.info("setConsistency: {}: Starting...", elementType.getSimpleName());
+ int count = 0;
+
+ try {
+ Iterable<JanusGraphIndex> iterable = mgmt.getGraphIndexes(elementType);
+ for (JanusGraphIndex index : iterable) {
+ if (!index.isCompositeIndex() || !index.isUnique() || mgmt.getConsistency(index) == ConsistencyModifier.LOCK) {
+ continue;
+ }
+
+ for (PropertyKey propertyKey : index.getFieldKeys()) {
+ LOG.info("setConsistency: {}: {}", count, propertyKey.name());
+ }
+
+ mgmt.setConsistency(index, ConsistencyModifier.LOCK);
+ count++;
+ }
+ }
+ finally {
+ LOG.info("setConsistency: {}: {}: Done!", elementType.getSimpleName(), count);
+ }
+ }
}
\ No newline at end of file
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
index 093edf9..b142a2a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/AtlasPatchManager.java
@@ -53,7 +53,8 @@ public class AtlasPatchManager {
new UniqueAttributePatch(context),
new ClassificationTextPatch(context),
new FreeTextRequestHandlerPatch(context),
- new SuggestionsRequestHandlerPatch(context)
+ new SuggestionsRequestHandlerPatch(context),
+ new IndexConsistencyPatch(context)
};
try {
diff --git a/repository/src/main/java/org/apache/atlas/repository/patches/IndexConsistencyPatch.java b/repository/src/main/java/org/apache/atlas/repository/patches/IndexConsistencyPatch.java
new file mode 100644
index 0000000..2abe73c
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/patches/IndexConsistencyPatch.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.patches;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.atlas.model.patches.AtlasPatch.PatchStatus.APPLIED;
+
+public class IndexConsistencyPatch extends AtlasPatchHandler {
+ private static final Logger LOG = LoggerFactory.getLogger(IndexConsistencyPatch.class);
+
+ private static final String PATCH_ID = "JAVA_PATCH_0000_005";
+ private static final String PATCH_DESCRIPTION = "Sets index consistency for vertices and edges.";
+
+ private final PatchContext context;
+
+ public IndexConsistencyPatch(PatchContext context) {
+ super(context.getPatchRegistry(), PATCH_ID, PATCH_DESCRIPTION);
+ this.context = context;
+ }
+
+ @Override
+ public void apply() throws AtlasBaseException {
+ AtlasGraph graph = context.getGraph();
+
+ try {
+ LOG.info("IndexConsistencyPatch: Starting...");
+ graph.getManagementSystem().updateUniqueIndexesForConsistencyLock();
+ } finally {
+ LOG.info("IndexConsistencyPatch: Done!");
+ }
+
+ setStatus(APPLIED);
+
+ LOG.info("IndexConsistencyPatch.apply(): patchId={}, status={}", getPatchId(), getStatus());
+ }
+}
[atlas] 02/02: ATLAS-3398: introduced consitency-lock configuration
to avoid multiple entities with same unique attribute value
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 20127459a59fd0a64966046df3a1b0f64363c4f7
Author: Damian Warszawski <da...@ing.com>
AuthorDate: Thu Jul 30 17:11:34 2020 -0700
ATLAS-3398: introduced consitency-lock configuration to avoid multiple entities with same unique attribute value
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
(cherry picked from commit 80efe92630a9198ed4b4c564f6e78c8079045930)
---
.../graphdb/janus/AtlasJanusGraphManagement.java | 29 +++++++++++-----------
.../java/org/apache/atlas/AtlasConfiguration.java | 3 ++-
.../discovery/FreeTextSearchProcessorTest.java | 4 +--
.../resources/solr/core-template/solrconfig.xml | 2 +-
4 files changed, 19 insertions(+), 19 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
index d0cda71..2a2ef92 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphManagement.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.graphdb.janus;
import com.google.common.base.Preconditions;
+import org.apache.atlas.AtlasConfiguration;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Element;
@@ -46,6 +47,7 @@ import java.util.Set;
* Janus implementation of AtlasGraphManagement.
*/
public class AtlasJanusGraphManagement implements AtlasGraphManagement {
+ private static final boolean lockEnabled = AtlasConfiguration.STORAGE_CONSISTENCY_LOCK_ENABLED.getBoolean();
private static final Parameter[] STRING_PARAMETER_ARRAY = new Parameter[]{Mapping.STRING.asParameter()};
private static final Logger LOG = LoggerFactory.getLogger(AtlasJanusGraphManagement.class);
@@ -246,23 +248,16 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
@Override
public void createVertexCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
- IndexBuilder indexBuilder = management.buildIndex(propertyName, Vertex.class);
-
- for (AtlasPropertyKey key : propertyKeys) {
- PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
- indexBuilder.addKey(janusKey);
- }
-
- if (isUnique) {
- indexBuilder.unique();
- }
-
- indexBuilder.buildCompositeIndex();
+ createCompositeIndex(propertyName, isUnique, propertyKeys, Vertex.class);
}
@Override
public void createEdgeCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys) {
- IndexBuilder indexBuilder = management.buildIndex(propertyName, Edge.class);
+ createCompositeIndex(propertyName, isUnique, propertyKeys, Edge.class);
+ }
+
+ private void createCompositeIndex(String propertyName, boolean isUnique, List<AtlasPropertyKey> propertyKeys, Class<? extends Element> elementType) {
+ IndexBuilder indexBuilder = management.buildIndex(propertyName, elementType);
for (AtlasPropertyKey key : propertyKeys) {
PropertyKey janusKey = AtlasJanusObjectFactory.createPropertyKey(key);
@@ -273,7 +268,11 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
indexBuilder.unique();
}
- indexBuilder.buildCompositeIndex();
+ JanusGraphIndex index = indexBuilder.buildCompositeIndex();
+
+ if (lockEnabled && isUnique) {
+ management.setConsistency(index, ConsistencyModifier.LOCK);
+ }
}
@Override
@@ -309,4 +308,4 @@ public class AtlasJanusGraphManagement implements AtlasGraphManagement {
LOG.info("setConsistency: {}: {}: Done!", elementType.getSimpleName(), count);
}
}
-}
\ No newline at end of file
+}
diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
index 2c007ca..a942b9f 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java
@@ -71,7 +71,8 @@ public enum AtlasConfiguration {
MIGRATION_IMPORT_START_POSITION("atlas.migration.import.start.position", 0),
LINEAGE_USING_GREMLIN("atlas.lineage.query.use.gremlin", false),
- HTTP_HEADER_SERVER_VALUE("atlas.http.header.server.value","Apache Atlas");
+ HTTP_HEADER_SERVER_VALUE("atlas.http.header.server.value","Apache Atlas"),
+ STORAGE_CONSISTENCY_LOCK_ENABLED("atlas.graph.storage.consistency-lock.enabled", true);
private static final Configuration APPLICATION_PROPERTIES;
diff --git a/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java
index 464b281..2a38d87 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/FreeTextSearchProcessorTest.java
@@ -57,7 +57,7 @@ public class FreeTextSearchProcessorTest extends BasicTestSetup {
}
@Test
- public void searchTablesByName() throws AtlasBaseException, InterruptedException {
+ public void searchTablesByName() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName("hive_table");
params.setQuery("sales");
@@ -73,7 +73,7 @@ public class FreeTextSearchProcessorTest extends BasicTestSetup {
}
@Test
- public void searchByNameSortBy() throws AtlasBaseException, InterruptedException {
+ public void searchByNameSortBy() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName("hive_table");
params.setQuery("sales");
diff --git a/test-tools/src/main/resources/solr/core-template/solrconfig.xml b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
index 39cc6ab..8ebbeff 100644
--- a/test-tools/src/main/resources/solr/core-template/solrconfig.xml
+++ b/test-tools/src/main/resources/solr/core-template/solrconfig.xml
@@ -434,7 +434,7 @@
-->
<lst name="defaults">
<str name="defType">edismax</str>
- <str name="qf">3eh1_t^3 yrp_t^10 m4l_t^10 kjp_t^10 iyt_t^10 cn9_t^1 x6t_t^3 hdx_t^3 3ksl_s^3 3qbp_s^3 3mdh_s^3 3oqt_s^3 3nyd_t^3 7uv9_t^3 7ldx_t^3</str>
+ <str name="qf">3hmt_t 35x_t f0l_t i6d_l 7f2d_t 7gn9_t 3oqt_s jr9_t 3rwl_t lc5_t mx1_t 7dhh_t iyt_l 3j7p_t 7klh_t 7hfp_t 7i85_t ohx_t 7bwl_l 7cp1_l</str>
<str name="hl.fl">*</str>
<bool name="hl.requireFieldMatch">true</bool>
<bool name="lowercaseOperators">true</bool>