You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/04/05 00:56:14 UTC
[4/5] atlas git commit: ATLAS-2460: implementation of
migration-import to migrate Atlas data in Titan to JanusGraph
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/pc/WorkItemManager.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/pc/WorkItemManager.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/pc/WorkItemManager.java
new file mode 100644
index 0000000..7e21495
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/pc/WorkItemManager.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration.pc;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+public class WorkItemManager<T, U extends WorkItemConsumer> {
+ private static final Logger LOG = LoggerFactory.getLogger(WorkItemManager.class);
+
+ private final BlockingQueue<T> workQueue;
+ private final ExecutorService service;
+ private final List<U> consumers = new ArrayList<>();
+
+ public WorkItemManager(WorkItemBuilder builder, int batchSize, int numWorkers) {
+ workQueue = new LinkedBlockingQueue<>(batchSize * numWorkers);
+ service = Executors.newFixedThreadPool(numWorkers);
+
+ for (int i = 0; i < numWorkers; i++) {
+ U c = (U) builder.build(workQueue);
+
+ service.submit(c);
+ consumers.add(c);
+ }
+ }
+
+ public void produce(T item) {
+ try {
+ workQueue.put(item);
+ } catch (InterruptedException ex) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ public void shutdown() throws InterruptedException {
+ int avgCommitTimeSeconds = getAvgCommitTimeSeconds() * 2;
+
+ LOG.info("WorkItemManager: Shutdown started. Will wait for: {} seconds...", avgCommitTimeSeconds);
+
+ service.shutdown();
+ service.awaitTermination(avgCommitTimeSeconds, TimeUnit.MINUTES);
+
+ LOG.info("WorkItemManager: Shutdown done!");
+ }
+
+ private int getAvgCommitTimeSeconds() {
+ int commitTimeSeconds = 0;
+
+ for (U c : consumers) {
+ commitTimeSeconds += c.getMaxCommitTimeSeconds();
+ }
+
+ return commitTimeSeconds / consumers.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
----------------------------------------------------------------------
diff --git a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
index 4409009..82a800e 100644
--- a/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
+++ b/graphdb/titan0/src/main/java/org/apache/atlas/repository/graphdb/titan0/Titan0Graph.java
@@ -55,6 +55,7 @@ import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.IOException;
+import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collection;
@@ -414,7 +415,12 @@ public class Titan0Graph implements AtlasGraph<Titan0Vertex, Titan0Edge> {
return multiProperties.contains(propertyName);
}
+ @Override
+ public void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
+ }
+
public void addMultiProperties(Set<String> names) {
multiProperties.addAll(names);
}
+
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
index 1ea961d..b12d43e 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportResult.java
@@ -50,7 +50,7 @@ public class AtlasExportResult implements Serializable {
public final static String ENTITY_COUNT = "entityCount";
public enum OperationStatus {
- SUCCESS, PARTIAL_SUCCESS, FAIL
+ SUCCESS, PARTIAL_SUCCESS, INPROGRESS, FAIL
}
private AtlasExportRequest request;
@@ -143,6 +143,9 @@ public class AtlasExportResult implements Serializable {
this.operationStatus = operationStatus;
}
+ public void setMetric(String key, int value) {
+ metrics.put(key, value);
+ }
public void incrementMeticsCounter(String key) {
incrementMeticsCounter(key, 1);
@@ -222,7 +225,6 @@ public class AtlasExportResult implements Serializable {
public void setEntityCreationOrder(List<String> entityCreationOrder) { this.entityCreationOrder = entityCreationOrder; }
-
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
sb = new StringBuilder();
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/intg/src/main/java/org/apache/atlas/model/impexp/MigrationStatus.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/MigrationStatus.java b/intg/src/main/java/org/apache/atlas/model/impexp/MigrationStatus.java
new file mode 100644
index 0000000..9014945
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/MigrationStatus.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.model.impexp;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+
+import java.io.Serializable;
+import java.util.Date;
+
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
+import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
+public class MigrationStatus implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private String operationStatus;
+ private Date startTime;
+ private Date endTime;
+ private long currentIndex;
+ private long totalCount;
+
+ public void setOperationStatus(String operationStatus) {
+ this.operationStatus = operationStatus;
+ }
+
+ public String getOperationStatus() {
+ return operationStatus;
+ }
+
+ public void setStartTime(Date startTime) {
+ this.startTime = startTime;
+ }
+
+ public Date getStartTime() {
+ return startTime;
+ }
+
+ public void setEndTime(Date endTime) {
+ this.endTime = endTime;
+ }
+
+ public Date getEndTime() {
+ return endTime;
+ }
+
+ public void setCurrentIndex(long currentIndex) {
+ this.currentIndex = currentIndex;
+ }
+
+ public long getCurrentIndex() {
+ return currentIndex;
+ }
+
+ public void setTotalCount(long totalCount) {
+ this.totalCount = totalCount;
+ }
+
+ public long getTotalCount() {
+ return this.totalCount;
+ }
+
+ public StringBuilder toString(StringBuilder sb) {
+ sb.append(", operationStatus=").append(operationStatus);
+ sb.append(", startTime=").append(startTime);
+ sb.append(", endTime=").append(endTime);
+ sb.append(", currentIndex=").append(currentIndex);
+ sb.append(", totalCount=").append(totalCount);
+
+ return sb;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
index c63dc24..b05754f 100644
--- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
+++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
@@ -28,7 +28,9 @@ import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import java.io.InputStream;
import java.util.List;
+import java.util.Map;
/**
* Interface to persistence store of TypeDef
@@ -105,4 +107,6 @@ public interface AtlasTypeDefStore {
AtlasBaseTypeDef getByName(String name) throws AtlasBaseException;
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
+
+ void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bfbb953..aa5cadd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1757,7 +1757,10 @@
<version>2.7</version>
<configuration>
<encoding>UTF-8</encoding>
- <nonFilteredFileExtensions>zip</nonFilteredFileExtensions>
+ <nonFilteredFileExtensions>
+ <nonFilteredFileExtension>zip</nonFilteredFileExtension>
+ <nonFilteredFileExtension>json</nonFilteredFileExtension>
+ </nonFilteredFileExtensions>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/pom.xml
----------------------------------------------------------------------
diff --git a/repository/pom.xml b/repository/pom.xml
index b1d6b1f..d348b76 100755
--- a/repository/pom.xml
+++ b/repository/pom.xml
@@ -18,7 +18,8 @@
~ limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
@@ -68,7 +69,7 @@
<artifactId>commons-lang3</artifactId>
</dependency>
- <dependency>
+ <dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>
</dependency>
@@ -112,7 +113,7 @@
<type>pom</type>
<scope>test</scope>
</dependency>
-
+
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-authorization</artifactId>
@@ -256,7 +257,7 @@
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
- <id>copy-resources</id>
+ <id>copy-resources-solr</id>
<phase>validate</phase>
<goals>
<goal>copy-resources</goal>
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 5672d9d..f517445 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -67,27 +67,7 @@ import java.util.List;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
-import static org.apache.atlas.repository.Constants.BACKING_INDEX;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.CREATED_BY_KEY;
-import static org.apache.atlas.repository.Constants.EDGE_INDEX;
-import static org.apache.atlas.repository.Constants.ENTITY_TEXT_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.FULLTEXT_INDEX;
-import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.MODIFIED_BY_KEY;
-import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.VERTEX_INDEX;
-import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE;
@@ -101,6 +81,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
+ private static final String VERTEX_ID_IN_IMPORT_KEY = "__vIdInImport";
+ private static final String EDGE_ID_IN_IMPORT_KEY = "__eIdInImport";
private static final List<Class> INDEX_EXCLUSION_CLASSES = new ArrayList() {
{
add(Boolean.class);
@@ -284,6 +266,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, String.class, false, LIST, true, true);
createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true);
createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true);
+ createVertexIndex(management, CLASSIFICATION_ENTITY_GUID, String.class, false, SINGLE, true, true);
+
+ createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, Long.class, false, SINGLE, true, false);
// create vertex-centric index
createVertexCentricIndex(management, CLASSIFICATION_LABEL, AtlasEdgeDirection.BOTH, CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, String.class, SINGLE);
@@ -292,6 +277,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
// create edge indexes
createEdgeIndex(management, RELATIONSHIP_GUID_PROPERTY_KEY, String.class, SINGLE, true);
+ createEdgeIndex(management, EDGE_ID_IN_IMPORT_KEY, String.class, SINGLE, true);
// create fulltext indexes
createFullTextIndex(management, ENTITY_TEXT_PROPERTY_KEY, String.class, SINGLE);
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java
new file mode 100644
index 0000000..1daf371
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/MigrationProgressService.java
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import org.apache.atlas.annotation.AtlasService;
+import org.apache.atlas.model.impexp.MigrationStatus;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+@AtlasService
+@Singleton
+public class MigrationProgressService {
+ private static final Logger LOG = LoggerFactory.getLogger(MigrationProgressService.class);
+
+ private static final String MIGRATION_STATUS_TYPE_NAME = "__MigrationStatus";
+ private static final String CURRENT_INDEX_PROPERTY = "currentIndex";
+ private static final String OPERATION_STATUS_PROPERTY = "operationStatus";
+ private static final String START_TIME_PROPERTY = "startTime";
+ private static final String END_TIME_PROPERTY = "endTime";
+ private static final String TOTAL_COUNT_PROPERTY = "totalCount";
+ private static final String MIGRATION_STATUS_KEY = "1";
+
+ private final AtlasGraph graph;
+ private final MigrationStatus defaultStatus = new MigrationStatus();
+ private LoadingCache<String, MigrationStatus> cache;
+
+ @Inject
+ public MigrationProgressService(AtlasGraph graph) {
+ this.graph = graph;
+ }
+
+ public MigrationStatus getStatus() {
+ try {
+ if (cache == null) {
+ initCache();
+ cache.get(MIGRATION_STATUS_KEY);
+ }
+
+ if(cache.size() > 0) {
+ return cache.get(MIGRATION_STATUS_KEY);
+ }
+
+ return defaultStatus;
+ } catch (ExecutionException e) {
+ return defaultStatus;
+ }
+ }
+
+ private void initCache() {
+ this.cache = CacheBuilder.newBuilder().refreshAfterWrite(30, TimeUnit.SECONDS).
+ build(new CacheLoader<String, MigrationStatus>() {
+ @Override
+ public MigrationStatus load(String key) {
+ try {
+ return from(fetchStatusVertex());
+ } catch (Exception e) {
+ LOG.error("Error retrieving status.", e);
+ return defaultStatus;
+ }
+ }
+
+ private MigrationStatus from(AtlasVertex vertex) {
+ if (vertex == null) {
+ return null;
+ }
+
+ MigrationStatus ms = new MigrationStatus();
+
+ ms.setStartTime(GraphHelper.getSingleValuedProperty(vertex, START_TIME_PROPERTY, Date.class));
+ ms.setEndTime(GraphHelper.getSingleValuedProperty(vertex, END_TIME_PROPERTY, Date.class));
+ ms.setCurrentIndex(GraphHelper.getSingleValuedProperty(vertex, CURRENT_INDEX_PROPERTY, Long.class));
+ ms.setOperationStatus(GraphHelper.getSingleValuedProperty(vertex, OPERATION_STATUS_PROPERTY, String.class));
+ ms.setTotalCount(GraphHelper.getSingleValuedProperty(vertex, TOTAL_COUNT_PROPERTY, Long.class));
+
+ return ms;
+ }
+
+ private AtlasVertex fetchStatusVertex() {
+ Iterator<AtlasVertex> itr = graph.query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, MIGRATION_STATUS_TYPE_NAME).vertices().iterator();
+ return itr.hasNext() ? itr.next() : null;
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
new file mode 100644
index 0000000..2fad333
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.impexp.ImportTypeDefProcessor;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.configuration.Configuration;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.service.Service;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.file.Paths;
+
+import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME;
+
+@Component
+public class DataMigrationService implements Service {
+ private static final Logger LOG = LoggerFactory.getLogger(DataMigrationService.class);
+
+ private static String ATLAS_MIGRATION_DATA_NAME = "atlas-migration-data.json";
+ private static String ATLAS_MIGRATION_TYPESDEF_NAME = "atlas-migration-typesdef.json";
+
+ private final Configuration configuration;
+ private final Thread thread;
+
+ @Inject
+ public DataMigrationService(AtlasTypeDefStore typeDefStore, Configuration configuration,
+ GraphBackedSearchIndexer indexer, AtlasTypeDefStoreInitializer storeInitializer,
+ AtlasTypeRegistry typeRegistry) {
+ this.configuration = configuration;
+ this.thread = new Thread(new FileImporter(typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
+ }
+
+ @Override
+ public void start() {
+ Runtime.getRuntime().addShutdownHook(thread);
+ thread.start();
+ }
+
+ @Override
+ public void stop() {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ LOG.error("Data Migration: Interrupted", e);
+ }
+ }
+
+ public String getFileName() {
+ return configuration.getString(ATLAS_MIGRATION_MODE_FILENAME, "");
+ }
+
+ public static class FileImporter implements Runnable {
+ private final AtlasTypeDefStore typeDefStore;
+ private final String importDirectory;
+ private final GraphBackedSearchIndexer indexer;
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasTypeDefStoreInitializer storeInitializer;
+
+ public FileImporter(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
+ AtlasTypeDefStoreInitializer storeInitializer,
+ String directoryName, GraphBackedSearchIndexer indexer) {
+ this.typeDefStore = typeDefStore;
+ this.typeRegistry = typeRegistry;
+ this.storeInitializer = storeInitializer;
+ this.importDirectory = directoryName;
+ this.indexer = indexer;
+ }
+
+ public void performImport() throws AtlasBaseException {
+ try {
+ performInit();
+
+ FileInputStream fs = new FileInputStream(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_DATA_NAME));
+
+ typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs);
+ } catch (Exception ex) {
+ LOG.error("Import failed!", ex);
+ throw new AtlasBaseException(ex);
+ }
+ }
+
+ private void performInit() throws AtlasBaseException, AtlasException {
+ storeInitializer.init();
+ processIncomingTypesDef(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_TYPESDEF_NAME));
+ indexer.instanceIsActive();
+ }
+
+ @VisibleForTesting
+ void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
+ try {
+ String jsonStr = FileUtils.readFileToString(typesDefFile);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+ ImportTypeDefProcessor processor = new ImportTypeDefProcessor(typeDefStore, typeRegistry);
+ processor.processTypes(typesDef, new AtlasImportResult());
+ } catch (IOException e) {
+ LOG.error("processIncomingTypesDef: Could not process file: {}! Imported data may not be usable.", typesDefFile.getName());
+ }
+ }
+
+ private File getFileFromImportDirectory(String importDirectory, String fileName) {
+ return Paths.get(importDirectory, fileName).toFile();
+ }
+
+ @Override
+ public void run() {
+ try {
+ performImport();
+ } catch (AtlasBaseException e) {
+ LOG.error("Data Migration:", e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java b/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
new file mode 100644
index 0000000..2550daf
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public class RelationshipCacheGenerator {
+
+ public static Map<String, String> get(AtlasTypeRegistry typeRegistry) {
+ Map<String, String> ret = new HashMap<>();
+ Collection<AtlasRelationshipType> relationshipTypes = typeRegistry.getAllRelationshipTypes();
+
+ for (AtlasRelationshipType rt : relationshipTypes) {
+ AtlasRelationshipDef rd = rt.getRelationshipDef();
+ String relTypeName = rt.getTypeName();
+
+ add(ret, getKey(rd.getEndDef1(), rt.getEnd1Type()), relTypeName);
+ add(ret, getKey(rd.getEndDef2(), rt.getEnd2Type()), relTypeName);
+ }
+
+ return ret;
+ }
+
+ private static String getKey(AtlasRelationshipEndDef ed, AtlasEntityType rt) {
+ return getKey(ed.getIsLegacyAttribute(), rt.getTypeName(), ed.getName());
+ }
+
+ private static String getKey(String lhs, String rhs) {
+ return String.format("%s%s.%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, lhs, rhs);
+ }
+
+ private static String getKey(boolean isLegacy, String typeName, String name) {
+ if(!isLegacy) {
+ return "";
+ }
+
+ return getKey(typeName, name);
+ }
+
+ private static void add(Map<String, String> map, String key, String value) {
+ if(StringUtils.isEmpty(key) || map.containsKey(key)) {
+ return;
+ }
+
+ map.put(key, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 6676200..4109abf 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -63,6 +63,7 @@ import java.util.Map;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME;
/**
@@ -88,10 +89,10 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
@PostConstruct
public void init() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefStoreInitializer.init()");
+ boolean isMigrationEnabled = !StringUtils.isEmpty(conf.getString(ATLAS_MIGRATION_MODE_FILENAME));
- if (!HAConfiguration.isHAEnabled(conf)) {
+ if (!HAConfiguration.isHAEnabled(conf) || isMigrationEnabled) {
atlasTypeDefStore.init();
-
loadBootstrapTypeDefs();
} else {
LOG.info("AtlasTypeDefStoreInitializer.init(): deferring type loading until instance activation");
@@ -151,44 +152,50 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
if (typeDefFiles == null || typeDefFiles.length == 0) {
- LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName );
+ LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName);
} else {
-
// sort the files by filename
Arrays.sort(typeDefFiles);
-
for (File typeDefFile : typeDefFiles) {
- if (typeDefFile.isFile()) {
- try {
- String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
- AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+ try {
+ readTypesFromFile(typeDefFile);
+ } catch (Throwable t) {
+ LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
+ }
+ }
+ }
- if (typesDef == null || typesDef.isEmpty()) {
- LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
+ LOG.info("<== AtlasTypeDefStoreInitializer({})", typesDir);
+ }
- continue;
- }
+ public void readTypesFromFile(File typeDefFile) {
+ if (!typeDefFile.isFile()) {
+ return;
+ }
- AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, atlasTypeRegistry);
- AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry, true);
+ try {
+ String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
- if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) {
- atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
+ if (typesDef == null || typesDef.isEmpty()) {
+ LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
+ return;
+ }
- LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath());
- } else {
- LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
- }
+ AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, atlasTypeRegistry);
+ AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry, true);
- } catch (Throwable t) {
- LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
- }
- }
+ if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) {
+ atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
+
+ LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath());
+ } else {
+ LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
}
- applyTypePatches(typesDir.getPath());
+ } catch (Throwable t) {
+ LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
}
- LOG.info("<== AtlasTypeDefStoreInitializer({})", typesDir);
}
public static AtlasTypesDef getTypesToCreate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
index 5bec16e..316d78c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1.java
@@ -43,6 +43,7 @@ import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,6 +52,7 @@ import org.springframework.stereotype.Component;
import javax.inject.Inject;
import java.util.*;
+import static org.apache.atlas.AtlasConstants.ATLAS_MIGRATION_MODE_FILENAME;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.DELETE;
import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.UPDATE;
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
index 1a04418..9e22869 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
@@ -24,6 +24,7 @@ import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.VERTEX_TYPE;
+import java.io.InputStream;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
@@ -37,10 +38,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.model.typedef.*;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.*;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -112,6 +110,11 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
}
+ @Override
+ public void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
+ getAtlasGraph().loadLegacyGraphSON(relationshipCache, fs);
+ }
+
AtlasGraph getAtlasGraph() { return atlasGraph; }
@VisibleForTesting
@@ -275,9 +278,20 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
String updatedBy = vertex.getProperty(Constants.MODIFIED_BY_KEY, String.class);
Long createTime = vertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
Long updateTime = vertex.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
- Long version = vertex.getProperty(Constants.VERSION_PROPERTY_KEY, Long.class);
+ Object versionObj = vertex.getProperty(Constants.VERSION_PROPERTY_KEY, Object.class);
String options = vertex.getProperty(Constants.TYPEOPTIONS_PROPERTY_KEY, String.class);
+ Long version = null;
+
+ if(versionObj instanceof Number) {
+ version = ((Number)versionObj).longValue();
+ } else if (versionObj != null) {
+ version = Long.valueOf(versionObj.toString());
+ } else {
+ version = Long.valueOf(0);
+ }
+
+
typeDef.setName(name);
typeDef.setDescription(description);
typeDef.setTypeVersion(typeVersion);
@@ -313,7 +327,15 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
boolean ret = false;
if (isTypeVertex(vertex)) {
- TypeCategory vertexCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, TypeCategory.class);
+ Object objTypeCategory = vertex.getProperty(Constants.TYPE_CATEGORY_PROPERTY_KEY, Object.class);
+
+ TypeCategory vertexCategory = null;
+
+ if(objTypeCategory instanceof TypeCategory) {
+ vertexCategory = (TypeCategory) objTypeCategory;
+ } else if (objTypeCategory != null) {
+ vertexCategory = TypeCategory.valueOf(objTypeCategory.toString());
+ }
ret = category.equals(vertexCategory);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java
new file mode 100644
index 0000000..bd1fb4c
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionIT.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.runner.LocalSolrRunner;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.testng.Assert.assertEquals;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class HiveParititionIT extends MigrationBaseAsserts {
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStoreInitializer storeInitializer;
+
+ @Inject
+ private GraphBackedSearchIndexer indexer;
+
+ @Inject
+ public HiveParititionIT(AtlasGraph graph) {
+ super(graph);
+ }
+
+ @AfterClass
+ public void clear() throws Exception {
+ AtlasGraphProvider.cleanup();
+
+ if (useLocalSolr()) {
+ LocalSolrRunner.stop();
+ }
+ }
+
+ @Test
+ public void fileImporterTest() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
+
+ String directoryName = TestResourceFileUtils.getDirectory("parts_db");
+ DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry,
+ storeInitializer, directoryName, indexer);
+
+ fi.run();
+
+
+ assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1);
+ assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1);
+ assertHiveVertices(1, 2, 7);
+
+ assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95");
+ assertTypeCountNameGuid("hive_process", 1, "", "");
+ assertEdges("hive_db", "parts_db", AtlasEdgeDirection.IN,1, 1, "");
+ assertEdges("hive_table", "t1", AtlasEdgeDirection.OUT, 1, 1, "hive_db_tables");
+ assertEdges("hive_table", "tv1", AtlasEdgeDirection.OUT, 1, 1, "hive_db_tables");
+
+ assertMigrationStatus(136);
+ }
+
+ private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) {
+ List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys");
+ assertEquals(keys.size(), expectedCount);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java
new file mode 100644
index 0000000..7fc8f2a
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksIT.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.runner.LocalSolrRunner;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.testng.ITestContext;
+import org.testng.annotations.*;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class HiveStocksIT extends MigrationBaseAsserts {
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ public HiveStocksIT(AtlasGraph graph) {
+ super(graph);
+ }
+
+ @BeforeTest
+ public void setupTest() {
+ RequestContextV1.clear();
+ RequestContextV1.get().setUser(TestUtilsV2.TEST_USER, null);
+ }
+
+ @AfterClass
+ public void clear() throws Exception {
+ AtlasGraphProvider.cleanup();
+
+ if (useLocalSolr()) {
+ LocalSolrRunner.stop();
+ }
+ }
+
+ @DataProvider(name = "stocks-2-branch08-tag")
+ public static Object[][] getStocksTag(ITestContext context) throws IOException {
+ return new Object[][]{{ TestResourceFileUtils.getFileInputStream("stocks-2-0.8-extended-tag.json") }};
+ }
+
+ @Test(dataProvider = "stocks-2-branch08-tag")
+ public void migrateFromEarlierVersionWithTag(FileInputStream fs) throws AtlasBaseException, IOException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
+
+ typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs);
+
+ assertHiveVertices(1, 1, 7);
+ assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0");
+ assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774");
+ assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns");
+ assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891");
+ assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec");
+ assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160");
+ assertTypeCountNameGuid("hive_column", 1, "low", "38caeaf7-49e6-4d6d-8727-231406a46821");
+ assertTypeCountNameGuid("hive_column", 1, "close", "3bae9b76-f812-4745-b4d2-2a72d2773d07");
+ assertTypeCountNameGuid("hive_column", 1, "volume", "bee376a4-3d8d-4943-b7e8-9bce042c2657");
+ assertTypeCountNameGuid("hive_column", 1, "adj_close", "fcba2002-cb38-4c2e-b853-68d421d66703");
+ assertTypeCountNameGuid("hive_process", 0, "", "");
+ assertTypeCountNameGuid("hive_storagedesc", 1, "", "294290d8-4498-4677-973c-c266d594b039");
+ assertTypeCountNameGuid("Tag1", 1, "", "");
+
+ assertEdges(getVertex("hive_db", "stocks").getEdges(AtlasEdgeDirection.IN).iterator(),1, 1, "");
+ assertEdges(getVertex("hive_table", "stocks_daily").getEdges(AtlasEdgeDirection.OUT).iterator(), 1, 1, "hive_db_tables");
+ assertEdges(getVertex("hive_column", "high").getEdges(AtlasEdgeDirection.OUT).iterator(), 1,1, "hive_table_columns");
+
+ assertMigrationStatus(164);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
new file mode 100644
index 0000000..ed2793f
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.*;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Iterator;
+import java.util.List;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.AssertJUnit.assertTrue;
+
+public class MigrationBaseAsserts {
+ protected static final String ASSERT_NAME_PROPERTY = "Asset.name";
+ private final String TYPE_NAME_PROPERTY = "__typeName";
+ private final String R_GUID_PROPERTY_NAME = "_r__guid";
+
+ protected AtlasGraph graph;
+
+ protected MigrationBaseAsserts(AtlasGraph graph) {
+ this.graph = graph;
+ }
+
+ protected void assertHiveVertices(int dbCount, int tableCount, int columnCount) {
+ int i = 0;
+
+ Iterator<AtlasVertex> results = getVertices("hive_db", null);
+ for (Iterator<AtlasVertex> it = results; it.hasNext(); i++) {
+ assertNotNull(it.next());
+ }
+ assertEquals(i, dbCount);
+
+ i = 0;
+ results = getVertices("hive_table", null);
+ for (Iterator<AtlasVertex> it = results; it.hasNext(); i++) {
+ assertNotNull(it.next());
+ }
+ assertEquals(i, tableCount);
+
+ i = 0;
+ results = getVertices("hive_column", null);
+ for (Iterator<AtlasVertex> it = results; it.hasNext(); i++) {
+ assertNotNull(it.next());
+ }
+
+ assertTrue(i > 0);
+ assertEquals(i, columnCount);
+ }
+
+ protected Iterator<AtlasVertex> getVertices(String typeName, String name) {
+ AtlasGraphQuery query = graph.query().has(TYPE_NAME_PROPERTY, typeName);
+
+ if(!StringUtils.isEmpty(name)) {
+ query = query.has(ASSERT_NAME_PROPERTY, name);
+ }
+
+ return query.vertices().iterator();
+ }
+
+ protected AtlasVertex getVertex(String typeName, String name) {
+ Iterator<AtlasVertex> iterator = getVertices(typeName, name);
+
+ return iterator.hasNext() ? iterator.next() : null;
+ }
+
+ protected void assertEdges(String typeName, String assetName, AtlasEdgeDirection edgeDirection, int startIdx, int expectedItems, String edgeTypeName) {
+ assertEdges(getVertex(typeName, assetName).getEdges(edgeDirection).iterator(),startIdx, expectedItems, edgeTypeName);
+ }
+
+ protected void assertEdges(Iterator<AtlasEdge> results, int startIdx, int expectedItems, String edgeTypeName) {
+ int count = 0;
+ AtlasEdge e = null;
+ for (Iterator<AtlasEdge> it = results; it.hasNext() && count < startIdx; count++) {
+ e = it.next();
+ }
+
+ assertNotNull(GraphHelper.getProperty(e, R_GUID_PROPERTY_NAME));
+ assertNotNull(GraphHelper.getProperty(e, "tagPropagation"));
+
+ if(StringUtils.isNotEmpty(edgeTypeName)) {
+ assertEquals(GraphHelper.getProperty(e, TYPE_NAME_PROPERTY), edgeTypeName, edgeTypeName);
+ }
+
+ assertEquals(count, expectedItems, String.format("%s", edgeTypeName));
+ }
+
+ protected void assertTypeAttribute(String typeName, int expectedSize, String name, String guid, String propertyName) {
+ AtlasVertex v = getVertex(typeName, name);
+ String guidActual = GraphHelper.getGuid(v);
+ List list = (List) GraphHelper.getProperty(v, propertyName);
+
+ assertEquals(guidActual, guid);
+ assertNotNull(list);
+ assertEquals(list.size(), expectedSize);
+ }
+
+ protected void assertTypeCountNameGuid(String typeName, int expectedItems, String name, String guid) {
+ Iterator<AtlasVertex> results = getVertices(typeName, name);
+
+ int count = 0;
+ for (Iterator<AtlasVertex> it = results; it.hasNext(); ) {
+ AtlasVertex v = it.next();
+
+ assertEquals(GraphHelper.getTypeName(v), typeName);
+
+ if(StringUtils.isNotEmpty(guid)) {
+ assertEquals(GraphHelper.getGuid(v), guid, name);
+ }
+
+ if(StringUtils.isNotEmpty(name)) {
+ assertEquals(GraphHelper.getProperty(v, ASSERT_NAME_PROPERTY), name, name);
+ }
+
+ count++;
+ }
+
+ assertEquals(count, expectedItems, String.format("%s:%s", typeName, name));
+ }
+
+ protected void assertMigrationStatus(int expectedTotalCount) {
+ AtlasVertex v = getVertex("__MigrationStatus", "");
+ assertEquals((long) GraphHelper.getProperty(v, "currentIndex"), expectedTotalCount);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
new file mode 100644
index 0000000..0be4be8
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+import org.jcodings.util.Hash;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Map;
+
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class RelationshipMappingTest {
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private AtlasTypeRegistry typeRegistry;
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
+ }
+
+ @Test
+ public void createLookup() {
+ Map<String, String> cache = RelationshipCacheGenerator.get(typeRegistry);
+ assertEquals(cache.size(), getLegacyAttributeCount() - 1);
+ for (Map.Entry<String, String> entry : cache.entrySet()) {
+ assertTrue(StringUtils.isNotEmpty(entry.getKey()));
+ assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey());
+ }
+ }
+
+ private int getLegacyAttributeCount() {
+ int count = 0;
+ for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) {
+ AtlasRelationshipDef rd = rt.getRelationshipDef();
+ if(rd.getEndDef1().getIsLegacyAttribute()) {
+ count++;
+ }
+
+ if(rd.getEndDef2().getIsLegacyAttribute()) {
+ count++;
+ }
+ }
+
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
index 8257faa..5d08924 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
@@ -54,6 +54,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -107,6 +108,8 @@ public class AtlasEntityStoreV1Test {
@Inject
private EntityGraphMapper graphMapper;
+ @Inject
+ private Configuration configuration;
@BeforeClass
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
index ac35860..8fc0327 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1Test.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.store.graph.v1;
+import org.apache.commons.configuration.Configuration;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
http://git-wip-us.apache.org/repos/asf/atlas/blob/3ded3ee2/repository/src/test/java/org/apache/atlas/utils/TestResourceFileUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/utils/TestResourceFileUtils.java b/repository/src/test/java/org/apache/atlas/utils/TestResourceFileUtils.java
index 1a01e3a..3f84057 100644
--- a/repository/src/test/java/org/apache/atlas/utils/TestResourceFileUtils.java
+++ b/repository/src/test/java/org/apache/atlas/utils/TestResourceFileUtils.java
@@ -55,6 +55,11 @@ public class TestResourceFileUtils {
return fs;
}
+ public static String getDirectory(String subDir) {
+ final String userDir = System.getProperty("user.dir");
+ return getTestFilePath(userDir, subDir, "");
+ }
+
public static <T> T readObjectFromJson(String subDir, String filename, Class<T> objectClass) throws IOException {
final String userDir = System.getProperty("user.dir");
String filePath = getTestJsonPath(userDir, subDir, filename);