You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/03/26 20:48:26 UTC
atlas git commit: ATLAS-2461: Migration export utility. Improved for
performance using GraphSON.
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 5f877dda4 -> ef642b4cb
ATLAS-2461: Migration export utility. Improved for performance using GraphSON.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/ef642b4c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/ef642b4c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/ef642b4c
Branch: refs/heads/branch-0.8
Commit: ef642b4cb612fb15c433932967c462e6705d005a
Parents: 5f877dd
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Mon Mar 26 13:43:23 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Mon Mar 26 13:48:05 2018 -0700
----------------------------------------------------------------------
.../src/main/assemblies/migration-exporter.xml | 4 +-
tools/atlas-migration-exporter/pom.xml | 23 +-
.../org/apache/atlas/migration/Exporter.java | 232 +++++++++----------
.../src/main/resources/README | 20 +-
.../src/main/resources/atlas-log4j.xml | 73 ++++++
.../src/main/resources/atlas_migration.py | 141 -----------
.../main/resources/atlas_migration_export.py | 154 ++++++++++++
.../resources/migration-export-request.json | 4 -
8 files changed, 359 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/distro/src/main/assemblies/migration-exporter.xml
----------------------------------------------------------------------
diff --git a/distro/src/main/assemblies/migration-exporter.xml b/distro/src/main/assemblies/migration-exporter.xml
index 8f751ff..be0a9d5 100644
--- a/distro/src/main/assemblies/migration-exporter.xml
+++ b/distro/src/main/assemblies/migration-exporter.xml
@@ -25,6 +25,8 @@
<format>zip</format>
</formats>
+ <baseDirectory>atlas-migration-exporter</baseDirectory>
+
<fileSets>
<fileSet>
<includes>
@@ -37,7 +39,7 @@
<includes>
<include>*.py</include>
<include>migrationContext.xml</include>
- <include>*.json</include>
+ <include>atlas-log4j.xml</include>
<include>README</include>
</includes>
<fileMode>0755</fileMode>
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/pom.xml
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/pom.xml b/tools/atlas-migration-exporter/pom.xml
index 5c6c61e..8ecd306 100644
--- a/tools/atlas-migration-exporter/pom.xml
+++ b/tools/atlas-migration-exporter/pom.xml
@@ -26,13 +26,12 @@
<artifactId>apache-atlas</artifactId>
<version>0.8.3-SNAPSHOT</version>
</parent>
- <artifactId>atlas-migration</artifactId>
+ <artifactId>atlas-migration-exporter</artifactId>
<description>Apache Atlas Migration Exporter</description>
<name>Apache Atlas Migration Exporter</name>
<packaging>jar</packaging>
<dependencies>
-
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
@@ -40,11 +39,7 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework</groupId>
- <artifactId>spring-context</artifactId>
- <version>4.3.8.RELEASE</version>
+ <version>${spring.version}</version>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
@@ -54,6 +49,18 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.tinkerpop.blueprints</groupId>
+ <artifactId>blueprints-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.thinkaurelius.titan</groupId>
+ <artifactId>titan-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.atlas</groupId>
+ <artifactId>atlas-graphdb-titan0</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
-
</project>
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
index a9873df..2af8b7d 100644
--- a/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
+++ b/tools/atlas-migration-exporter/src/main/java/org/apache/atlas/migration/Exporter.java
@@ -18,11 +18,19 @@
package org.apache.atlas.migration;
-import org.apache.atlas.model.impexp.AtlasExportRequest;
-import org.apache.atlas.repository.impexp.ExportService;
-import org.apache.atlas.repository.impexp.ZipSink;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Graph;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONWriter;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.graphdb.titan0.Titan0GraphDatabase;
import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Options;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
@@ -30,184 +38,156 @@ import org.springframework.context.support.ClassPathXmlApplicationContext;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.IOException;
import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
+import java.util.ArrayList;
+
public class Exporter {
private static final Logger LOG = LoggerFactory.getLogger(Exporter.class);
- private static final String EXPORT_REQUEST_JSON_FILE = "migration-export-request.json";
- private static final String ATLAS_EXPORT_SERVICE = "exportService";
- private static final String APPLICATION_CONTEXT = "migrationContext.xml";
- private static final int PROGRAM_ERROR_STATUS = -1;
-
-
- private ApplicationContext applicationContext;
+ private static final String ATLAS_TYPE_REGISTRY = "atlasTypeRegistry";
+ private static final String APPLICATION_CONTEXT = "migrationContext.xml";
+ private static final String MIGRATION_TYPESDEF_FILENAME = "atlas-migration-typesdef.json";
+ private static final String MIGRATION_DATA_FILENAME = "atlas-migration-data.json";
+ private static final String LOG_MSG_PREFIX = "atlas-migration-export: ";
+ private static final int PROGRAM_ERROR_STATUS = -1;
+ private static final int PROGRAM_SUCCESS_STATUS = 0;
+ private final String typesDefFileName;
+ private final String dataFileName;
+ private final AtlasTypeRegistry typeRegistry;
public static void main(String args[]) {
- int result = PROGRAM_ERROR_STATUS;
+ int result;
try {
- display("=== Atlas Migration: Export === >>");
- String fileName = getExportToFileName(args);
+ String logFileName = System.getProperty("atlas.log.dir") + File.separatorChar + System.getProperty("atlas.log.file");
- Exporter exporter = new Exporter(APPLICATION_CONTEXT);
+ displayMessage("starting migration export. Log file location " + logFileName);
- result = exporter.perform(fileName);
+ Options options = new Options();
+ options.addOption("d", "outputdir", true, "Output directory");
- display("<< === Atlas Migration: Export: Done! ===");
- } catch (Exception e) {
- LOG.error("<=== Atlas Migration: Export: Failed! ===", e);
+ CommandLine cmd = (new BasicParser()).parse(options, args);
+ String outputDir = cmd.getOptionValue("d");
- result = PROGRAM_ERROR_STATUS;
- }
+ if (StringUtils.isEmpty(outputDir)) {
+ outputDir = System.getProperty("user.dir");
+ }
- System.exit(result);
- }
+ String typesDefFileName = outputDir + File.separatorChar + MIGRATION_TYPESDEF_FILENAME;
+ String dataFileName = outputDir + File.separatorChar + MIGRATION_DATA_FILENAME;
- public Exporter(String contextXml) {
- try {
- applicationContext = new ClassPathXmlApplicationContext(contextXml);
- } catch (Exception ex) {
- LOG.error("Initialization failed!", ex);
+ Exporter exporter = new Exporter(typesDefFileName, dataFileName, APPLICATION_CONTEXT);
- throw ex;
- }
- }
+ exporter.perform();
- public int perform(String fileName) {
- LOG.info("Starting export to {}", fileName);
+ result = PROGRAM_SUCCESS_STATUS;
- int ret = 0;
- AtlasExportRequest req = getRequest(getExportRequestFile(), getDefaultExportRequest());
- OutputStream os = null;
- ZipSink zs = null;
+ displayMessage("completed migration export!");
+ } catch (Exception e) {
+ displayError("Failed", e);
- try {
- os = new FileOutputStream(fileName);
- zs = new ZipSink(os);
+ result = PROGRAM_ERROR_STATUS;
+ }
- ExportService svc = getExportService();
+ System.exit(result);
+ }
- svc.run(zs, req, getUserName(), getHostName(), getIPAddress());
+ public Exporter(String typesDefFileName, String dataFileName, String contextXml) throws Exception {
+ validate(typesDefFileName, dataFileName);
- ret = 0;
- } catch (Exception ex) {
- LOG.error("Export failed!", ex);
+ displayMessage("initializing");
- ret = PROGRAM_ERROR_STATUS;
- } finally {
- if (zs != null) {
- try {
- zs.close();
- } catch (Throwable t) {
- // ignore
- }
- }
+ ApplicationContext applicationContext = new ClassPathXmlApplicationContext(contextXml);
- if (os != null) {
- try {
- os.close();
- } catch (Throwable t) {
- // ignore
- }
- }
- }
+ this.typesDefFileName = typesDefFileName;
+ this.dataFileName = dataFileName;
+ this.typeRegistry = applicationContext.getBean(ATLAS_TYPE_REGISTRY, AtlasTypeRegistry.class);;
- return ret;
+ displayMessage("initialized");
}
- private AtlasExportRequest getRequest(File requestFile, String defaultJson) {
- String reqJson = null;
-
- try {
- if (requestFile.exists()) {
- LOG.info("Using request from the file {}", requestFile.getPath());
+ public void perform() throws Exception {
+ exportTypes();
+ exportData();
+ }
- reqJson = FileUtils.readFileToString(requestFile);
- } else {
- LOG.info("Using default request...");
+ private void validate(String typesDefFileName, String dataFileName) throws Exception {
+ File typesDefFile = new File(typesDefFileName);
+ File dataFile = new File(dataFileName);
- reqJson = defaultJson;
- }
- } catch (IOException e) {
- LOG.error("Error reading request from {}", requestFile.getPath());
+ if (typesDefFile.exists()) {
+ throw new Exception("output file " + typesDefFileName + " already exists");
+ }
- reqJson = defaultJson;
+ if (dataFile.exists()) {
+ throw new Exception("output file " + dataFileName + " already exists");
}
+ }
- LOG.info("Export request: {}", reqJson);
+ private void exportTypes() throws Exception {
+ displayMessage("exporting typesDef to file " + typesDefFileName);
- return AtlasType.fromJson(reqJson, AtlasExportRequest.class);
- }
+ AtlasTypesDef typesDef = getTypesDef(typeRegistry);
- private ExportService getExportService() {
- return applicationContext.getBean(ATLAS_EXPORT_SERVICE, ExportService.class);
- }
+ FileUtils.write(new File(typesDefFileName), AtlasType.toJson(typesDef));
- private String getUserName() {
- return System.getProperty("user.name");
+ displayMessage("exported typesDef to file " + typesDefFileName);
}
- private String getHostName() {
- try {
- return InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- LOG.error("faild to get hostname; using localhost", e);
+ private void exportData() throws Exception {
+ displayMessage("exporting data to file " + dataFileName);
- return "localhost";
- }
- }
+ OutputStream os = null;
- private String getIPAddress() {
try {
- return InetAddress.getLocalHost().toString();
- } catch (UnknownHostException e) {
- LOG.error("failed to get IP address; using 127.0.0.1", e);
+ os = new FileOutputStream(dataFileName);
+
+ Graph graph = getTitan0GraphDatabase();
- return "127.0.0.1";
+ GraphSONWriter.outputGraph(graph, os, GraphSONMode.EXTENDED);
+ } finally {
+ if (os != null) {
+ try {
+ os.close();
+ } catch (Exception excp) {
+ // ignore
+ }
+ }
}
- }
- private File getExportRequestFile() {
- return getFile(getCurrentDirectory(), EXPORT_REQUEST_JSON_FILE);
+ displayMessage("exported data to file " + dataFileName);
}
- private File getFile(String currentDir, String fileName) {
- LOG.info("Attempting to use request file: {}/{}", currentDir, fileName);
- return new File(currentDir, fileName);
+ private AtlasTypesDef getTypesDef(AtlasTypeRegistry registry) {
+ return new AtlasTypesDef(new ArrayList<>(registry.getAllEnumDefs()),
+ new ArrayList<>(registry.getAllStructDefs()),
+ new ArrayList<>(registry.getAllClassificationDefs()),
+ new ArrayList<>(registry.getAllEntityDefs()));
}
- private String getCurrentDirectory() {
- return System.getProperty("user.dir");
+ private TitanGraph getTitan0GraphDatabase() {
+ return Titan0GraphDatabase.getGraphInstance();
}
- private String getDefaultExportRequest() {
- return "{ \"itemsToExport\": [ { \"typeName\": \"hive_db\" } ], \"options\": { \"fetchType\": \"FULL\", \"matchType\": \"forType\"} }";
- }
+ private static void displayMessage(String msg) {
+ LOG.info(LOG_MSG_PREFIX + msg);
- private static void display(String s) {
- LOG.info(s);
+ System.out.println(LOG_MSG_PREFIX + msg);
+ System.out.flush();
}
- private static String getExportToFileName(String[] args) {
- String fileName = (args.length > 0) ? args[0] : getDefaultFileName();
+ private static void displayError(String msg, Throwable t) {
+ LOG.error(LOG_MSG_PREFIX + msg, t);
- if (args.length == 0) {
- printUsage(fileName);
- }
+ System.out.println(LOG_MSG_PREFIX + msg);
+ System.out.flush();
- return fileName;
- }
-
- private static void printUsage(String fileName) {
- display("Exporting to file " + fileName + ". To export data to a different file, please specify the file path as argument");
- }
+ if (t != null) {
+ System.out.println("ERROR: " + t.getMessage());
+ }
- private static String getDefaultFileName() {
- return String.format("atlas-export-%s.zip", System.currentTimeMillis());
+ System.out.flush();
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/src/main/resources/README
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/resources/README b/tools/atlas-migration-exporter/src/main/resources/README
index 2f2bf3e..30dd493 100644
--- a/tools/atlas-migration-exporter/src/main/resources/README
+++ b/tools/atlas-migration-exporter/src/main/resources/README
@@ -17,25 +17,21 @@
#
Introduction
- The purpose of this utility is to export data from Atlas to a ZIP file.
+ The purpose of this utility is to export type definitions and data from an Atlas repository.
What is Exported?
- Metadata of all Hive databases and all entities connected to these databases (tables, columns, processes, paths) are
- exported. To change the scope of export, modify the accompanying migration-export-request.json. More details of this
- JSON file format can be found in http://atlas.apache.org/Export-API.html
+ All data and types are exported.
How Much Time Will this Take?
- The duration of the export process depends on the number of entities present in your database. Approximately it takes
- about 2.5ms for each entity in the database. For a database with 1 million entities, export will take about 40 minutes.
+ The duration of the export process depends on the number of entities present in your database. While cluster configuration determines speed of operation,
+ on an average, for cluster with reasonable configuration, it takes 30 minutes to export 1 million entities.
Steps to Start Export step of Migration
- - Shutdown Atlas. This is critical to ensure that no updates are being made to Atlas database while export is in progress.
+ - Shutdown Atlas. This is critical to ensure that no updates are being made to Atlas database while the operation is in progress.
- Execute the following commands in the host where Atlas server runs:
- - unzip atlas-migration-kit.zip
- - cd atlas-migration-kit
- - python atlas_migration.py atlas-export.zip
- if zip file path is no specified, exported data will be saved to a file named atlas-export-<timestamp>.zip
- - To watch the progress: 'tail -f /var/log/atlas/application.log'.
+ - unzip atlas-migration-exporter.zip
+ - cd atlas-migration-exporter
+ - python atlas_migration_export.py
Next Steps
Once done, please use the Atlas Migration Guide for next steps.
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml b/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml
new file mode 100755
index 0000000..ae0d27d
--- /dev/null
+++ b/tools/atlas-migration-exporter/src/main/resources/atlas-log4j.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+ <appender name="console" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <appender name="FILE" class="org.apache.log4j.RollingFileAppender">
+ <param name="File" value="${atlas.log.dir}/${atlas.log.file}"/>
+ <param name="Append" value="true"/>
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p - [%t:%x] ~ %m (%C{1}:%L)%n"/>
+ </layout>
+ </appender>
+
+ <logger name="org.apache.atlas" additivity="false">
+ <level value="info"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="com.thinkaurelius.titan" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <logger name="org.springframework" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </logger>
+
+ <logger name="org.eclipse" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </logger>
+
+ <logger name="com.sun.jersey" additivity="false">
+ <level value="warn"/>
+ <appender-ref ref="console"/>
+ </logger>
+
+ <!-- to avoid logs - The configuration log.flush.interval.messages = 1 was supplied but isn't a known config -->
+ <logger name="org.apache.kafka.common.config.AbstractConfig" additivity="false">
+ <level value="error"/>
+ <appender-ref ref="FILE"/>
+ </logger>
+
+ <root>
+ <priority value="warn"/>
+ <appender-ref ref="FILE"/>
+ </root>
+</log4j:configuration>
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py b/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py
deleted file mode 100755
index 199cde2..0000000
--- a/tools/atlas-migration-exporter/src/main/resources/atlas_migration.py
+++ /dev/null
@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-import sys
-sys.path.insert(0, '/usr/hdp/current/atlas-server/bin/')
-
-import traceback
-import atlas_config as mc
-
-ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=%s.log"
-ATLAS_COMMAND_OPTS="-Datlas.home=%s"
-ATLAS_CONFIG_OPTS="-Datlas.conf=%s"
-DEFAULT_JVM_HEAP_OPTS="-Xmx4096m -XX:MaxPermSize=512m"
-DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true -server"
-
-def main():
- is_setup = (len(sys.argv)>1) and sys.argv[1] is not None and sys.argv[1] == '-setup'
- atlas_home = mc.atlasDir()
- confdir = mc.dirMustExist(mc.confDir(atlas_home))
- mc.executeEnvSh(confdir)
- logdir = mc.dirMustExist(mc.logDir(atlas_home))
- mc.dirMustExist(mc.dataDir(atlas_home))
-
- if mc.isCygwin():
- # Pathnames that are passed to JVM must be converted to Windows format.
- jvm_atlas_home = mc.convertCygwinPath(atlas_home)
- jvm_confdir = mc.convertCygwinPath(confdir)
- jvm_logdir = mc.convertCygwinPath(logdir)
- else:
- jvm_atlas_home = atlas_home
- jvm_confdir = confdir
- jvm_logdir = logdir
-
- #create sys property for conf dirs
- if not is_setup:
- jvm_opts_list = (ATLAS_LOG_OPTS % (jvm_logdir, "application")).split()
- else:
- jvm_opts_list = (ATLAS_LOG_OPTS % (jvm_logdir, "atlas_setup")).split()
-
- cmd_opts = (ATLAS_COMMAND_OPTS % jvm_atlas_home)
- jvm_opts_list.extend(cmd_opts.split())
-
- config_opts = (ATLAS_CONFIG_OPTS % jvm_confdir)
- jvm_opts_list.extend(config_opts.split())
-
- atlas_server_heap_opts = os.environ.get(mc.ATLAS_SERVER_HEAP, DEFAULT_JVM_HEAP_OPTS)
- jvm_opts_list.extend(atlas_server_heap_opts.split())
-
- atlas_server_jvm_opts = os.environ.get(mc.ATLAS_SERVER_OPTS)
- if atlas_server_jvm_opts:
- jvm_opts_list.extend(atlas_server_jvm_opts.split())
-
- atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, DEFAULT_JVM_OPTS)
- jvm_opts_list.extend(atlas_jvm_opts.split())
-
- #expand web app dir
- web_app_dir = mc.webAppDir(atlas_home)
- mc.expandWebApp(atlas_home)
-
- p = os.pathsep
- atlas_classpath = confdir + p \
- + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \
- + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \
- + os.path.join(atlas_home, "libext", "*") + p \
- + os.path.join(os.getcwd(), ".", "*")
-
- is_hbase = mc.is_hbase(confdir)
-
- if is_hbase:
- #add hbase-site.xml to classpath
- hbase_conf_dir = mc.hbaseConfDir(atlas_home)
-
- if os.path.exists(hbase_conf_dir):
- atlas_classpath = atlas_classpath + p \
- + hbase_conf_dir
- else:
- if mc.is_hbase(confdir):
- raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir)
-
- if mc.isCygwin():
- atlas_classpath = mc.convertCygwinPath(atlas_classpath, True)
-
- atlas_pid_file = mc.pidFile(atlas_home)
-
- if os.path.isfile(atlas_pid_file):
- #Check if process listed in atlas.pid file is still running
- pf = file(atlas_pid_file, 'r')
- pid = pf.read().strip()
- pf.close()
-
- if mc.exist_pid((int)(pid)):
- mc.server_already_running(pid)
- else:
- mc.server_pid_not_running(pid)
-
- if is_hbase and mc.is_hbase_local(confdir):
- print "configured for local hbase."
- mc.configure_hbase(atlas_home)
- mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
- print "hbase started."
-
- web_app_path = os.path.join(web_app_dir, "atlas")
- if (mc.isCygwin()):
- web_app_path = mc.convertCygwinPath(web_app_path)
-
- start_atlas_server(atlas_classpath, atlas_pid_file, jvm_logdir, jvm_opts_list, web_app_path)
- print "Apache Atlas Migration started!\n"
-
-def start_atlas_server(atlas_classpath, atlas_pid_file, jvm_logdir, jvm_opts_list, web_app_path):
- args = []
- args.extend(sys.argv[1:])
- print "Arguments: ".join(str(x) for x in args)
- process = mc.java("org.apache.atlas.migration.Exporter", args, atlas_classpath, jvm_opts_list, jvm_logdir)
- print "To watch the progress use: tail -f /var/log/atlas/application.log"
- mc.writePid(atlas_pid_file, process)
-
-
-if __name__ == '__main__':
- try:
- returncode = main()
- except Exception as e:
- print "Exception: %s " % str(e)
- print traceback.format_exc()
- returncode = -1
-
- sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py b/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py
new file mode 100755
index 0000000..ab7c8e5
--- /dev/null
+++ b/tools/atlas-migration-exporter/src/main/resources/atlas_migration_export.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import sys
+sys.path.insert(0, '/usr/hdp/current/atlas-server/bin/')
+
+import traceback
+import subprocess
+import atlas_config as mc
+
+ATLAS_LOG_OPTS="-Datlas.log.dir=%s -Datlas.log.file=atlas-migration-exporter.log"
+ATLAS_COMMAND_OPTS="-Datlas.home=%s"
+ATLAS_CONFIG_OPTS="-Datlas.conf=%s"
+DEFAULT_JVM_HEAP_OPTS="-Xmx4096m -XX:MaxPermSize=512m"
+DEFAULT_JVM_OPTS="-Dlog4j.configuration=atlas-log4j.xml -Djava.net.preferIPv4Stack=true -server"
+
+def main():
+ is_setup = (len(sys.argv)>1) and sys.argv[1] is not None and sys.argv[1] == '-setup'
+ atlas_home = mc.atlasDir()
+ confdir = mc.dirMustExist(mc.confDir(atlas_home))
+ mc.executeEnvSh(confdir)
+ logdir = mc.dirMustExist(mc.logDir(atlas_home))
+ mc.dirMustExist(mc.dataDir(atlas_home))
+
+ if mc.isCygwin():
+ # Pathnames that are passed to JVM must be converted to Windows format.
+ jvm_atlas_home = mc.convertCygwinPath(atlas_home)
+ jvm_confdir = mc.convertCygwinPath(confdir)
+ jvm_logdir = mc.convertCygwinPath(logdir)
+ else:
+ jvm_atlas_home = atlas_home
+ jvm_confdir = confdir
+ jvm_logdir = logdir
+
+ #create sys property for conf dirs
+ jvm_opts_list = (ATLAS_LOG_OPTS % (jvm_logdir)).split()
+
+ cmd_opts = (ATLAS_COMMAND_OPTS % jvm_atlas_home)
+ jvm_opts_list.extend(cmd_opts.split())
+
+ config_opts = (ATLAS_CONFIG_OPTS % jvm_confdir)
+ jvm_opts_list.extend(config_opts.split())
+
+ atlas_server_heap_opts = os.environ.get(mc.ATLAS_SERVER_HEAP, DEFAULT_JVM_HEAP_OPTS)
+ jvm_opts_list.extend(atlas_server_heap_opts.split())
+
+ atlas_server_jvm_opts = os.environ.get(mc.ATLAS_SERVER_OPTS)
+ if atlas_server_jvm_opts:
+ jvm_opts_list.extend(atlas_server_jvm_opts.split())
+
+ atlas_jvm_opts = os.environ.get(mc.ATLAS_OPTS, DEFAULT_JVM_OPTS)
+ jvm_opts_list.extend(atlas_jvm_opts.split())
+
+ #expand web app dir
+ web_app_dir = mc.webAppDir(atlas_home)
+ mc.expandWebApp(atlas_home)
+
+ p = os.pathsep
+ atlas_classpath = os.path.join(os.getcwd(), ".", "*") + p \
+ + confdir + p \
+ + os.path.join(web_app_dir, "atlas", "WEB-INF", "classes" ) + p \
+ + os.path.join(web_app_dir, "atlas", "WEB-INF", "lib", "*" ) + p \
+ + os.path.join(atlas_home, "libext", "*")
+
+ is_hbase = mc.is_hbase(confdir)
+
+ if is_hbase:
+ #add hbase-site.xml to classpath
+ hbase_conf_dir = mc.hbaseConfDir(atlas_home)
+
+ if os.path.exists(hbase_conf_dir):
+ atlas_classpath = atlas_classpath + p \
+ + hbase_conf_dir
+ else:
+ if mc.is_hbase(confdir):
+ raise Exception("Could not find hbase-site.xml in %s. Please set env var HBASE_CONF_DIR to the hbase client conf dir", hbase_conf_dir)
+
+ if mc.isCygwin():
+ atlas_classpath = mc.convertCygwinPath(atlas_classpath, True)
+
+ atlas_pid_file = mc.pidFile(atlas_home)
+
+ if os.path.isfile(atlas_pid_file):
+ #Check if process listed in atlas.pid file is still running
+ pf = file(atlas_pid_file, 'r')
+ pid = pf.read().strip()
+ pf.close()
+
+ if mc.exist_pid((int)(pid)):
+ mc.server_already_running(pid)
+ else:
+ mc.server_pid_not_running(pid)
+
+ if is_hbase and mc.is_hbase_local(confdir):
+ print "configured for local hbase."
+ mc.configure_hbase(atlas_home)
+ mc.run_hbase_action(mc.hbaseBinDir(atlas_home), "start", hbase_conf_dir, logdir)
+ print "hbase started."
+
+ web_app_path = os.path.join(web_app_dir, "atlas")
+ if (mc.isCygwin()):
+ web_app_path = mc.convertCygwinPath(web_app_path)
+
+ start_migration_export(atlas_classpath, atlas_pid_file, jvm_logdir, jvm_opts_list, web_app_path)
+
+def start_migration_export(atlas_classpath, atlas_pid_file, jvm_logdir, jvm_opts_list, web_app_path):
+ args = []
+ args.extend(sys.argv[1:])
+ process = java("org.apache.atlas.migration.Exporter", args, atlas_classpath, jvm_opts_list)
+
+def java(classname, args, classpath, jvm_opts_list):
+ java_home = os.environ.get("JAVA_HOME", None)
+ if java_home:
+ prg = os.path.join(java_home, "bin", "java")
+ else:
+ prg = mc.which("java")
+
+ if prg is None:
+ raise EnvironmentError('The java binary could not be found in your path or JAVA_HOME')
+
+ commandline = [prg]
+ commandline.extend(jvm_opts_list)
+ commandline.append("-classpath")
+ commandline.append(classpath)
+ commandline.append(classname)
+ commandline.extend(args)
+
+ p = subprocess.Popen(commandline)
+ p.communicate()
+
+if __name__ == '__main__':
+ try:
+ returncode = main()
+ except Exception as e:
+ print "Exception: %s " % str(e)
+ print traceback.format_exc()
+ returncode = -1
+
+ sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/atlas/blob/ef642b4c/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
----------------------------------------------------------------------
diff --git a/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json b/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
deleted file mode 100644
index 64002af..0000000
--- a/tools/atlas-migration-exporter/src/main/resources/migration-export-request.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
- "itemsToExport": [ { "typeName": "hive_db" } ],
- "options": { "fetchType": "FULL", "matchType": "forType" }
-}