You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2016/11/08 00:00:15 UTC
[4/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap
to load from new format typedef JSON files
ATLAS-1272: updated types bootstrap to load from new format typedef JSON files
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/def9e385
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/def9e385
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/def9e385
Branch: refs/heads/master
Commit: def9e385cc17c2bceb9450c82947370fadc09c3c
Parents: 2ea3a45
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Sun Nov 6 19:32:16 2016 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Nov 7 14:06:57 2016 -0800
----------------------------------------------------------------------
addons/falcon-bridge/pom.xml | 18 -
.../atlas/falcon/bridge/FalconBridge.java | 46 +-
.../falcon/model/FalconDataModelGenerator.java | 214 --------
.../apache/atlas/falcon/hook/FalconHookIT.java | 22 +-
addons/hdfs-model/pom.xml | 18 -
.../atlas/fs/model/FSDataModelGenerator.java | 38 --
.../org/apache/atlas/fs/model/FSDataModel.scala | 95 ----
.../apache/atlas/fs/model/HDFSModelTest.java | 83 ----
addons/hive-bridge/pom.xml | 18 -
.../atlas/hive/bridge/ColumnLineageUtils.java | 5 +-
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 101 ++--
.../org/apache/atlas/hive/hook/HiveHook.java | 9 +-
.../hive/model/HiveDataModelGenerator.java | 350 -------------
.../java/org/apache/atlas/hive/HiveITBase.java | 4 +-
.../hive/bridge/HiveMetaStoreBridgeTest.java | 6 +-
.../org/apache/atlas/hive/hook/HiveHookIT.java | 73 ++-
addons/models/0010-base_model.json | 97 ++++
addons/models/0020-fs_model.json | 191 +++++++
addons/models/0030-hive_model.json | 498 +++++++++++++++++++
addons/models/0040-sqoop_model.json | 97 ++++
addons/models/0050-falcon_model.json | 147 ++++++
addons/models/0060-hbase_model.json | 24 +
addons/models/0070-kafka_model.json | 57 +++
addons/models/0080-storm_model.json | 150 ++++++
.../patches/001-hive_column_add_position.json | 21 +
addons/sqoop-bridge/pom.xml | 18 -
.../org/apache/atlas/sqoop/hook/SqoopHook.java | 46 +-
.../sqoop/model/SqoopDataModelGenerator.java | 187 -------
.../apache/atlas/sqoop/hook/SqoopHookIT.java | 26 -
addons/storm-bridge/pom.xml | 18 -
.../apache/atlas/storm/hook/StormAtlasHook.java | 6 +-
.../storm/model/StormDataModelGenerator.java | 41 --
.../atlas/storm/model/StormDataModel.scala | 104 ----
.../atlas/storm/hook/StormAtlasHookIT.java | 56 ---
.../src/main/assemblies/standalone-package.xml | 30 +-
.../java/org/apache/atlas/AtlasErrorCode.java | 4 +-
.../atlas/model/typedef/AtlasStructDef.java | 26 +-
.../atlas/model/typedef/AtlasTypesDef.java | 2 +
.../org/apache/atlas/type/AtlasStructType.java | 14 +-
.../apache/atlas/type/AtlasTypeRegistry.java | 8 +
.../apache/atlas/RepositoryMetadataModule.java | 4 -
.../graph/GraphBackedSearchIndexer.java | 4 +-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 364 ++++++++++++++
.../store/graph/AtlasTypeDefGraphStore.java | 13 +
.../store/graph/v1/AtlasStructDefStoreV1.java | 35 +-
.../atlas/services/AtlasPatchHandler.java | 173 -------
.../atlas/services/AtlasTypeAttributePatch.java | 296 -----------
.../apache/atlas/services/AtlasTypePatch.java | 104 ----
.../atlas/services/DefaultMetadataService.java | 82 +--
.../services/IBootstrapTypesRegistrar.java | 27 -
.../atlas/services/ReservedTypesRegistrar.java | 135 -----
.../ReservedTypesRegistrationException.java | 26 -
.../org/apache/atlas/BaseRepositoryTest.java | 42 ++
.../test/java/org/apache/atlas/TestUtils.java | 2 -
.../store/graph/AtlasTypeDefGraphStoreTest.java | 4 +-
.../DefaultMetadataServiceMockTest.java | 151 ------
.../services/ReservedTypesRegistrarTest.java | 103 ----
57 files changed, 1917 insertions(+), 2616 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index de3e810..96743e7 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -355,24 +355,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.falcon.model.FalconDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/falcon_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
index 05d072f..d1a7e87 100644
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
@@ -21,11 +21,8 @@ package org.apache.atlas.falcon.bridge;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.falcon.Util.EventUtil;
-import org.apache.atlas.falcon.model.FalconDataModelGenerator;
import org.apache.atlas.falcon.model.FalconDataTypes;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.commons.collections.CollectionUtils;
@@ -61,6 +58,15 @@ import java.util.Map;
public class FalconBridge {
private static final Logger LOG = LoggerFactory.getLogger(FalconBridge.class);
+ public static final String COLO = "colo";
+ public static final String TAGS = "tags";
+ public static final String GROUPS = "groups";
+ public static final String PIPELINES = "pipelines";
+ public static final String WFPROPERTIES = "workflow-properties";
+ public static final String RUNSON = "runs-on";
+ public static final String STOREDIN = "stored-in";
+ public static final String FREQUENCY = "frequency";
+
/**
* Creates cluster entity
*
@@ -77,14 +83,14 @@ public class FalconBridge {
clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription());
clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
- clusterRef.set(FalconDataModelGenerator.COLO, cluster.getColo());
+ clusterRef.set(FalconBridge.COLO, cluster.getColo());
if (cluster.getACL() != null) {
clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup());
}
if (StringUtils.isNotEmpty(cluster.getTags())) {
- clusterRef.set(FalconDataModelGenerator.TAGS,
+ clusterRef.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(cluster.getTags()));
}
@@ -100,19 +106,19 @@ public class FalconBridge {
String feedQualifiedName =
getFeedQualifiedName(feed.getName(), (String) clusterReferenceable.get(AtlasClient.NAME));
feedEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName);
- feedEntity.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency().toString());
- feedEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable);
+ feedEntity.set(FalconBridge.FREQUENCY, feed.getFrequency().toString());
+ feedEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
if (feed.getACL() != null) {
feedEntity.set(AtlasClient.OWNER, feed.getACL().getOwner());
}
if (StringUtils.isNotEmpty(feed.getTags())) {
- feedEntity.set(FalconDataModelGenerator.TAGS,
+ feedEntity.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(feed.getTags()));
}
if (feed.getGroups() != null) {
- feedEntity.set(FalconDataModelGenerator.GROUPS, feed.getGroups());
+ feedEntity.set(FalconBridge.GROUPS, feed.getGroups());
}
return feedEntity;
@@ -165,7 +171,7 @@ public class FalconBridge {
feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
}
- feedCreateEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable);
+ feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
entities.add(feedCreateEntity);
}
@@ -244,7 +250,7 @@ public class FalconBridge {
processEntity.set(AtlasClient.NAME, process.getName());
processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
getProcessQualifiedName(process.getName(), cluster.getName()));
- processEntity.set(FalconDataModelGenerator.FREQUENCY, process.getFrequency().toString());
+ processEntity.set(FalconBridge.FREQUENCY, process.getFrequency().toString());
if (!inputs.isEmpty()) {
processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
@@ -254,7 +260,7 @@ public class FalconBridge {
}
// set cluster
- processEntity.set(FalconDataModelGenerator.RUNSON, clusterReferenceable);
+ processEntity.set(FalconBridge.RUNSON, clusterReferenceable);
// Set user
if (process.getACL() != null) {
@@ -262,15 +268,15 @@ public class FalconBridge {
}
if (StringUtils.isNotEmpty(process.getTags())) {
- processEntity.set(FalconDataModelGenerator.TAGS,
+ processEntity.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(process.getTags()));
}
if (process.getPipelines() != null) {
- processEntity.set(FalconDataModelGenerator.PIPELINES, process.getPipelines());
+ processEntity.set(FalconBridge.PIPELINES, process.getPipelines());
}
- processEntity.set(FalconDataModelGenerator.WFPROPERTIES,
+ processEntity.set(FalconBridge.WFPROPERTIES,
getProcessEntityWFProperties(process.getWorkflow(),
process.getName()));
@@ -319,7 +325,7 @@ public class FalconBridge {
private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) {
List<Referenceable> entities = new ArrayList<>();
- Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+ Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
ref.set("path", pathUri);
// Path path = new Path(pathUri);
// ref.set("name", path.getName());
@@ -352,7 +358,7 @@ public class FalconBridge {
tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
- tableRef.set(HiveDataModelGenerator.DB, dbRef);
+ tableRef.set(HiveMetaStoreBridge.DB, dbRef);
entities.add(tableRef);
return entities;
@@ -364,7 +370,7 @@ public class FalconBridge {
Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName));
clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName);
- clusterRef.set(FalconDataModelGenerator.COLO, colo);
+ clusterRef.set(FalconBridge.COLO, colo);
return clusterRef;
}
@@ -375,8 +381,8 @@ public class FalconBridge {
feedDatasetRef.set(AtlasClient.NAME, feed.getName());
feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(),
(String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)));
- feedDatasetRef.set(FalconDataModelGenerator.STOREDIN, clusterReference);
- feedDatasetRef.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency());
+ feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference);
+ feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency());
return feedDatasetRef;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
deleted file mode 100644
index fc0a9b2..0000000
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.falcon.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates falcon data model.
- */
-public class FalconDataModelGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(FalconDataModelGenerator.class);
- public static final String FREQUENCY = "frequency";
-
- private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
-
- public static final String COLO = "colo";
- public static final String TAGS = "tags";
- public static final String GROUPS = "groups";
- public static final String PIPELINES = "pipelines";
- public static final String WFPROPERTIES = "workflow-properties";
- public static final String RUNSON = "runs-on";
- public static final String STOREDIN = "stored-in";
-
- public FalconDataModelGenerator() {
- classTypeDefinitions = new HashMap<>();
- }
-
- public void createDataModel() throws AtlasException {
- LOG.info("Generating the Falcon Data Model");
-
- // classes
- createClusterEntityClass();
- createProcessEntityClass();
- createFeedCreationEntityClass();
- createFeedEntityClass();
- createReplicationFeedEntityClass();
- }
-
- private TypesDef getTypesDef() {
- return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- getTraitTypeDefinitions(), getClassTypeDefinitions());
- }
-
- public String getDataModelAsJSON() {
- return TypesSerialization.toJson(getTypesDef());
- }
-
- private ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
- return ImmutableList.copyOf(classTypeDefinitions.values());
- }
-
- private ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
- return ImmutableList.of();
- }
-
- private void createClusterEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- // map of tags
- new AttributeDefinition(TAGS,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_CLUSTER.getName(), null,
- ImmutableSet.of(AtlasClient.INFRASTRUCTURE_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_CLUSTER.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_CLUSTER.getName());
- }
-
- private void createFeedCreationEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, false, true, null)
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED_CREATION.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_CREATION.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_CREATION.getName());
- }
-
- private void createFeedEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
- new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, false, true, null),
- new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- // map of tags
- new AttributeDefinition(TAGS,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null)
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED.getName(), null,
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_FEED.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED.getName());
- }
-
-
- private void createReplicationFeedEntityClass() throws AtlasException {
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class,
- FalconDataTypes.FALCON_FEED_REPLICATION.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), null);
- classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_REPLICATION.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_REPLICATION.getName());
- }
-
- private void createProcessEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
- new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, false, true, null),
- // map of tags
- new AttributeDefinition(TAGS,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
- // wf properties
- new AttributeDefinition(WFPROPERTIES,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS.getName());
- }
-
-
- public String getModelAsJson() throws AtlasException {
- createDataModel();
- return getDataModelAsJSON();
- }
-
- public static void main(String[] args) throws Exception {
- FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator();
- String modelAsJson = falconDataModelGenerator.getModelAsJson();
-
- if (args.length == 1) {
- ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
- return;
- }
-
- System.out.println("falconDataModelAsJSON = " + modelAsJson);
-
- TypesDef typesDef = falconDataModelGenerator.getTypesDef();
- for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
- Arrays.toString(enumType.enumValues)));
- }
- for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
- System.out.println(
- String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
- Arrays.toString(structType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
- ClassType.class.getSimpleName(),
- StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
- Arrays.toString(traitType.attributeDefinitions)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
index 31b70fd..8d0a47a 100644
--- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
@@ -23,9 +23,7 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.falcon.bridge.FalconBridge;
-import org.apache.atlas.falcon.model.FalconDataModelGenerator;
import org.apache.atlas.falcon.model.FalconDataTypes;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
@@ -47,7 +45,6 @@ import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.security.CurrentUser;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.slf4j.Logger;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -84,24 +81,9 @@ public class FalconHookIT {
AtlasService service = new AtlasService();
service.init();
STORE.registerListener(service);
- registerFalconDataModel();
CurrentUser.authenticate(System.getProperty("user.name"));
}
- private void registerFalconDataModel() throws Exception {
- if (isDataModelAlreadyRegistered()) {
- LOG.info("Falcon data model is already registered!");
- return;
- }
-
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
-
- FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator();
- LOG.info("Registering Falcon data model");
- atlasClient.createType(dataModelGenerator.getModelAsJson());
- }
-
private boolean isDataModelAlreadyRegistered() throws Exception {
try {
atlasClient.getType(FalconDataTypes.FALCON_PROCESS.getName());
@@ -196,7 +178,7 @@ public class FalconHookIT {
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
Referenceable pathEntity = atlasClient.getEntity(inputId);
- assertEquals(pathEntity.getTypeName(), FSDataTypes.HDFS_PATH().toString());
+ assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH.toString());
List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA);
@@ -243,7 +225,7 @@ public class FalconHookIT {
private void assertFeedAttributes(String feedId) throws Exception {
Referenceable feedEntity = atlasClient.getEntity(feedId);
assertEquals(feedEntity.get(AtlasClient.OWNER), "testuser");
- assertEquals(feedEntity.get(FalconDataModelGenerator.FREQUENCY), "hours(1)");
+ assertEquals(feedEntity.get(FalconBridge.FREQUENCY), "hours(1)");
assertEquals(feedEntity.get(AtlasClient.DESCRIPTION), "test input");
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/pom.xml b/addons/hdfs-model/pom.xml
index e2ab029..1376fdc 100644
--- a/addons/hdfs-model/pom.xml
+++ b/addons/hdfs-model/pom.xml
@@ -180,24 +180,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.fs.model.FSDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/src/test/resources/</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/fs_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java b/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
deleted file mode 100644
index 555d565..0000000
--- a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.fs.model;
-
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-
-import java.io.IOException;
-
-public class FSDataModelGenerator {
-
- public static void main(String[] args) throws IOException {
- FSDataModel.main(args);
- TypesDef typesDef = FSDataModel.typesDef();
- String fsTypesAsJSON = TypesSerialization.toJson(typesDef);
- if (args.length == 1) {
- ModelDefinitionDump.dumpModelToFile(args[0], fsTypesAsJSON);
- return;
- }
- System.out.println("FS Data Model as JSON = " + fsTypesAsJSON);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
deleted file mode 100644
index ad40340..0000000
--- a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.fs.model
-
-import org.apache.atlas.{AtlasConstants, AtlasClient}
-import org.apache.atlas.typesystem.TypesDef
-import org.apache.atlas.typesystem.builders.TypesBuilder
-import org.apache.atlas.typesystem.json.TypesSerialization
-import org.apache.atlas.typesystem.types.DataTypes.MapType
-import org.apache.hadoop.fs.permission.FsAction
-
-import scala.tools.scalap.scalax.rules.scalasig.ClassFileParser.EnumConstValue
-
-/**
- * This represents the data model for a HDFS Path
- */
-object FSDataModel extends App {
-
- val typesBuilder = new TypesBuilder
- import typesBuilder._
-
- val typesDef : TypesDef = types {
-
- // FS DataSet
- _class(FSDataTypes.FS_PATH.toString, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- //fully qualified path/URI to the filesystem path is stored in 'qualifiedName' and 'path'.
- "path" ~ (string, required, indexed)
- "createTime" ~ (date, optional, indexed)
- "modifiedTime" ~ (date, optional, indexed)
- //Is a regular file or a directory. If true, it is a file else a directory
- "isFile" ~ (boolean, optional, indexed)
- //Is a symlink or not
- "isSymlink" ~ (boolean, optional)
- //Optional and may not be set for a directory
- "fileSize" ~ (long, optional)
- "group" ~ (string, optional, indexed)
- "posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
- }
-
- enum(FSDataTypes.FS_ACTION.toString, FsAction.values().map(x => x.name()) : _*)
-
- struct(FSDataTypes.FS_PERMISSIONS.toString) {
- PosixPermissions.PERM_USER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
- PosixPermissions.PERM_GROUP.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
- PosixPermissions.PERM_OTHER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
- PosixPermissions.STICKY_BIT.toString ~ (boolean, required, indexed)
- }
-
- //HDFS DataSet
- _class(FSDataTypes.HDFS_PATH.toString, List(FSDataTypes.FS_PATH.toString)) {
- //Making cluster optional since path is already unique containing the namenode URI
- AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
- "numberOfReplicas" ~ (int, optional, indexed)
- "extendedAttributes" ~ (map(string, string), optional)
- }
- //TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
- }
-
- // add the types to atlas
- val typesAsJSON = TypesSerialization.toJson(typesDef)
- println("FS Data Model as JSON: ")
- println(typesAsJSON)
-
-}
-
-object FSDataTypes extends Enumeration {
- type FSDataTypes = Value
- val FS_ACTION = Value("file_action")
- val FS_PATH = Value("fs_path")
- val HDFS_PATH = Value("hdfs_path")
- val FS_PERMISSIONS = Value("fs_permissions")
-}
-
-object PosixPermissions extends Enumeration {
- type PosixPermissions = Value
- val PERM_USER = Value("user")
- val PERM_GROUP = Value("group")
- val PERM_OTHER = Value("others")
- val STICKY_BIT = Value("sticky")
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java b/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
deleted file mode 100644
index 5944da8..0000000
--- a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.fs.model;
-
-import javax.inject.Inject;
-
-import org.apache.atlas.RepositoryMetadataModule;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import scala.Enumeration;
-import scala.collection.Iterator;
-
-@Test
-@Guice(modules = RepositoryMetadataModule.class)
-public class HDFSModelTest {
-
- public static final Logger LOG = LoggerFactory.getLogger(HDFSModelTest.class);
- private static final String ATLAS_URL = "http://localhost:21000/";
-
- @Inject
- private MetadataService metadataService;
-
- @BeforeClass
- public void setUp() throws Exception {
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- TypeSystem.getInstance().reset();
- AtlasGraphProvider.cleanup();
- }
-
- @Test
- public void testCreateDataModel() throws Exception {
- FSDataModel.main(new String[]{});
- TypesDef fsTypesDef = FSDataModel.typesDef();
-
- String fsTypesAsJSON = TypesSerialization.toJson(fsTypesDef);
- LOG.info("fsTypesAsJSON = {}", fsTypesAsJSON);
-
- metadataService.createType(fsTypesAsJSON);
-
- // verify types are registered
- final Iterator<Enumeration.Value> valueIterator = FSDataTypes.values().iterator();
- while (valueIterator.hasNext() ) {
- final Enumeration.Value typeEnum = valueIterator.next();
- String typeDefStr = metadataService.getTypeDefinition(typeEnum.toString());
- Assert.assertNotNull(typeDefStr);
-
- TypesDef typesDef = TypesSerialization.fromJson(typeDefStr);
- Assert.assertNotNull(typesDef);
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 884791d..7ab47f1 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -399,24 +399,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.hive.model.HiveDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/hive_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
index e4a20e1..c54fdb3 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
@@ -19,8 +19,6 @@
package org.apache.atlas.hive.bridge;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.hive.hook.HiveHook;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
@@ -31,7 +29,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
public class ColumnLineageUtils {
public static final Logger LOG = LoggerFactory.getLogger(ColumnLineageUtils.class);
@@ -92,7 +89,7 @@ public class ColumnLineageUtils {
if (r.getTypeName().equals(HiveDataTypes.HIVE_TABLE.getName())) {
String qName = (String) r.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
String[] qNameComps = extractComponents(qName);
- for (Referenceable col : (List<Referenceable>) r.get(HiveDataModelGenerator.COLUMNS)) {
+ for (Referenceable col : (List<Referenceable>) r.get(HiveMetaStoreBridge.COLUMNS)) {
String cName = (String) col.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
String[] colQNameComps = extractComponents(cName);
String colQName = colQNameComps[0] + "." + colQNameComps[1] + "." + colQNameComps[2];
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 2d2039b..cbc51cc 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -26,15 +26,11 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.fs.model.FSDataModel;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.hook.HiveHook;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.json.InstanceSerialization;
-import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.cli.BasicParser;
@@ -79,6 +75,25 @@ public class HiveMetaStoreBridge {
public static final String ATLAS_ENDPOINT = "atlas.rest.address";
+ public static final String COMMENT = "comment";
+ public static final String PARAMETERS = "parameters";
+ public static final String COLUMNS = "columns";
+ public static final String POSITION = "position";
+ public static final String PART_COLS = "partitionKeys";
+ public static final String TABLE_ALIAS_LIST = "aliases";
+ public static final String STORAGE_NUM_BUCKETS = "numBuckets";
+ public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
+ public static final String TABLE = "table";
+ public static final String DB = "db";
+ public static final String STORAGE_DESC = "sd";
+ public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
+ public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
+ public static final String LOCATION = "location";
+ public static final String TABLE_TYPE_ATTR = "tableType";
+ public static final String CREATE_TIME = "createTime";
+ public static final String LAST_ACCESS_TIME = "lastAccessTime";
+ public static final String HDFS_PATH = "hdfs_path";
+
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
public final Hive hiveClient;
@@ -174,8 +189,8 @@ public class HiveMetaStoreBridge {
dbRef.set(AtlasClient.NAME, dbName);
dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription());
- dbRef.set(HiveDataModelGenerator.LOCATION, hiveDB.getLocationUri());
- dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters());
+ dbRef.set(LOCATION, hiveDB.getLocationUri());
+ dbRef.set(PARAMETERS, hiveDB.getParameters());
dbRef.set(AtlasClient.OWNER, hiveDB.getOwnerName());
if (hiveDB.getOwnerType() != null) {
dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
@@ -431,7 +446,7 @@ public class HiveMetaStoreBridge {
try {
createDate = getTableCreatedTime(hiveTable);
LOG.debug("Setting create time to {} ", createDate);
- tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate);
+ tableReference.set(CREATE_TIME, createDate);
} catch(Exception ne) {
LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne);
}
@@ -441,19 +456,19 @@ public class HiveMetaStoreBridge {
if ( hiveTable.getLastAccessTime() > 0) {
lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR);
}
- tableReference.set(HiveDataModelGenerator.LAST_ACCESS_TIME, lastAccessTime);
+ tableReference.set(LAST_ACCESS_TIME, lastAccessTime);
tableReference.set("retention", hiveTable.getRetention());
- tableReference.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT));
+ tableReference.set(COMMENT, hiveTable.getParameters().get(COMMENT));
// add reference to the database
- tableReference.set(HiveDataModelGenerator.DB, dbReference);
+ tableReference.set(DB, dbReference);
// add reference to the StorageDescriptor
Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId());
- tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReferenceable);
+ tableReference.set(STORAGE_DESC, sdReferenceable);
- tableReference.set(HiveDataModelGenerator.PARAMETERS, hiveTable.getParameters());
+ tableReference.set(PARAMETERS, hiveTable.getParameters());
if (hiveTable.getViewOriginalText() != null) {
tableReference.set("viewOriginalText", hiveTable.getViewOriginalText());
@@ -463,14 +478,14 @@ public class HiveMetaStoreBridge {
tableReference.set("viewExpandedText", hiveTable.getViewExpandedText());
}
- tableReference.set(HiveDataModelGenerator.TABLE_TYPE_ATTR, hiveTable.getTableType().name());
+ tableReference.set(TABLE_TYPE_ATTR, hiveTable.getTableType().name());
tableReference.set("temporary", hiveTable.isTemporary());
// add reference to the Partition Keys
List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference);
tableReference.set("partitionKeys", partKeys);
- tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableReference));
+ tableReference.set(COLUMNS, getColumns(hiveTable.getCols(), tableReference));
return tableReference;
}
@@ -523,12 +538,12 @@ public class HiveMetaStoreBridge {
serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName());
serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
- serdeInfoStruct.set(HiveDataModelGenerator.PARAMETERS, serdeInfo.getParameters());
+ serdeInfoStruct.set(PARAMETERS, serdeInfo.getParameters());
sdReferenceable.set("serdeInfo", serdeInfoStruct);
- sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
+ sdReferenceable.set(STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
sdReferenceable
- .set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
+ .set(STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
List<Struct> sortColsStruct = new ArrayList<>();
for (Order sortcol : storageDesc.getSortCols()) {
@@ -543,7 +558,7 @@ public class HiveMetaStoreBridge {
sdReferenceable.set("sortCols", sortColsStruct);
}
- sdReferenceable.set(HiveDataModelGenerator.LOCATION, storageDesc.getLocation());
+ sdReferenceable.set(LOCATION, storageDesc.getLocation());
sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
sdReferenceable.set("compressed", storageDesc.isCompressed());
@@ -552,15 +567,15 @@ public class HiveMetaStoreBridge {
sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
}
- sdReferenceable.set(HiveDataModelGenerator.PARAMETERS, storageDesc.getParameters());
+ sdReferenceable.set(PARAMETERS, storageDesc.getParameters());
sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
- sdReferenceable.set(HiveDataModelGenerator.TABLE, tableId);
+ sdReferenceable.set(TABLE, tableId);
return sdReferenceable;
}
public Referenceable fillHDFSDataSet(String pathUri) {
- Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+ Referenceable ref = new Referenceable(HDFS_PATH.toString());
ref.set("path", pathUri);
Path path = new Path(pathUri);
ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
@@ -586,9 +601,9 @@ public class HiveMetaStoreBridge {
colReferenceable.set(AtlasClient.NAME, fs.getName());
colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER));
colReferenceable.set("type", fs.getType());
- colReferenceable.set(HiveDataModelGenerator.POSITION, columnPosition++);
- colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment());
- colReferenceable.set(HiveDataModelGenerator.TABLE, tableReference.getId());
+ colReferenceable.set(POSITION, columnPosition++);
+ colReferenceable.set(COMMENT, fs.getComment());
+ colReferenceable.set(TABLE, tableReference.getId());
colList.add(colReferenceable);
@@ -596,43 +611,6 @@ public class HiveMetaStoreBridge {
return colList;
}
- /**
- * Register the Hive DataModel in Atlas, if not already defined.
- *
- * The method checks for the presence of the type {@link HiveDataTypes#HIVE_PROCESS} with the Atlas server.
- * If this type is defined, then we assume the Hive DataModel is registered.
- * @throws Exception
- */
- public synchronized void registerHiveDataModel() throws Exception {
- HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
- AtlasClient dgiClient = getAtlasClient();
-
- try {
- dgiClient.getType(FSDataTypes.HDFS_PATH().toString());
- LOG.info("HDFS data model is already registered!");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Trigger val definition
- FSDataModel.main(null);
-
- final String hdfsModelJson = TypesSerialization.toJson(FSDataModel.typesDef());
- //Expected in case types do not exist
- LOG.info("Registering HDFS data model : " + hdfsModelJson);
- dgiClient.createType(hdfsModelJson);
- }
- }
-
- try {
- dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
- LOG.info("Hive data model is already registered!");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Expected in case types do not exist
- LOG.info("Registering Hive data model");
- dgiClient.createType(dataModelGenerator.getModelAsJson());
- }
- }
- }
public static void main(String[] args) throws Exception {
@@ -661,7 +639,6 @@ public class HiveMetaStoreBridge {
}
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
hiveMetaStoreBridge.importHiveMetadata(failOnError);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index eaef337..0c6582b 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -25,7 +25,6 @@ import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.bridge.ColumnLineageUtils;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.notification.hook.HookNotification;
@@ -411,10 +410,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
Referenceable tableEntity = tables.get(Type.TABLE);
//Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
- replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.COLUMNS), oldQualifiedName, newQualifiedName);
+ replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.COLUMNS), oldQualifiedName, newQualifiedName);
//Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
- replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.PART_COLS), oldQualifiedName, newQualifiedName);
+ replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.PART_COLS), oldQualifiedName, newQualifiedName);
//Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits
replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName);
@@ -437,7 +436,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
ArrayList<String> alias_list = new ArrayList<>();
alias_list.add(oldTable.getTableName().toLowerCase());
- newEntity.set(HiveDataModelGenerator.TABLE_ALIAS_LIST, alias_list);
+ newEntity.set(HiveMetaStoreBridge.TABLE_ALIAS_LIST, alias_list);
event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(),
HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
oldTableQFName, newEntity));
@@ -466,7 +465,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private Referenceable replaceSDQFName(final HiveEventContext event, Referenceable tableEntity, final String oldTblQFName, final String newTblQFName) {
//Reset storage desc QF Name to old Name
- final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveDataModelGenerator.STORAGE_DESC));
+ final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveMetaStoreBridge.STORAGE_DESC));
sdRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getStorageDescQFName(oldTblQFName));
//Replace SD QF name first to retain tags
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
deleted file mode 100755
index d0fc43f..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.hive.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasConstants;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.EnumValue;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates hive data model for both metastore entities and DDL/DML queries.
- */
-public class HiveDataModelGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
-
- private static final DataTypes.MapType STRING_MAP_TYPE =
- new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
-
- private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
- private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
- private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
-
- public static final String COMMENT = "comment";
- public static final String PARAMETERS = "parameters";
- public static final String COLUMNS = "columns";
- public static final String POSITION = "position";
- public static final String PART_COLS = "partitionKeys";
- public static final String TABLE_ALIAS_LIST = "aliases";
-
- public static final String STORAGE_NUM_BUCKETS = "numBuckets";
- public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
-
- public static final String TABLE = "table";
- public static final String DB = "db";
-
- public static final String STORAGE_DESC = "sd";
- public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
- public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
- public static final String LOCATION = "location";
-
- public static final String TABLE_TYPE_ATTR = "tableType";
-
- public static final String CREATE_TIME = "createTime";
- public static final String LAST_ACCESS_TIME = "lastAccessTime";
-
- public HiveDataModelGenerator() {
- classTypeDefinitions = new HashMap<>();
- enumTypeDefinitionMap = new HashMap<>();
- structTypeDefinitionMap = new HashMap<>();
- }
-
- public void createDataModel() throws AtlasException {
- LOG.info("Generating the Hive Data Model....");
-
- // enums
- createHivePrincipalTypeEnum();
- // structs
- createSerDeStruct();
- createOrderStruct();
- createStorageDescClass();
-
- // classes
- createDBClass();
- createColumnClass();
- createTableClass();
-
- // DDL/DML Process
- createProcessClass();
- createColumnLineageClass();
- }
-
- public TypesDef getTypesDef() {
- return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
- getClassTypeDefinitions());
- }
-
- public String getDataModelAsJSON() {
- return TypesSerialization.toJson(getTypesDef());
- }
-
- public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
- return ImmutableList.copyOf(enumTypeDefinitionMap.values());
- }
-
- public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
- return ImmutableList.copyOf(structTypeDefinitionMap.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
- return ImmutableList.copyOf(classTypeDefinitions.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
- return ImmutableList.of();
- }
-
- private void createHivePrincipalTypeEnum() throws AtlasException {
- EnumValue values[] = {new EnumValue("USER", 1), new EnumValue("ROLE", 2), new EnumValue("GROUP", 3),};
-
- EnumTypeDefinition definition = new EnumTypeDefinition(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
-
- enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
- }
-
- private void createSerDeStruct() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),
- new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
- StructTypeDefinition definition =
- new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
- }
-
- private void createOrderStruct() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("order", DataTypes.INT_TYPE.getName(), Multiplicity.REQUIRED, false, null),};
-
- StructTypeDefinition definition =
- new StructTypeDefinition(HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
- }
-
- private void createStorageDescClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- //Optional to keep it backward-compatible
- new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false,
- STORAGE_DESC),
- new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(), Multiplicity.OPTIONAL, true,
- null),
- new AttributeDefinition("bucketCols", String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("sortCols", String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
- // Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
- }
-
- /** Revisit later after nested array types are handled by the typesystem **/
-
- private void createDBClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
- }
-
- private void createColumnClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- //Making this optional since this is an incompatible change
- //Reverse attribute to 'columns' in Table
- new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false, COLUMNS),
- new AttributeDefinition(POSITION, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, false, false, null)};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, "1.1",
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
- }
-
- private void createTableClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(CREATE_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition(LAST_ACCESS_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(STORAGE_DESC, HiveDataTypes.HIVE_STORAGEDESC.getName(), Multiplicity.OPTIONAL, true,
- null),
- new AttributeDefinition(PART_COLS, DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition(TABLE_ALIAS_LIST, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition("columns", DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),
- new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),
- new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
- false, true, null),};
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
- }
-
- private void createProcessClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("recentQueries", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
- }
-
- public String getModelAsJson() throws AtlasException {
- createDataModel();
- return getDataModelAsJSON();
- }
-
- public static void main(String[] args) throws Exception {
- HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
- String modelAsJson = hiveDataModelGenerator.getModelAsJson();
-
- if (args.length==1) {
- ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
- return;
- }
-
- System.out.println("hiveDataModelAsJSON = " + modelAsJson);
-
- TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
- for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
- Arrays.toString(enumType.enumValues)));
- }
- for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
- Arrays.toString(structType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, ClassType.class.getSimpleName(),
- StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
- Arrays.toString(traitType.attributeDefinitions)));
- }
- }
-
- private void createColumnLineageClass() throws AtlasException {
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("query", HiveDataTypes.HIVE_PROCESS.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("depenendencyType",DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("expression",DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null)
- };
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN_LINEAGE.getName());
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index bdf0d2d..5abf2df 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -20,7 +20,6 @@ package org.apache.atlas.hive;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.hook.HiveHookIT;
import org.apache.atlas.hive.model.HiveDataTypes;
@@ -93,7 +92,6 @@ public class HiveITBase {
}
hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
HiveConf conf = new HiveConf();
conf.set("hive.exec.post.hooks", "");
@@ -232,7 +230,7 @@ public class HiveITBase {
private String assertHDFSPathIsRegistered(String path) throws Exception {
LOG.debug("Searching for hdfs path {}", path);
- return assertEntityIsRegistered(FSDataTypes.HDFS_PATH().toString(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
+ return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
}
protected String assertDatabaseIsRegistered(String dbName) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
index e488f93..0cba27e 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
@@ -20,7 +20,6 @@ package org.apache.atlas.hive.bridge;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.hadoop.fs.Path;
@@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.mapred.TextInputFormat;
-import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.mockito.ArgumentMatcher;
import org.mockito.Mock;
@@ -110,7 +108,7 @@ public class HiveMetaStoreBridgeTest {
// verify update is called on table
verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
- (Referenceable) argThat(new MatchesReferenceableProperty(HiveDataModelGenerator.TABLE_TYPE_ATTR,
+ (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
TableType.EXTERNAL_TABLE.name())));
}
@@ -228,7 +226,7 @@ public class HiveMetaStoreBridgeTest {
private Referenceable createTableReference() {
Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
- tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReference);
+ tableReference.set(HiveMetaStoreBridge.STORAGE_DESC, sdReference);
return tableReference;
}