You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2016/11/08 00:00:13 UTC
[2/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap
to load from new format typedef JSON files
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala b/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala
deleted file mode 100644
index 7caf5e8..0000000
--- a/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.storm.model
-
-import org.apache.atlas.AtlasClient
-import org.apache.atlas.typesystem.TypesDef
-import org.apache.atlas.typesystem.builders.TypesBuilder
-import org.apache.atlas.typesystem.json.TypesSerialization
-
-
-/**
- * This represents the data model for a storm topology.
- */
-object StormDataModel extends App {
-
- var typesDef : TypesDef = null
-
- val typesBuilder = new TypesBuilder
- import typesBuilder._
-
- typesDef = types {
-
- /**
- * Model is represented as:
- * Topology is a Process Super Type inheriting inputs/outputs
- * Input DataSet(s) => Topology => Output DataSet(s)
- * Also, Topology contains the Graph of Nodes
- * Topology => Node(s) -> Spouts/Bolts
- */
- _class(StormDataTypes.STORM_TOPOLOGY.getName, List(AtlasClient.PROCESS_SUPER_TYPE)) {
- "id" ~ (string, required, indexed, unique)
- "startTime" ~ date
- "endTime" ~ date
- "conf" ~ (map(string, string), optional)
- "clusterName" ~ (string, optional, indexed)
-
- // Nodes in the Graph
- "nodes" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
- }
-
- // Base class for DataProducer aka Spouts and
- // DataProcessor aka Bolts, also links from Topology
- _class(StormDataTypes.STORM_NODE.getName) {
- "name" ~ (string, required, indexed)
- "description" ~ (string, optional, indexed)
- // fully qualified driver java class name
- "driverClass" ~ (string, required, indexed)
- // spout or bolt configuration NVPs
- "conf" ~ (map(string, string), optional)
- }
-
- // Data Producer and hence only outputs
- _class(StormDataTypes.STORM_SPOUT.getName, List(StormDataTypes.STORM_NODE.getName)) {
- // "outputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
- "outputs" ~ (array(string), collection)
- }
-
- // Data Processor and hence both inputs and outputs (inherited from Spout)
- _class(StormDataTypes.STORM_BOLT.getName, List(StormDataTypes.STORM_NODE.getName)) {
- // "inputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
- "inputs" ~ (array(string), collection)
- "outputs" ~ (array(string), collection, optional)
- }
-
- // Kafka Data Set
- _class(StormDataTypes.KAFKA_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- "topic" ~ (string, required, unique, indexed)
- "uri" ~ (string, required)
- }
-
- // JMS Data Set
- _class(StormDataTypes.JMS_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- "topic" ~ (string, required, unique, indexed)
- "uri" ~ (string, required)
- }
-
- // HBase Data Set
- _class(StormDataTypes.HBASE_TABLE.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- "uri" ~ (string, required)
- }
- // Hive table data set already exists in atlas.
- }
-
- // add the types to atlas
- val typesAsJSON = TypesSerialization.toJson(typesDef)
- println("Storm Data Model as JSON: ")
- println(typesAsJSON)
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
index fe4c001..e0800b8 100644
--- a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
+++ b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
@@ -18,19 +18,11 @@
package org.apache.atlas.storm.hook;
-import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
-import org.apache.atlas.hive.model.HiveDataTypes;
-import org.apache.atlas.storm.model.StormDataModel;
import org.apache.atlas.storm.model.StormDataTypes;
import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration;
import org.apache.storm.ILocalCluster;
@@ -67,37 +59,6 @@ public class StormAtlasHookIT {
} else {
atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
}
- registerDataModel(new HiveDataModelGenerator());
- }
-
- private void registerDataModel(HiveDataModelGenerator dataModelGenerator) throws AtlasException,
- AtlasServiceException {
- try {
- atlasClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
- LOG.info("Hive data model is already registered! Going ahead with registration of Storm Data model");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Expected in case types do not exist
- LOG.info("Registering Hive data model");
- atlasClient.createType(dataModelGenerator.getModelAsJson());
- } else {
- throw ase;
- }
- }
-
-
- try {
- atlasClient.getType(StormDataTypes.STORM_TOPOLOGY.getName());
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- LOG.info("Registering Storm/Kafka data model");
- StormDataModel.main(new String[]{});
- TypesDef typesDef = StormDataModel.typesDef();
- String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
- LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON);
- atlasClient.createType(stormTypesAsJSON);
- }
- }
}
@@ -109,23 +70,6 @@ public class StormAtlasHookIT {
atlasClient = null;
}
- @Test
- public void testCreateDataModel() throws Exception {
- StormDataModel.main(new String[]{});
- TypesDef stormTypesDef = StormDataModel.typesDef();
-
- String stormTypesAsJSON = TypesSerialization.toJson(stormTypesDef);
- LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON);
-
- registerDataModel(new HiveDataModelGenerator());
-
- // verify types are registered
- for (StormDataTypes stormDataType : StormDataTypes.values()) {
- Assert.assertNotNull(atlasClient.getType(stormDataType.getName()));
- }
- }
-
- @Test (dependsOnMethods = "testCreateDataModel")
public void testAddEntities() throws Exception {
StormTopology stormTopology = StormTestUtil.createTestTopology();
StormTestUtil.submitTopology(stormCluster, TOPOLOGY_NAME, stormTopology);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/distro/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml
index 39c6512..215cb23 100755
--- a/distro/src/main/assemblies/standalone-package.xml
+++ b/distro/src/main/assemblies/standalone-package.xml
@@ -106,9 +106,9 @@
<outputDirectory>examples</outputDirectory>
</fileSet>
- <!-- addons/hdfs-model -->
+ <!-- out-of-box-models -->
<fileSet>
- <directory>../addons/hdfs-model/target/models</directory>
+ <directory>../addons/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
@@ -130,55 +130,29 @@
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/hive-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- addons/falcon -->
<fileSet>
<directory>../addons/falcon-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/falcon-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- addons/sqoop -->
<fileSet>
<directory>../addons/sqoop-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/sqoop-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- addons/storm -->
<fileSet>
<directory>../addons/storm-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/storm-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- for kafka topic setup -->
<fileSet>
<directory>../notification/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
-
- <!-- for patches -->
- <fileSet>
- <directory>../addons/hive-bridge/src/patches</directory>
- <outputDirectory>models/patches</outputDirectory>
- </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 709fcbc..fe38fba 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -53,7 +53,9 @@ public enum AtlasErrorCode {
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
- INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}")
+ INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"),
+ PATCH_NOT_APPLICABLE_FOR_TYPE(500, "ATLAS5004E", "{0} - invalid patch for type {1}"),
+ PATCH_FOR_UNKNOWN_TYPE(500, "ATLAS5005E", "{0} - patch references unknown type {1}")
;
private String errorCode;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
index d5ed0bc..7421da8 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
@@ -279,12 +279,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
List<AtlasConstraintDef> constraintDefs) {
setName(name);
setTypeName(typeName);
- setOptional(isOptional);
+ setIsOptional(isOptional);
setCardinality(cardinality);
setValuesMinCount(valuesMinCount);
setValuesMaxCount(valuesMaxCount);
- setUnique(isUnique);
- setIndexable(isIndexable);
+ setIsUnique(isUnique);
+ setIsIndexable(isIndexable);
setConstraintDefs(constraintDefs);
}
@@ -292,12 +292,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
if (other != null) {
setName(other.getName());
setTypeName(other.getTypeName());
- setOptional(other.isOptional());
+ setIsOptional(other.getIsOptional());
setCardinality(other.getCardinality());
setValuesMinCount(other.getValuesMinCount());
setValuesMaxCount(other.getValuesMaxCount());
- setUnique(other.isUnique());
- setIndexable(other.isIndexable());
+ setIsUnique(other.getIsUnique());
+ setIsIndexable(other.getIsIndexable());
setConstraintDefs(other.getConstraintDefs());
}
}
@@ -318,11 +318,11 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.typeName = typeName;
}
- public boolean isOptional() {
+ public boolean getIsOptional() {
return isOptional;
}
- public void setOptional(boolean optional) { isOptional = optional; }
+ public void setIsOptional(boolean optional) { isOptional = optional; }
public void setCardinality(Cardinality cardinality) {
this.cardinality = cardinality;
@@ -348,19 +348,19 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.valuesMaxCount = valuesMaxCount;
}
- public boolean isUnique() {
+ public boolean getIsUnique() {
return isUnique;
}
- public void setUnique(boolean unique) {
+ public void setIsUnique(boolean unique) {
isUnique = unique;
}
- public boolean isIndexable() {
+ public boolean getIsIndexable() {
return isIndexable;
}
- public void setIndexable(boolean idexable) {
+ public void setIsIndexable(boolean idexable) {
isIndexable = idexable;
}
@@ -399,7 +399,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append("AtlasAttributeDef{");
sb.append("name='").append(name).append('\'');
sb.append(", typeName='").append(typeName).append('\'');
- sb.append(", isOptional=").append(isOptional);
+ sb.append(", getIsOptional=").append(isOptional);
sb.append(", cardinality=").append(cardinality);
sb.append(", valuesMinCount=").append(valuesMinCount);
sb.append(", valuesMaxCount=").append(valuesMaxCount);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
index fb2029d..899e53f 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import org.apache.commons.collections.CollectionUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -91,6 +92,7 @@ public class AtlasTypesDef {
this.entityDefs = entityDefs;
}
+ @JsonIgnore
public boolean isEmpty() {
return CollectionUtils.isEmpty(enumDefs) &&
CollectionUtils.isEmpty(structDefs) &&
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 78b29fd..c8c2216 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -198,7 +198,7 @@ public class AtlasStructType extends AtlasType {
if (value != null) {
ret = dataType.validateValue(value, fieldName, messages) && ret;
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
@@ -218,7 +218,7 @@ public class AtlasStructType extends AtlasType {
if (value != null) {
ret = dataType.validateValue(value, fieldName, messages) && ret;
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
@@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType {
Object attributeValue = getNormalizedValue(obj.getAttribute(attributeName), attributeDef);
obj.setAttribute(attributeName, attributeValue);
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
obj.setAttribute(attributeName, createDefaultValue(attributeDef));
}
}
@@ -260,7 +260,7 @@ public class AtlasStructType extends AtlasType {
Object attributeValue = getNormalizedValue(obj.get(attributeName), attributeDef);
obj.put(attributeName, attributeValue);
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
obj.put(attributeName, createDefaultValue(attributeDef));
}
}
@@ -276,7 +276,7 @@ public class AtlasStructType extends AtlasType {
}
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
- if (!attributeDef.isOptional()) {
+ if (!attributeDef.getIsOptional()) {
attributes.put(attributeDef.getName(), createDefaultValue(attributeDef));
}
}
@@ -310,7 +310,7 @@ public class AtlasStructType extends AtlasType {
ret = false; // invalid value
}
}
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
ret = false; // mandatory attribute not present
}
@@ -322,7 +322,7 @@ public class AtlasStructType extends AtlasType {
if (attrType != null) {
if (value == null) {
- if (!attributeDef.isOptional()) {
+ if (!attributeDef.getIsOptional()) {
return attrType.createDefaultValue();
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 8924d44..0c118f0 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -64,6 +64,10 @@ public class AtlasTypeRegistry {
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
+ public boolean isRegisteredType(String typeName) {
+ return registryData.allTypes.isKnownType(typeName);
+ }
+
public AtlasType getType(String typeName) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.getType({})", typeName);
@@ -677,6 +681,10 @@ class TypeCache {
}
}
+ public boolean isKnownType(String typeName) {
+ return typeNameMap.containsKey(typeName);
+ }
+
public Collection<String> getAllTypeNames() {
return Collections.unmodifiableCollection(typeNameMap.keySet());
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
index cd44318..aabf269 100755
--- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
+++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
@@ -44,9 +44,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service;
import org.apache.atlas.services.DefaultMetadataService;
-import org.apache.atlas.services.IBootstrapTypesRegistrar;
import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.TypeSystem;
@@ -92,8 +90,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the MetadataService interface to an implementation
bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
- bind(IBootstrapTypesRegistrar.class).to(ReservedTypesRegistrar.class);
-
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 67b5362..aea54fa 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -267,8 +267,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
- boolean isUnique = attributeDef.isUnique();
- boolean isIndexable = attributeDef.isIndexable();
+ boolean isUnique = attributeDef.getIsUnique();
+ boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
new file mode 100644
index 0000000..e52ac6d
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.bootstrap;
+
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
+import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+
+/**
+ * Class that handles initial loading of models and patches into typedef store
+ */
+public class AtlasTypeDefStoreInitializer {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
+
+ public void initializeStore(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
+ File typesDir = new File(typesDirName);
+ File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
+
+ if (typeDefFiles == null || typeDefFiles.length == 0) {
+ LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName);
+
+ return;
+ }
+
+ // sort the files by filename
+ Arrays.sort(typeDefFiles);
+
+ for (File typeDefFile : typeDefFiles) {
+ if (!typeDefFile.isFile()) {
+ continue;
+ }
+
+ try {
+ String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+
+ if (typesDef == null || typesDef.isEmpty()) {
+ LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
+
+ continue;
+ }
+
+ AtlasTypesDef typesToCreate = new AtlasTypesDef();
+
+ if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
+ for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
+ if (!typeRegistry.isRegisteredType(enumDef.getName())) {
+ typesToCreate.getEnumDefs().add(enumDef);
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
+ for (AtlasStructDef structDef : typesDef.getStructDefs()) {
+ if (!typeRegistry.isRegisteredType(structDef.getName())) {
+ typesToCreate.getStructDefs().add(structDef);
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
+ for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
+ if (!typeRegistry.isRegisteredType(classificationDef.getName())) {
+ typesToCreate.getClassificationDefs().add(classificationDef);
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
+ for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
+ if (!typeRegistry.isRegisteredType(entityDef.getName())) {
+ typesToCreate.getEntityDefs().add(entityDef);
+ }
+ }
+ }
+
+ if (typesToCreate.isEmpty()) {
+ LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
+
+ continue;
+ }
+
+ LOG.info("Loading types defined in file {}", typeDefFile.getAbsolutePath());
+
+ typeDefStore.createTypesDef(typesDef);
+ } catch (Throwable t) {
+ LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), t);
+ }
+ }
+
+ applyTypePatches(typeDefStore, typeRegistry, typesDirName);
+ }
+
+ private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
+ String typePatchesDirName = typesDirName + File.separator + "patches";
+ File typePatchesDir = new File(typePatchesDirName);
+ File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null;
+
+ if (typePatchFiles == null || typePatchFiles.length == 0) {
+ LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName);
+
+ return;
+ }
+
+ // sort the files by filename
+ Arrays.sort(typePatchFiles);
+
+ PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry) };
+
+ Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
+
+ for (PatchHandler patchHandler : patchHandlers) {
+ for (String supportedAction : patchHandler.getSupportedActions()) {
+ patchHandlerRegistry.put(supportedAction, patchHandler);
+ }
+ }
+
+ for (File typePatchFile : typePatchFiles) {
+ if (!typePatchFile.isFile()) {
+ continue;
+ }
+
+ try {
+ String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
+ TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
+
+ if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
+ LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
+
+ continue;
+ }
+
+ for (TypeDefPatch patch : patches.getPatches()) {
+ PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
+
+ if (patchHandler == null) {
+ LOG.error("Unknown patch action {} in file {}. Ignored",
+ patch.getAction(), typePatchFile.getAbsolutePath());
+
+ continue;
+ }
+
+ try {
+ patchHandler.applyPatch(patch);
+ } catch (AtlasBaseException excp) {
+ LOG.error("Failed to apply " + patch.getAction() + " patch in file " +
+ typePatchFile.getAbsolutePath() + ". Ignored", excp);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to apply patches in file " + typePatchFile.getAbsolutePath() + ". Ignored", t);
+ }
+ }
+ }
+
+ /**
+ * typedef patch details
+ */
+ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @XmlRootElement
+ @XmlAccessorType(XmlAccessType.PROPERTY)
+ static class TypeDefPatch {
+ private String action;
+ private String typeName;
+ private String applyToVersion;
+ private String updateToVersion;
+ private Map<String, Object> params;
+ private List<AtlasAttributeDef> attributeDefs;
+
+ public String getAction() {
+ return action;
+ }
+
+ public void setAction(String action) {
+ this.action = action;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+
+ public String getApplyToVersion() {
+ return applyToVersion;
+ }
+
+ public void setApplyToVersion(String applyToVersion) {
+ this.applyToVersion = applyToVersion;
+ }
+
+ public String getUpdateToVersion() {
+ return updateToVersion;
+ }
+
+ public void setUpdateToVersion(String updateToVersion) {
+ this.updateToVersion = updateToVersion;
+ }
+
+ public Map<String, Object> getParams() {
+ return params;
+ }
+
+ public void setParams(Map<String, Object> params) {
+ this.params = params;
+ }
+
+ public List<AtlasAttributeDef> getAttributeDefs() {
+ return attributeDefs;
+ }
+
+ public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) {
+ this.attributeDefs = attributeDefs;
+ }
+ }
+
+ /**
+ * list of typedef patches
+ */
+ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @XmlRootElement
+ @XmlAccessorType(XmlAccessType.PROPERTY)
+ static class TypeDefPatches {
+ private List<TypeDefPatch> patches;
+
+ public List<TypeDefPatch> getPatches() {
+ return patches;
+ }
+
+ public void setPatches(List<TypeDefPatch> patches) {
+ this.patches = patches;
+ }
+ }
+
+ abstract class PatchHandler {
+ protected final AtlasTypeDefStore typeDefStore;
+ protected final AtlasTypeRegistry typeRegistry;
+ protected final String[] supportedActions;
+
+ protected PatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String[] supportedActions) {
+ this.typeDefStore = typeDefStore;
+ this.typeRegistry = typeRegistry;
+ this.supportedActions = supportedActions;
+ }
+
+ public String[] getSupportedActions() { return supportedActions; }
+
+ public abstract void applyPatch(TypeDefPatch patch) throws AtlasBaseException;
+
+ protected boolean isPatchApplicable(TypeDefPatch patch, AtlasBaseTypeDef currentTypeDef) {
+ String currentVersion = currentTypeDef.getTypeVersion();
+ String applyToVersion = patch.getApplyToVersion();
+
+ return currentVersion == null ||
+ currentVersion.equalsIgnoreCase(applyToVersion) ||
+ currentVersion.startsWith(applyToVersion + ".");
+ }
+ }
+
+ class AddAttributePatchHandler extends PatchHandler {
+ public AddAttributePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+ super(typeDefStore, typeRegistry, new String[] { "ADD_ATTRIBUTE" });
+ }
+
+ @Override
+ public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
+ String typeName = patch.getTypeName();
+
+ if (!typeRegistry.isRegisteredType(typeName)) {
+ throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
+ }
+
+ AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
+
+ if (isPatchApplicable(patch, typeDef)) {
+ if (typeDef.getClass().equals(AtlasEntityDef.class)) {
+ AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateEntityDefByName(typeName, updatedDef);
+ } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
+ AtlasClassificationDef updatedDef = new AtlasClassificationDef((AtlasClassificationDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateClassificationDefByName(typeName, updatedDef);
+ } else if (typeDef.getClass().equals(AtlasStructDef.class)) {
+ AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateStructDefByName(typeName, updatedDef);
+ } else {
+ throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
+ patch.getAction(), typeDef.getClass().getSimpleName());
+ }
+ } else {
+ LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
+ patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index 2163e01..816832b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumDefs;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -45,6 +46,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -89,6 +91,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
ttr.addTypes(typesDef);
typeRegistry.commitTransientTypeRegistry(ttr);
+
+ bootstrapTypes();
}
@Override
@@ -886,6 +890,15 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
LOG.info("Not reacting to a Passive state change");
}
+ private void bootstrapTypes() {
+ AtlasTypeDefStoreInitializer storeInitializer = new AtlasTypeDefStoreInitializer();
+
+ String atlasHomeDir = System.getProperty("atlas.home");
+ String typesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
+
+ storeInitializer.initializeStore(this, typeRegistry, typesDirName);
+ }
+
private void updateTypeRegistryPostCommit(AtlasTransientTypeRegistry ttr) {
new TypeRegistryUpdateHook(ttr);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
index 0dd7164..6dfe8cf 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
@@ -421,7 +421,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) {
// new attribute - only allow if optional
- if (!attributeDef.isOptional()) {
+ if (!attributeDef.getIsOptional()) {
throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName());
}
}
@@ -510,13 +510,30 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
attribInfo.put("name", attributeDef.getName());
attribInfo.put("dataType", attributeDef.getTypeName());
- attribInfo.put("isUnique", attributeDef.isUnique());
- attribInfo.put("isIndexable", attributeDef.isIndexable());
+ attribInfo.put("isUnique", attributeDef.getIsUnique());
+ attribInfo.put("isIndexable", attributeDef.getIsIndexable());
attribInfo.put("isComposite", isComposite);
attribInfo.put("reverseAttributeName", reverseAttribName);
+
+ final int lower;
+ final int upper;
+
+ if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) {
+ lower = attributeDef.getIsOptional() ? 0 : 1;
+ upper = 1;
+ } else {
+ if(attributeDef.getIsOptional()) {
+ lower = 0;
+ } else {
+ lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount();
+ }
+
+ upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount();
+ }
+
Map<String, Object> multiplicity = new HashMap<>();
- multiplicity.put("lower", attributeDef.getValuesMinCount());
- multiplicity.put("upper", attributeDef.getValuesMaxCount());
+ multiplicity.put("lower", lower);
+ multiplicity.put("upper", upper);
multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality()));
attribInfo.put("multiplicity", AtlasType.toJson(multiplicity));
@@ -532,8 +549,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
ret.setName((String) attribInfo.get("name"));
ret.setTypeName((String) attribInfo.get("dataType"));
- ret.setUnique((Boolean) attribInfo.get("isUnique"));
- ret.setIndexable((Boolean) attribInfo.get("isIndexable"));
+ ret.setIsUnique((Boolean) attribInfo.get("isUnique"));
+ ret.setIsIndexable((Boolean) attribInfo.get("isIndexable"));
String attrTypeName = ret.getTypeName();
@@ -608,10 +625,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
Boolean isUnique = (Boolean) multiplicity.get("isUnique");
if (minCount == null || minCount.intValue() == 0) {
- ret.setOptional(true);
+ ret.setIsOptional(true);
ret.setValuesMinCount(0);
} else {
- ret.setOptional(false);
+ ret.setIsOptional(false);
ret.setValuesMinCount(minCount.intValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java b/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java
deleted file mode 100644
index f916169..0000000
--- a/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-
-import com.google.gson.FieldNamingPolicy;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonSyntaxException;
-import org.apache.atlas.services.AtlasTypePatch.PatchContent;
-import org.apache.atlas.services.AtlasTypePatch.PatchData;
-import org.apache.atlas.services.AtlasTypePatch.PatchResult;
-import org.apache.atlas.services.AtlasTypePatch.PatchStatus;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUpdateException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class AtlasPatchHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchHandler.class);
-
- public static void handlePatches(DefaultMetadataService metadataService, TypeSystem typeSystem) throws TypeUpdateException {
-
- Map<String, AtlasTypePatch> patchHandlerMap = initializePatchHandlerMap(metadataService, typeSystem);
-
- if (patchHandlerMap == null || patchHandlerMap.isEmpty())
- return;
-
- String patchDirName = ReservedTypesRegistrar.getTypesDir() + File.separator + "patches";
- LOG.info("Checking for any patches to be applied to the type system in " + patchDirName);
-
- File patchDir = new File(patchDirName);
- if (!patchDir.exists()) {
- LOG.info("Patch directory {} doesn't exist, not applying any patches", patchDirName);
- return;
- }
-
- File[] patchFiles = patchDir.listFiles();
- if (patchFiles == null || patchFiles.length == 0) {
- LOG.info("No patch files found in {}, not applying any patches", patchDirName);
- return;
- }
-
- // Sort the patch files based on file name.
- Arrays.sort(patchFiles, new Comparator<File>() {
- public int compare(File f1, File f2) {
- return String.valueOf(f1.getName()).compareTo(f2.getName());
- }
- });
-
- LOG.info("Found " + patchFiles.length + " patch files to process.");
- int patchNumber = 0;
- Gson gson = initializeGson();
- AtlasTypePatch typePatch;
-
- for (File patchFile : patchFiles) {
- try {
- LOG.info("Processing patch file " + (++patchNumber) + " - " + patchFile.getAbsolutePath());
- String content = new String(Files.readAllBytes(patchFile.toPath()), StandardCharsets.UTF_8);
- PatchContent patchContent = gson.fromJson(content, PatchContent.class);
- PatchData[] patchDatas = patchContent.getPatches();
- PatchResult result;
- int patchCounter = 0;
-
- for (PatchData patch : patchDatas) {
- typePatch = patchHandlerMap.get(patch.getAction());
- if (typePatch != null) {
- result = typePatch.applyPatch(patch);
-
- if (result != null) {
- LOG.info(result.getMessage() + " Patch " + (++patchCounter) + " out of " + patchDatas.length + " processed in : " + patchFile.toPath());
- if (result.getStatus().equals(PatchStatus.FAILED)) {
- throw new TypeUpdateException(result.getMessage() + " patch " + patchNumber + " failed in :" + patchFile.getAbsolutePath());
- }
- }
- }
- }
-
- } catch (IOException e) {
- throw new TypeUpdateException("Unable to read patch file from " + patchFile.getAbsolutePath());
- } catch (JsonSyntaxException e) {
- throw new TypeUpdateException("Invalid non-parseable JSON patch file in " + patchFile.getAbsolutePath());
- }
- }
-
- LOG.info("Processed " + patchFiles.length + " patch files.");
- }
-
- private static Map<String, AtlasTypePatch> initializePatchHandlerMap(DefaultMetadataService metadataService, TypeSystem typeSystem) {
- Map<String, AtlasTypePatch> patchHandlerMap = new HashMap<String, AtlasTypePatch>();
- List<AtlasTypePatch> patchers = new ArrayList<AtlasTypePatch>();
-
- // Register new patch classes here
- patchers.add(new AtlasTypeAttributePatch(metadataService, typeSystem));
-
- for (AtlasTypePatch patcher : patchers) {
- for (String action : patcher.getSupportedActions()) {
- patchHandlerMap.put(action, patcher);
- }
- }
-
- return patchHandlerMap;
- }
-
- public static Gson initializeGson() {
- GsonBuilder gsonBuilder = new GsonBuilder();
- gsonBuilder.registerTypeAdapter(Multiplicity.class, new MultiplicityDeserializer());
- gsonBuilder.setFieldNamingPolicy(FieldNamingPolicy.IDENTITY);
- Gson gson = gsonBuilder.create();
-
- return gson;
- }
-
- static class MultiplicityDeserializer implements JsonDeserializer<Multiplicity> {
- @Override
- public Multiplicity deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
- throws JsonParseException {
- String multiplicityString = json.getAsString().toLowerCase();
- Multiplicity m = null;
- switch (multiplicityString) {
- case "optional":
- m = Multiplicity.OPTIONAL;
- break;
- case "required":
- m = Multiplicity.REQUIRED;
- break;
- case "collection":
- m = Multiplicity.COLLECTION;
- break;
- case "set":
- m = Multiplicity.SET;
- break;
- default:
- break;
- }
- return m;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java b/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java
deleted file mode 100644
index b918c87..0000000
--- a/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-
-import com.google.common.collect.ImmutableList;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.codehaus.jettison.json.JSONException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-public class AtlasTypeAttributePatch extends AtlasTypePatch {
-
- private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeAttributePatch.class);
- private static final String STRUCT_TYPE = "STRUCT";
- private static final String CLASS_TYPE = "CLASS";
- private static final String TRAIT_TYPE = "TRAIT";
- private static final String[] SUPPORTED_ACTIONS = new String[] { "ADD_ATTRIBUTE", "UPDATE_ATTRIBUTE", "DELETE_ATTRIBUTE" };
-
- public AtlasTypeAttributePatch(DefaultMetadataService metadataService, TypeSystem typeSystem) {
- super(metadataService, typeSystem, SUPPORTED_ACTIONS);
- }
-
- /********* SAMPLE PATCH DATA ***********
- *
- *
- {
- "patches": [
- {
- "action": "ADD_ATTRIBUTE",
- "typeName": "hive_column",
- "applyToVersion": "1.0",
- "updateToVersion": "2.0",
- "params": {
- "optionalParam1" : "true",
- "optionalParam2" : "false"
- },
- "attributeDefinitions": [
- {
- "name": "position",
- "dataTypeName": "string",
- "multiplicity": "optional",
- "isComposite": false,
- "isUnique": false,
- "isIndexable": false,
- "reverseAttributeName": null
- }
- ]
- }
- ]
- }
- *
- */
-
- @Override
- public PatchResult applyPatch(PatchData patchData) {
- String typeName = patchData.getTypeName();
- String applyVersion = patchData.getApplyToVersion();
- TypesDef updatedTypeDef;
- PatchResult result;
- try {
- // Check if type exists in type system
- if (checkIfTypeExists(typeName, metadataService)) {
- TypesDef typesDef = getTypeDef(typeName);
- String currentVersion = getTypeVersion(typeName);
-
- // Check version to apply patch
- if (currentVersion == null || currentVersion.equalsIgnoreCase(applyVersion) || currentVersion.startsWith(applyVersion + ".")) {
- updatedTypeDef = updateTypesDef(typesDef, patchData);
-
- if (updatedTypeDef != null) {
- metadataService.updateType(TypesSerialization.toJson(updatedTypeDef));
- LOG.info("updated " + patchData.getTypeName() + " type from version " + patchData.getApplyToVersion() + " to " + patchData.getUpdateToVersion());
- result = new PatchResult("patch applied successfully!", PatchStatus.SUCCESS);
- } else {
- LOG.error("failed to create updated typedef for type=" +typeName+ "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() );
- result = new PatchResult("patch failed!", PatchStatus.FAILED);
- }
- } else {
- LOG.info("patch skipped for " + patchData.getTypeName());
- result = new PatchResult("patch skipped!", PatchStatus.SKIPPED);
- }
-
- } else {
- LOG.error("failed to apply patch (typeName=" + typeName + "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() + "): type doesn't exist");
- result = new PatchResult("patch failed!", PatchStatus.FAILED);
- }
-
- } catch (AtlasException e) {
- LOG.error("error in updating type for " + patchData.getTypeName());
- result = new PatchResult("unable to update type", PatchStatus.FAILED);
- } catch (JSONException e) {
- LOG.error("error in updating typedef for " + patchData.getTypeName());
- result = new PatchResult("unable to update typedef", PatchStatus.FAILED);
- }
-
- return result;
- }
-
- public TypesDef updateTypesDef(TypesDef typesDef, PatchData patchData) throws AtlasException, JSONException {
- AttributeDefinition[] typeAttributes = getAttributesFromTypesDef(typesDef, patchData.getTypeName());
- AttributeDefinition[] patchAttributes = patchData.getAttributeDefinitions();
- AttributeDefinition[] newAttributes = new AttributeDefinition[0];
- String patchAction = patchData.getAction();
- TypesDef newTypeDef = null;
-
- if (patchAction != null && typeAttributes != null && patchAttributes != null) {
- switch (patchAction) {
- case "ADD_ATTRIBUTE":
- LOG.info("adding new attribute to type {}", patchData.getTypeName());
- newAttributes = addAttributes(typeAttributes, patchAttributes);
- break;
- case "DELETE_ATTRIBUTE":
- LOG.info("deleting attribute from type {}", patchData.getTypeName());
- newAttributes = deleteAttributes(typeAttributes, patchAttributes);
- break;
- case "UPDATE_ATTRIBUTE":
- LOG.info("updating attribute in type {}", patchData.getTypeName());
- newAttributes = updateAttributes(typeAttributes, patchAttributes);
- break;
- default:
- LOG.info("invalid action " + patchAction + ", supported actions: " + Arrays.toString(SUPPORTED_ACTIONS));
- break;
- }
-
- newTypeDef = createTypeDefWithNewAttributes(typesDef, patchData.getTypeName(), newAttributes, patchData.getUpdateToVersion());
- }
-
- return newTypeDef;
- }
-
- private AttributeDefinition[] addAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
- ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(Arrays.asList(typeAttributes));
- Map<String, AttributeDefinition> typeAttrs = getAttributeNamesFromDefinition(typeAttributes);
-
- for (AttributeDefinition attr : patchAttributes) {
- if (!typeAttrs.containsKey(attr.name))
- newAttrList.add(attr);
- }
-
- return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
- }
-
- private AttributeDefinition[] deleteAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
- ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
- Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
-
- for (AttributeDefinition attr : typeAttributes) {
- if (!patchAttrs.containsKey(attr.name))
- newAttrList.add(attr);
- }
-
- return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
- }
-
- private AttributeDefinition[] updateAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
- ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
- Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
- AttributeDefinition newAttr;
-
- for (AttributeDefinition attr : typeAttributes) {
- newAttr = patchAttrs.get(attr.name);
- if (patchAttrs.containsKey(attr.name) && checkIfAttributeUpdatable(attr, newAttr)) {
- newAttrList.add(newAttr);
- } else {
- newAttrList.add(attr);
- }
- }
-
- return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
- }
-
- private TypesDef createTypeDefWithNewAttributes(TypesDef typesDef, String typeName, AttributeDefinition[] newAttributes, String newVersion) throws AtlasException {
- ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
- ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
- ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
- ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
- String dataType = getDataType(typeName).toUpperCase();
- switch (dataType) {
- case STRUCT_TYPE:
- StructTypeDefinition structType = typesDef.structTypesAsJavaList().get(0);
- structs.add(new StructTypeDefinition(structType.typeName, structType.typeDescription, newVersion, newAttributes));
- break;
- case CLASS_TYPE:
- HierarchicalTypeDefinition<ClassType> classType = typesDef.classTypesAsJavaList().get(0);
- classTypes.add(new HierarchicalTypeDefinition(ClassType.class, classType.typeName, classType.typeDescription, newVersion, classType.superTypes, newAttributes));
- break;
- case TRAIT_TYPE:
- HierarchicalTypeDefinition<TraitType> traitType = typesDef.traitTypesAsJavaList().get(0);
- traits.add(new HierarchicalTypeDefinition(TraitType.class, traitType.typeName, traitType.typeDescription, newVersion, traitType.superTypes, newAttributes));
- break;
- default:
- LOG.error("unhandled datatype {} when creating new typedef", dataType);
- }
-
- return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
- }
-
- private AttributeDefinition[] getAttributesFromTypesDef(TypesDef typesDef, String typeName) throws AtlasException {
- AttributeDefinition[] typeAttributes = null;
- String dataType = getDataType(typeName).toUpperCase();
- switch (dataType) {
- case STRUCT_TYPE:
- typeAttributes = typesDef.structTypesAsJavaList().get(0).attributeDefinitions;
- break;
- case CLASS_TYPE:
- typeAttributes = typesDef.classTypesAsJavaList().get(0).attributeDefinitions;
- break;
- case TRAIT_TYPE:
- typeAttributes = typesDef.traitTypesAsJavaList().get(0).attributeDefinitions;
- break;
- default:
- LOG.error("unhandled datatype {}", dataType);
- }
-
- return typeAttributes;
- }
-
- private Map<String, AttributeDefinition> getAttributeNamesFromDefinition(AttributeDefinition[] attribDef) {
- Map<String, AttributeDefinition> attrsMap = new HashMap<String, AttributeDefinition>();
- for (AttributeDefinition attr : attribDef) {
- attrsMap.put(attr.name, attr);
- }
-
- return attrsMap;
- }
-
- private boolean checkIfAttributeUpdatable(AttributeDefinition attr, AttributeDefinition newAttr) {
- boolean result = false;
- if (!attr.equals(newAttr) && (attr.multiplicity == Multiplicity.REQUIRED
- && newAttr.multiplicity == Multiplicity.OPTIONAL)) {
- result = true;
- }
-
- return result;
- }
-
- // Returns the datatype the typename belongs to - PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT, CLASS
- private String getDataType(String typeName) throws AtlasException {
- IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
- return dataType.getTypeCategory().toString();
- }
-
- private String getTypeVersion(String typeName) throws AtlasException {
- return typeSystem.getDataType(IDataType.class, typeName).getVersion();
- }
-
- private TypesDef getTypeDef(String typeName) throws AtlasException {
- return TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
- }
-
- private static boolean checkIfTypeExists(String typeName, DefaultMetadataService metadataService) {
- boolean result = true;
- try {
- metadataService.getTypeDefinition(typeName);
- } catch (AtlasException e) {
- result = false;
- }
-
- return result;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java b/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java
deleted file mode 100644
index 0698efb..0000000
--- a/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.TypeSystem;
-
-import java.util.Map;
-
-
-public abstract class AtlasTypePatch {
- protected final TypeSystem typeSystem;
- protected final DefaultMetadataService metadataService;
- protected final String[] supportedActions;
-
- protected AtlasTypePatch(DefaultMetadataService metadataService, TypeSystem typeSystem, String[] supportedActions) {
- this.metadataService = metadataService;
- this.typeSystem = typeSystem;
- this.supportedActions = supportedActions;
- }
-
- public final String[] getSupportedActions() { return supportedActions; }
-
- public abstract PatchResult applyPatch(PatchData patch);
-
- public enum PatchStatus { SUCCESS, FAILED, SKIPPED }
-
- public class PatchResult {
- private String message;
- private PatchStatus status;
-
- public PatchResult(String message, PatchStatus status) {
- this.message = message;
- this.status = status;
- }
-
- public String getMessage() { return message; }
-
- public void setMessage(String message) { this.message = message; }
-
- public PatchStatus getStatus() { return status; }
-
- public void setStatus(PatchStatus status) { this.status = status; }
-
- }
-
- /**
- * A class to capture patch content.
- */
- public class PatchContent {
- private PatchData[] patches;
-
- public PatchData[] getPatches() {
- return patches;
- }
- }
-
- public static class PatchData {
- private String action;
- private String typeName;
- private String applyToVersion;
- private String updateToVersion;
- private Map<String, String> params;
- private AttributeDefinition[] attributeDefinitions;
-
- public PatchData(String action, String typeName, String applyToVersion, String updateToVersion, Map<String, String> params, AttributeDefinition[] attributeDefinitions) {
- this.action = action;
- this.typeName = typeName;
- this.applyToVersion = applyToVersion;
- this.updateToVersion = updateToVersion;
- this.params = params;
- this.attributeDefinitions = attributeDefinitions;
- }
-
- public String getAction() { return action; }
-
- public String getTypeName() { return typeName; }
-
- public String getApplyToVersion() { return applyToVersion; }
-
- public String getUpdateToVersion() { return updateToVersion; }
-
- public Map<String, String> getParams() { return params; }
-
- public AttributeDefinition[] getAttributeDefinitions() { return attributeDefinitions; }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 95c3dd9..69e8d12 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -20,7 +20,6 @@ package org.apache.atlas.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import org.apache.atlas.ApplicationProperties;
@@ -29,7 +28,6 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.RequestContext;
-import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.ActiveStateChangeHandler;
@@ -54,7 +52,6 @@ import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
@@ -83,9 +80,6 @@ import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
-import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
-import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
-
/**
@@ -103,35 +97,29 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private final TypeSystem typeSystem;
private final MetadataRepository repository;
private final ITypeStore typeStore;
- private IBootstrapTypesRegistrar typesRegistrar;
private final Collection<TypesChangeListener> typeChangeListeners = new LinkedHashSet<>();
private final Collection<EntityChangeListener> entityChangeListeners = new LinkedHashSet<>();
- private boolean wasInitialized = false;
-
@Inject
private EntityAuditRepository auditRepository;
@Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
- final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
throws AtlasException {
- this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
+ this(repository, typeStore, typeListenerProviders, entityListenerProviders,
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
//for testing only
public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
- final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
final TypeSystem typeSystem,
final Configuration configuration, TypeCache typeCache) throws AtlasException {
this.typeStore = typeStore;
- this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem;
/**
* Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
@@ -163,70 +151,20 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void restoreTypeSystem() throws AtlasException {
LOG.info("Restoring type system from the store");
+
TypesDef typesDef = typeStore.restore();
- if (!wasInitialized) {
- LOG.info("Initializing type system for the first time.");
- typeSystem.defineTypes(typesDef);
-
- // restore types before creating super types
- createSuperTypes();
- typesRegistrar.registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, this);
- wasInitialized = true;
- } else {
- LOG.info("Type system was already initialized, refreshing cache.");
- refreshCache(typesDef);
- }
+
+ refreshCache(typesDef);
+
LOG.info("Restored type system from the store");
}
private void refreshCache(TypesDef typesDef) throws AtlasException {
- TypeSystem.TransientTypeSystem transientTypeSystem
- = typeSystem.createTransientTypeSystem(typesDef, true);
- Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
- LOG.info("Number of types got from transient type system: " + typesAdded.size());
- typeSystem.commitTypes(typesAdded);
- }
-
- @InterfaceAudience.Private
- private void createSuperTypes() throws AtlasException {
- HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
- .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
- createType(referenceableType);
-
- HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
- .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
- new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
- createType(assetType);
-
- HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
- .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
- createType(infraType);
-
- HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
- .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
- createType(datasetType);
-
- HierarchicalTypeDefinition<ClassType> processType = TypesUtil
- .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE),
- new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null));
- createType(processType);
- }
-
- private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException {
- if (!typeSystem.isRegistered(type.typeName)) {
- TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(type));
- createType(TypesSerialization.toJson(typesDef));
+ if (typesDef != null && !typesDef.isEmpty()) {
+ TypeSystem.TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(typesDef, true);
+ Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
+ LOG.info("Number of types got from transient type system: " + typesAdded.size());
+ typeSystem.commitTypes(typesAdded);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java
deleted file mode 100644
index fce5cb3..0000000
--- a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.types.TypeSystem;
-
-public interface IBootstrapTypesRegistrar {
- void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
- throws AtlasException;
-}