You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ve...@apache.org on 2015/06/17 01:04:42 UTC
[08/58] [abbrv] incubator-atlas git commit: Refactor packages and
scripts to Atlas (cherry picked from commit 414beba)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
new file mode 100755
index 0000000..ba4782c
--- /dev/null
+++ b/typesystem/src/test/scala/org/apache/atlas/typesystem/json/TypesSerializationTest.scala
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.json
+
+import com.google.common.collect.ImmutableList
+import org.apache.atlas.typesystem.types._
+import org.junit.{Assert, Test}
+
+class TypesSerializationTest extends BaseTest with TypeHelpers {
+
+ @Test def test1: Unit = {
+
+ val ts = getTypeSystem
+
+ val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+ optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+ ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
+ )
+
+ val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE))
+ val B: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
+ val C: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
+ val D: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
+
+ defineTraits(ts, A, B, C, D)
+
+ ts.defineEnumType("HiveObjectType",
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5))
+
+ ts.defineEnumType("PrincipalType",
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3))
+
+ ts.defineEnumType("TxnState",
+ new EnumValue("COMMITTED", 1),
+ new EnumValue("ABORTED", 2),
+ new EnumValue("OPEN", 3))
+
+ ts.defineEnumType("LockLevel",
+ new EnumValue("DB", 1),
+ new EnumValue("TABLE", 2),
+ new EnumValue("PARTITION", 3))
+
+ defineClassType(ts, createClassTypeDef("t4", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("enum1", ts.getDataType(classOf[EnumType], "HiveObjectType")),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("enum2", ts.getDataType(classOf[EnumType], "PrincipalType")),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("enum3", ts.getDataType(classOf[EnumType], "TxnState")),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", ts.defineArrayType(DataTypes.INT_TYPE)),
+ optionalAttr("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ optionalAttr("enum4", ts.getDataType(classOf[EnumType], "LockLevel"))))
+
+ val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, true, "department"))
+ val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ )
+ val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
+ new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager")
+ )
+ val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
+ requiredAttr("level", DataTypes.INT_TYPE)
+ )
+ ts.defineTypes(ImmutableList.of[StructTypeDefinition],
+ ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
+ ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
+
+ val ser = TypesSerialization.toJson(ts, _ => true)
+
+ val typesDef1 = TypesSerialization.fromJson(ser)
+
+ val ts1 = TypeSystem.getInstance()
+ ts1.reset()
+
+ typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
+
+ ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
+ ImmutableList.copyOf(typesDef1.traitTypes.toArray),
+ ImmutableList.copyOf(typesDef1.classTypes.toArray)
+ )
+ val ser2 = TypesSerialization.toJson(ts1, _ => true)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+
+ Assert.assertEquals(typesDef1, typesDef2)
+ }
+
+ @Test def test2: Unit = {
+
+ val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+ optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+
+ val ser2 = TypesSerialization.toJson(sDef)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+
+ Assert.assertEquals(sDef, typesDef2.structTypes(0))
+ }
+
+ @Test def test3: Unit = {
+
+ val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
+ optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
+
+
+
+ val ser2 = TypesSerialization.toJson(sDef)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+
+ Assert.assertEquals(sDef, typesDef2.structTypes(0))
+ }
+
+ @Test def test4 : Unit = {
+
+ val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE))
+ val B: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
+ val C: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
+ val D: HierarchicalTypeDefinition[TraitType] =
+ createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
+
+ val typDefs = Seq(A,B,C,D)
+ typDefs.foreach { tDef =>
+ val ser2 = TypesSerialization.toJson(tDef, true)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+ Assert.assertEquals(tDef, typesDef2.traitTypes(0))
+
+ }
+ }
+
+ @Test def test5 : Unit = {
+ val e1 = new EnumTypeDefinition("HiveObjectType",
+ new EnumValue("GLOBAL", 1),
+ new EnumValue("DATABASE", 2),
+ new EnumValue("TABLE", 3),
+ new EnumValue("PARTITION", 4),
+ new EnumValue("COLUMN", 5))
+
+ val e2 = new EnumTypeDefinition("PrincipalType",
+ new EnumValue("USER", 1),
+ new EnumValue("ROLE", 2),
+ new EnumValue("GROUP", 3))
+
+ val e3 = new EnumTypeDefinition("TxnState",
+ new EnumValue("COMMITTED", 1),
+ new EnumValue("ABORTED", 2),
+ new EnumValue("OPEN", 3))
+
+ val e4 = new EnumTypeDefinition("LockLevel",
+ new EnumValue("DB", 1),
+ new EnumValue("TABLE", 2),
+ new EnumValue("PARTITION", 3))
+
+ val typDefs = Seq(e1,e2,e3,e4)
+ typDefs.foreach { tDef =>
+ val ser2 = TypesSerialization.toJson(tDef)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+ Assert.assertEquals(tDef, typesDef2.enumTypes(0))
+
+ }
+ }
+
+ @Test def test6 : Unit = {
+ val typDef = createClassTypeDef("t4", List(),
+ requiredAttr("a", DataTypes.INT_TYPE),
+ optionalAttr("b", DataTypes.BOOLEAN_TYPE),
+ optionalAttr("c", DataTypes.BYTE_TYPE),
+ optionalAttr("d", DataTypes.SHORT_TYPE),
+ optionalAttr("enum1", "HiveObjectType"),
+ optionalAttr("e", DataTypes.INT_TYPE),
+ optionalAttr("f", DataTypes.INT_TYPE),
+ optionalAttr("g", DataTypes.LONG_TYPE),
+ optionalAttr("enum2", "PrincipalType"),
+ optionalAttr("h", DataTypes.FLOAT_TYPE),
+ optionalAttr("i", DataTypes.DOUBLE_TYPE),
+ optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
+ optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("enum3", "TxnState"),
+ optionalAttr("l", DataTypes.DATE_TYPE),
+ optionalAttr("m", DataTypes.INT_TYPE),
+ optionalAttr("n", DataTypes.BIGDECIMAL_TYPE),
+ optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
+ optionalAttr("enum4", "LockLevel"))
+
+ val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, true, "department"))
+ val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
+ requiredAttr("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
+ new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
+ )
+ val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
+ new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
+ Multiplicity.COLLECTION, false, "manager")
+ )
+
+ val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
+ typDefs.foreach { tDef =>
+ val ser2 = TypesSerialization.toJson(tDef, false)
+ val typesDef2 = TypesSerialization.fromJson(ser2)
+ Assert.assertEquals(tDef, typesDef2.classTypes(0))
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
deleted file mode 100644
index 496ebf9..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/BuilderTest.scala
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.TypesDef
-import org.apache.hadoop.metadata.typesystem.types.TypeSystem
-import org.scalatest.{BeforeAndAfter, FunSuite}
-
-abstract class BuilderTest extends FunSuite with BeforeAndAfter {
-
- var tDef : TypesDef = null
-
- before {
- TypeSystem.getInstance().reset()
-
- val b = new TypesBuilder
- import b._
-
- tDef = types {
-
- _trait("Dimension") {}
- _trait("PII") {}
- _trait("Metric") {}
- _trait("ETL") {}
- _trait("JdbcAccess") {}
-
- _class("DB") {
- "name" ~ (string, required, indexed, unique)
- "owner" ~ (string)
- "createTime" ~ (int)
- }
-
- _class("StorageDesc") {
- "inputFormat" ~ (string, required)
- "outputFormat" ~ (string, required)
- }
-
- _class("Column") {
- "name" ~ (string, required)
- "dataType" ~ (string, required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("Table", List()) {
- "name" ~ (string, required, indexed)
- "db" ~ ("DB", required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("LoadProcess") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- "outputTable" ~ ("Table", required)
-
- }
-
- _class("View") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
deleted file mode 100644
index 227b524..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/InstanceBuilderTest.scala
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.types.{Multiplicity, ClassType, TypeSystem}
-
-
-class InstanceBuilderTest extends BuilderTest {
-
- test("test1") {
- TypeSystem.getInstance().defineTypes(tDef)
-
- val b = new InstanceBuilder
- import b._
-
- val instances = b create {
-
- val salesDB = instance("DB") { // use instance to create Referenceables. use closure to
- // set attributes of instance
- 'name ~ "Sales" // use '~' to set attributes. Use a Symbol (names starting with ') for
- // attribute names.
- 'owner ~ "John ETL"
- 'createTime ~ 1000
- }
-
- val salesFact = instance("Table") {
- 'name ~ "sales_fact"
- 'db ~ salesDB
- val sd = instance("StorageDesc") { // any valid scala allowed in closure.
- 'inputFormat ~ "TextIputFormat"
- 'outputFormat ~ "TextOutputFormat"
- }
- 'sd ~ sd // use ~ to set references, collections and maps.
- val columns = Seq(
- instance("Column") {
- 'name ~ "time_id"
- 'dataType ~ "int"
- 'sd ~ sd
- },
- instance("Column") {
- 'name ~ "product_id"
- 'dataType ~ "int"
- 'sd ~ sd
- },
- instance("Column") {
- 'name ~ "customer_id"
- 'dataType ~ "int"
- 'sd ~ sd
- },
- instance("Column", "Metric") {
- 'name ~ "sales"
- 'dataType ~ "int"
- 'sd ~ sd
- 'Metric("x") ~ 1 // use 'TraitName("attrName") to set values on traits.
- }
- )
-
- 'columns ~ columns
-
- }
-
- salesFact.sd.inputFormat ~ "TextInputFormat" // use dot navigation to alter attributes in the object graph.
- // here I am fixing the typo in "TextInputFormat"
- // dot navigation also works for arrays.
- // here I am fixing column(3). Metric trait has no attributes.
- val c = salesFact.columns
- c(3) = instance("Column", "Metric") {
- 'name ~ "sales"
- 'dataType ~ "int"
- 'sd ~ salesFact.sd
- }
-
- }
-
- val ts = TypeSystem.getInstance()
-
- import scala.collection.JavaConversions._
- val typedInstances = instances.map { i =>
- val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
- iTyp.convert(i, Multiplicity.REQUIRED)
- }
-
- typedInstances.foreach { i =>
- println(i)
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
deleted file mode 100644
index 73d912b..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/MultiplicityTest.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.MetadataException
-import org.apache.hadoop.metadata.typesystem.types.{Multiplicity, ClassType, TypeSystem}
-import org.scalatest.{BeforeAndAfterAll, FunSuite}
-
-
-class MultiplicityTest extends FunSuite with BeforeAndAfterAll {
-
- override def beforeAll() = {
- TypeSystem.getInstance().reset()
-
- val b = new TypesBuilder
- import b._
-
- val tDef = types {
-
- _trait("Dimension") {}
- _trait("PII") {}
- _trait("Metric") {}
- _trait("ETL") {}
- _trait("JdbcAccess") {}
-
- _class("DB") {
- "name" ~ (string, required, indexed, unique)
- "owner" ~ (string)
- "createTime" ~ (int)
- }
-
- _class("StorageDesc") {
- "inputFormat" ~ (string, required)
- "outputFormat" ~ (string, required)
- }
-
- _class("Column") {
- "name" ~ (string, required)
- "dataType" ~ (string, required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("Table", List()) {
- "name" ~ (string, required, indexed)
- "db" ~ ("DB", required)
- "sd" ~ ("StorageDesc", required)
- }
-
- _class("LoadProcess") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- "outputTable" ~ ("Table", required)
-
- }
-
- _class("View") {
- "name" ~ (string, required)
- "inputTables" ~ (array("Table"), collection)
- }
-
- _class("AT") {
- "name" ~ (string, required)
- "stringSet" ~ (array("string"), multiplicty(0, Int.MaxValue, true))
- }
- }
-
- TypeSystem.getInstance().defineTypes(tDef)
- }
-
- test("test1") {
-
- val b = new InstanceBuilder
- import b._
-
- val instances = b create {
- val a = instance("AT") { // use instance to create Referenceables. use closure to
- // set attributes of instance
- 'name ~ "A1" // use '~' to set attributes. Use a Symbol (names starting with ') for
- 'stringSet ~ Seq("a", "a")
- }
- }
-
- val ts = TypeSystem.getInstance()
- import scala.collection.JavaConversions._
- val typedInstances = instances.map { i =>
- val iTyp = ts.getDataType(classOf[ClassType], i.getTypeName)
- iTyp.convert(i, Multiplicity.REQUIRED)
- }
-
- typedInstances.foreach { i =>
- println(i)
- }
- }
-
- test("WrongMultiplicity") {
- val b = new TypesBuilder
- import b._
- val tDef = types {
- _class("Wrong") {
- "name" ~ (string, required)
- "stringSet" ~ (string, multiplicty(0, Int.MaxValue, true))
- }
- }
- val me = intercept[MetadataException] {
- TypeSystem.getInstance().defineTypes(tDef)
- }
- assert("A multiplicty of more than one requires a collection type for attribute 'stringSet'" == me.getMessage)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
deleted file mode 100644
index f1f9024..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/builders/TypesBuilderTest.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.builders
-
-import org.apache.hadoop.metadata.typesystem.json.TypesSerialization
-import org.apache.hadoop.metadata.typesystem.types.{TypeSystem, BaseTest}
-import org.junit.runner.RunWith
-import org.scalatest.{BeforeAndAfter, FunSuite}
-import org.scalatest.junit.JUnitRunner
-
-@RunWith(classOf[JUnitRunner])
-class TypesBuilderTest extends BuilderTest {
-
-
- test("test1") {
- TypeSystem.getInstance().defineTypes(tDef)
-
- println(TypesSerialization.toJson(TypeSystem.getInstance(), x => true))
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
deleted file mode 100755
index 86cdb72..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/SerializationTest.scala
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.persistence.{ReferenceableInstance, StructInstance, Id}
-import org.apache.hadoop.metadata.typesystem.types._
-import org.apache.hadoop.metadata.typesystem.types.utils.TypesUtil
-import org.apache.hadoop.metadata.typesystem.{ITypedReferenceableInstance, ITypedStruct, Referenceable, Struct}
-import org.json4s.native.JsonMethods._
-import org.json4s.native.Serialization.{write => swrite, _}
-import org.json4s.{NoTypeHints, _}
-import org.junit.{Assert, Before, Test}
-
-class SerializationTest extends BaseTest {
-
- private[metadata] var structType: StructType = null
- private[metadata] var recursiveStructType: StructType = null
-
- @Before
- override def setup {
- super.setup
- structType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_1).asInstanceOf[StructType]
- recursiveStructType = getTypeSystem.getDataType(classOf[StructType], BaseTest.STRUCT_TYPE_2).asInstanceOf[StructType]
- }
-
- @Test def test1 {
- val s: Struct = BaseTest.createStruct()
- val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
-
- Assert.assertEquals(ts.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.1, 1.1]\n\to : \t{b=2.0, a=1.0}\n}")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
-
- //Json representation
- val ser = swrite(ts)
- Assert.assertEquals(ser, "{\"$typeName$\":\"t1\",\"e\":1,\"n\":[1.1,1.1],\"h\":1.0,\"b\":true,\"k\":1,\"j\":1,\"d\":2,\"m\":[1,1],\"g\":1,\"a\":1,\"i\":1.0,\"c\":1,\"l\":\"" + BaseTest.TEST_DATE + "\",\"f\":1,\"o\":{\"b\":2.0,\"a\":1.0}}")
-
- // Typed Struct read back
- val ts1 = read[StructInstance](ser)
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t" + BaseTest.TEST_DATE + "\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
- }
-
- @Test def test2 {
- val s: Struct = BaseTest.createStruct()
- val ts: ITypedStruct = structType.convert(s, Multiplicity.REQUIRED)
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
-
- val ts1 = read[StructInstance](
- """
- {"$typeName$":"t1","e":1,"n":[1.1,1.1],"h":1.0,"b":true,"k":1,"j":1,"d":2,"m":[1,1],"g":1,"a":1,"i":1.0,
- "c":1,"l":"2014-12-03T19:38:55.053Z","f":1,"o":{"b":2.0,"a":1.0}}""")
- // Typed Struct read from string
- Assert.assertEquals(ts1.toString, "{\n\ta : \t1\n\tb : \ttrue\n\tc : \t1\n\td : \t2\n\te : \t1\n\tf : \t1\n\tg : \t1\n\th : \t1.0\n\ti : \t1.0\n\tj : \t1\n\tk : \t1\n\tl : \t2014-12-03T19:38:55.053Z\n\tm : \t[1, 1]\n\tn : \t[1.100000000000000088817841970012523233890533447265625, 1.100000000000000088817841970012523233890533447265625]\n\to : \t{b=2.0, a=1.0}\n}")
- }
-
- @Test def testTrait {
- val A: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef("A", null,
- TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
- val B: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
- "B", ImmutableList.of[String]("A"),
- TypesUtil.createOptionalAttrDef("b", DataTypes.BOOLEAN_TYPE))
- val C: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
- "C", ImmutableList.of[String]("A"),
- TypesUtil.createOptionalAttrDef("c", DataTypes.BYTE_TYPE))
- val D: HierarchicalTypeDefinition[TraitType] = TypesUtil.createTraitTypeDef(
- "D", ImmutableList.of[String]("B", "C"),
- TypesUtil.createOptionalAttrDef("d", DataTypes.SHORT_TYPE))
-
- defineTraits(A, B, C, D)
-
- val DType: TraitType = getTypeSystem.getDataType(classOf[TraitType], "D").asInstanceOf[TraitType]
- val s1: Struct = new Struct("D")
- s1.set("d", 1)
- s1.set("c", 1)
- s1.set("b", true)
- s1.set("a", 1)
- s1.set("A.B.D.b", true)
- s1.set("A.B.D.c", 2)
- s1.set("A.B.D.d", 2)
- s1.set("A.C.D.a", 3)
- s1.set("A.C.D.b", false)
- s1.set("A.C.D.c", 3)
- s1.set("A.C.D.d", 3)
-
- val s: Struct = BaseTest.createStruct()
- val ts: ITypedStruct = DType.convert(s1, Multiplicity.REQUIRED)
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new BigDecimalSerializer + new BigIntegerSerializer
-
- // Typed Struct :
- Assert.assertEquals(ts.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
-
- // Json representation :
- val ser = swrite(ts)
- Assert.assertEquals(ser, "{\"$typeName$\":\"D\",\"A.C.D.d\":3,\"A.B.D.c\":2,\"b\":true,\"A.C.D.c\":3,\"d\":1,\"A.B.D.b\":true,\"a\":1,\"A.C.D.b\":false,\"A.B.D.d\":2,\"c\":1,\"A.C.D.a\":3}")
-
- val ts1 = read[StructInstance](
- """
- {"$typeName$":"D","A.C.D.d":3,"A.B.D.c":2,"b":true,"A.C.D.c":3,"d":1,
- "A.B.D.b":true,"a":1,"A.C.D.b":false,"A.B.D.d":2,"c":1,"A.C.D.a":3}""")
- // Typed Struct read from string:
- Assert.assertEquals(ts1.toString, "{\n\td : \t1\n\tb : \ttrue\n\tc : \t1\n\ta : \t1\n\tA.B.D.b : \ttrue\n\tA.B.D.c : \t2\n\tA.B.D.d : \t2\n\tA.C.D.a : \t3\n\tA.C.D.b : \tfalse\n\tA.C.D.c : \t3\n\tA.C.D.d : \t3\n}")
- }
-
- def defineHRTypes(ts: TypeSystem) : Unit = {
- val deptTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
- "Department",
- ImmutableList.of[String],
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department"))
- val personTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
- "Person", ImmutableList.of[String],
- TypesUtil.createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates"))
- val managerTypeDef: HierarchicalTypeDefinition[ClassType] = TypesUtil.createClassTypeDef(
- "Manager", ImmutableList.of[String]("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager"))
- val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] =
- TypesUtil.createTraitTypeDef("SecurityClearance", ImmutableList.of[String],
- TypesUtil.createRequiredAttrDef("level", DataTypes.INT_TYPE))
-
- ts.defineTypes(ImmutableList.of[StructTypeDefinition],
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef)
- )
-
- }
-
- def defineHRDept() : Referenceable = {
- val hrDept: Referenceable = new Referenceable("Department")
- val john: Referenceable = new Referenceable("Person")
- val jane: Referenceable = new Referenceable("Manager", "SecurityClearance")
- hrDept.set("name", "hr")
- john.set("name", "John")
- john.set("department", hrDept.getId)
- jane.set("name", "Jane")
- jane.set("department", hrDept.getId)
- john.set("manager", jane.getId)
- hrDept.set("employees", ImmutableList.of[Referenceable](john, jane))
- jane.set("subordinates", ImmutableList.of[Id](john.getId))
- jane.getTrait("SecurityClearance").set("level", 1)
- hrDept
- }
-
- @Test def testClass {
-
- val ts: TypeSystem = getTypeSystem
- defineHRTypes(ts)
- val hrDept: Referenceable = defineHRDept()
-
- val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
- val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
-
- println(s"HR Dept Object Graph:\n${hrDept2}\n")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- val ser = swrite(hrDept2)
- println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
- println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
- }
-
- @Test def testReference {
-
- val ts: TypeSystem = getTypeSystem
- defineHRTypes(ts)
- val hrDept: Referenceable = defineHRDept()
-
-
- val jsonStr = InstanceSerialization.toJson(hrDept)
- val hrDept2 = InstanceSerialization.fromJsonReferenceable(jsonStr)
-
- val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
- val hrDept3: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
-
- println(s"HR Dept Object Graph:\n${hrDept3}\n")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- val ser = swrite(hrDept3)
- println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
- println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
- }
-
- @Test def testReference2 {
-
- val ts: TypeSystem = getTypeSystem
- defineHRTypes(ts)
- val hrDept: Referenceable = defineHRDept()
-
- val deptType: ClassType = ts.getDataType(classOf[ClassType], "Department")
- val hrDept2: ITypedReferenceableInstance = deptType.convert(hrDept, Multiplicity.REQUIRED)
-
- val jsonStr = InstanceSerialization.toJson(hrDept2)
- val hrDept3 = InstanceSerialization.fromJsonReferenceable(jsonStr)
-
- val hrDept4: ITypedReferenceableInstance = deptType.convert(hrDept2, Multiplicity.REQUIRED)
-
- println(s"HR Dept Object Graph:\n${hrDept4}\n")
-
- implicit val formats = org.json4s.native.Serialization.formats(NoTypeHints) + new TypedStructSerializer +
- new TypedReferenceableInstanceSerializer + new BigDecimalSerializer + new BigIntegerSerializer
-
- val ser = swrite(hrDept4)
- println(s"HR Dept JSON:\n${pretty(render(parse(ser)))}\n")
-
- println(s"HR Dept Object Graph read from JSON:${read[ReferenceableInstance](ser)}\n")
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
----------------------------------------------------------------------
diff --git a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala b/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
deleted file mode 100755
index 7b4e44d..0000000
--- a/typesystem/src/test/scala/org/apache/hadoop/metadata/typesystem/json/TypesSerializationTest.scala
+++ /dev/null
@@ -1,300 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.metadata.typesystem.json
-
-import com.google.common.collect.ImmutableList
-import org.apache.hadoop.metadata.typesystem.types._
-import org.junit.{Assert, Test}
-
-class TypesSerializationTest extends BaseTest with TypeHelpers {
-
- @Test def test1: Unit = {
-
- val ts = getTypeSystem
-
- val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
- optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
- ts.defineTypes(ImmutableList.of[StructTypeDefinition](sDef),
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]],
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]]
- )
-
- val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE))
- val B: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
- val C: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
- val D: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
-
- defineTraits(ts, A, B, C, D)
-
- ts.defineEnumType("HiveObjectType",
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5))
-
- ts.defineEnumType("PrincipalType",
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3))
-
- ts.defineEnumType("TxnState",
- new EnumValue("COMMITTED", 1),
- new EnumValue("ABORTED", 2),
- new EnumValue("OPEN", 3))
-
- ts.defineEnumType("LockLevel",
- new EnumValue("DB", 1),
- new EnumValue("TABLE", 2),
- new EnumValue("PARTITION", 3))
-
- defineClassType(ts, createClassTypeDef("t4", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("enum1", ts.getDataType(classOf[EnumType], "HiveObjectType")),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("enum2", ts.getDataType(classOf[EnumType], "PrincipalType")),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("enum3", ts.getDataType(classOf[EnumType], "TxnState")),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", ts.defineArrayType(DataTypes.INT_TYPE)),
- optionalAttr("n", ts.defineArrayType(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", ts.defineMapType(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- optionalAttr("enum4", ts.getDataType(classOf[EnumType], "LockLevel"))))
-
- val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department"))
- val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- )
- val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager")
- )
- val securityClearanceTypeDef: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("SecurityClearance", List(),
- requiredAttr("level", DataTypes.INT_TYPE)
- )
- ts.defineTypes(ImmutableList.of[StructTypeDefinition],
- ImmutableList.of[HierarchicalTypeDefinition[TraitType]](securityClearanceTypeDef),
- ImmutableList.of[HierarchicalTypeDefinition[ClassType]](deptTypeDef, personTypeDef, managerTypeDef))
-
- val ser = TypesSerialization.toJson(ts, _ => true)
-
- val typesDef1 = TypesSerialization.fromJson(ser)
-
- val ts1 = TypeSystem.getInstance()
- ts1.reset()
-
- typesDef1.enumTypes.foreach(ts1.defineEnumType(_))
-
- ts1.defineTypes(ImmutableList.copyOf(typesDef1.structTypes.toArray),
- ImmutableList.copyOf(typesDef1.traitTypes.toArray),
- ImmutableList.copyOf(typesDef1.classTypes.toArray)
- )
- val ser2 = TypesSerialization.toJson(ts1, _ => true)
- val typesDef2 = TypesSerialization.fromJson(ser2)
-
- Assert.assertEquals(typesDef1, typesDef2)
- }
-
- @Test def test2: Unit = {
-
- val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
- optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-
- val ser2 = TypesSerialization.toJson(sDef)
- val typesDef2 = TypesSerialization.fromJson(ser2)
-
- Assert.assertEquals(sDef, typesDef2.structTypes(0))
- }
-
- @Test def test3: Unit = {
-
- val sDef = structDef("ts1", requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.arrayTypeName(DataTypes.INT_TYPE)),
- optionalAttr("n", DataTypes.arrayTypeName(DataTypes.BIGDECIMAL_TYPE)),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)))
-
-
-
- val ser2 = TypesSerialization.toJson(sDef)
- val typesDef2 = TypesSerialization.fromJson(ser2)
-
- Assert.assertEquals(sDef, typesDef2.structTypes(0))
- }
-
- @Test def test4 : Unit = {
-
- val A: HierarchicalTypeDefinition[TraitType] = createTraitTypeDef("A", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE))
- val B: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("B", Seq("A"), optionalAttr("b", DataTypes.BOOLEAN_TYPE))
- val C: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("C", Seq("A"), optionalAttr("c", DataTypes.BYTE_TYPE))
- val D: HierarchicalTypeDefinition[TraitType] =
- createTraitTypeDef("D", Seq("B", "C"), optionalAttr("d", DataTypes.SHORT_TYPE))
-
- val typDefs = Seq(A,B,C,D)
- typDefs.foreach { tDef =>
- val ser2 = TypesSerialization.toJson(tDef, true)
- val typesDef2 = TypesSerialization.fromJson(ser2)
- Assert.assertEquals(tDef, typesDef2.traitTypes(0))
-
- }
- }
-
- @Test def test5 : Unit = {
- val e1 = new EnumTypeDefinition("HiveObjectType",
- new EnumValue("GLOBAL", 1),
- new EnumValue("DATABASE", 2),
- new EnumValue("TABLE", 3),
- new EnumValue("PARTITION", 4),
- new EnumValue("COLUMN", 5))
-
- val e2 = new EnumTypeDefinition("PrincipalType",
- new EnumValue("USER", 1),
- new EnumValue("ROLE", 2),
- new EnumValue("GROUP", 3))
-
- val e3 = new EnumTypeDefinition("TxnState",
- new EnumValue("COMMITTED", 1),
- new EnumValue("ABORTED", 2),
- new EnumValue("OPEN", 3))
-
- val e4 = new EnumTypeDefinition("LockLevel",
- new EnumValue("DB", 1),
- new EnumValue("TABLE", 2),
- new EnumValue("PARTITION", 3))
-
- val typDefs = Seq(e1,e2,e3,e4)
- typDefs.foreach { tDef =>
- val ser2 = TypesSerialization.toJson(tDef)
- val typesDef2 = TypesSerialization.fromJson(ser2)
- Assert.assertEquals(tDef, typesDef2.enumTypes(0))
-
- }
- }
-
- @Test def test6 : Unit = {
- val typDef = createClassTypeDef("t4", List(),
- requiredAttr("a", DataTypes.INT_TYPE),
- optionalAttr("b", DataTypes.BOOLEAN_TYPE),
- optionalAttr("c", DataTypes.BYTE_TYPE),
- optionalAttr("d", DataTypes.SHORT_TYPE),
- optionalAttr("enum1", "HiveObjectType"),
- optionalAttr("e", DataTypes.INT_TYPE),
- optionalAttr("f", DataTypes.INT_TYPE),
- optionalAttr("g", DataTypes.LONG_TYPE),
- optionalAttr("enum2", "PrincipalType"),
- optionalAttr("h", DataTypes.FLOAT_TYPE),
- optionalAttr("i", DataTypes.DOUBLE_TYPE),
- optionalAttr("j", DataTypes.BIGINTEGER_TYPE),
- optionalAttr("k", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("enum3", "TxnState"),
- optionalAttr("l", DataTypes.DATE_TYPE),
- optionalAttr("m", DataTypes.INT_TYPE),
- optionalAttr("n", DataTypes.BIGDECIMAL_TYPE),
- optionalAttr("o", DataTypes.mapTypeName(DataTypes.STRING_TYPE, DataTypes.DOUBLE_TYPE)),
- optionalAttr("enum4", "LockLevel"))
-
- val deptTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Department", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("employees", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, true, "department"))
- val personTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Person", List(),
- requiredAttr("name", DataTypes.STRING_TYPE),
- new AttributeDefinition("department", "Department", Multiplicity.REQUIRED, false, "employees"),
- new AttributeDefinition("manager", "Manager", Multiplicity.OPTIONAL, false, "subordinates")
- )
- val managerTypeDef: HierarchicalTypeDefinition[ClassType] = createClassTypeDef("Manager", List("Person"),
- new AttributeDefinition("subordinates", String.format("array<%s>", "Person"),
- Multiplicity.COLLECTION, false, "manager")
- )
-
- val typDefs = Seq(typDef, deptTypeDef, personTypeDef, managerTypeDef)
- typDefs.foreach { tDef =>
- val ser2 = TypesSerialization.toJson(tDef, false)
- val typesDef2 = TypesSerialization.fromJson(ser2)
- Assert.assertEquals(tDef, typesDef2.classTypes(0))
-
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/Main.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/Main.java b/webapp/src/main/java/org/apache/atlas/Main.java
new file mode 100755
index 0000000..76ca965
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/Main.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas;
+
+import org.apache.atlas.web.service.EmbeddedServer;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+
+/**
+ * Driver for running Metadata as a standalone server with embedded jetty server.
+ */
+public final class Main {
+ private static final Logger LOG = LoggerFactory.getLogger(Main.class);
+ private static final String APP_PATH = "app";
+ private static final String APP_PORT = "port";
+ private static final String METADATA_HOME = "atlas.home";
+ private static final String METADATA_LOG_DIR = "atlas.log.dir";
+
+ /**
+ * Prevent users from constructing this.
+ */
+ private Main() {
+ }
+
+ private static CommandLine parseArgs(String[] args) throws ParseException {
+ Options options = new Options();
+ Option opt;
+
+ opt = new Option(APP_PATH, true, "Application Path");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ opt = new Option(APP_PORT, true, "Application Port");
+ opt.setRequired(false);
+ options.addOption(opt);
+
+ return new GnuParser().parse(options, args);
+ }
+
+ public static void main(String[] args) throws Exception {
+ CommandLine cmd = parseArgs(args);
+ PropertiesConfiguration buildConfiguration =
+ new PropertiesConfiguration("atlas-buildinfo.properties");
+ String appPath = "webapp/target/atlas-webapp-" + getProjectVersion(buildConfiguration);
+
+ if (cmd.hasOption(APP_PATH)) {
+ appPath = cmd.getOptionValue(APP_PATH);
+ }
+
+ setApplicationHome();
+ PropertiesConfiguration configuration = PropertiesUtil.getApplicationProperties();
+ final String enableTLSFlag = configuration.getString("atlas.enableTLS");
+ final int appPort = getApplicationPort(cmd, enableTLSFlag);
+ final boolean enableTLS = isTLSEnabled(enableTLSFlag, appPort);
+ configuration.setProperty("atlas.enableTLS", String.valueOf(enableTLS));
+
+ showStartupInfo(buildConfiguration, enableTLS, appPort);
+ EmbeddedServer server = EmbeddedServer.newServer(appPort, appPath, enableTLS);
+ server.start();
+ }
+
+ private static void setApplicationHome() {
+ if (System.getProperty(METADATA_HOME) == null) {
+ System.setProperty(METADATA_HOME, "target");
+ }
+ if (System.getProperty(METADATA_LOG_DIR) == null) {
+ System.setProperty(METADATA_LOG_DIR, "target/logs");
+ }
+ }
+
+ public static String getProjectVersion(PropertiesConfiguration buildConfiguration) {
+ return buildConfiguration.getString("project.version");
+ }
+
+ private static int getApplicationPort(CommandLine cmd, String enableTLSFlag) {
+ final int appPort;
+ if (cmd.hasOption(APP_PORT)) {
+ appPort = Integer.valueOf(cmd.getOptionValue(APP_PORT));
+ } else {
+ // default : metadata.enableTLS is true
+ appPort = StringUtils.isEmpty(enableTLSFlag)
+ || enableTLSFlag.equals("true") ? 21443 : 21000;
+ }
+
+ return appPort;
+ }
+
+ private static boolean isTLSEnabled(String enableTLSFlag, int appPort) {
+ return Boolean.valueOf(StringUtils.isEmpty(enableTLSFlag)
+ ? System
+ .getProperty("atlas.enableTLS", (appPort % 1000) == 443 ? "true" : "false")
+ : enableTLSFlag);
+ }
+
+ private static void showStartupInfo(PropertiesConfiguration buildConfiguration,
+ boolean enableTLS, int appPort) {
+ StringBuilder buffer = new StringBuilder();
+ buffer.append("\n############################################");
+ buffer.append("############################################");
+ buffer.append("\n DGI Server (STARTUP)");
+ buffer.append("\n");
+ try {
+ final Iterator<String> keys = buildConfiguration.getKeys();
+ while (keys.hasNext()) {
+ String key = keys.next();
+ buffer.append('\n').append('\t').append(key).
+ append(":\t").append(buildConfiguration.getProperty(key));
+ }
+ } catch (Throwable e) {
+ buffer.append("*** Unable to get build info ***");
+ }
+ buffer.append("\n############################################");
+ buffer.append("############################################");
+ LOG.info(buffer.toString());
+ LOG.info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ LOG.info("Server starting with TLS ? {} on port {}", enableTLS, appPort);
+ LOG.info("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/30711973/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
new file mode 100755
index 0000000..4c0ffe8
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/examples/QuickStart.java
@@ -0,0 +1,478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.examples;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.MetadataServiceClient;
+import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+
+/**
+ * A driver that sets up sample types and data for testing purposes.
+ * Please take a look at QueryDSL in docs for the Meta Model.
+ * todo - move this to examples module.
+ */
+public class QuickStart {
+
+ public static void main(String[] args) throws Exception {
+ String baseUrl = getServerUrl(args);
+ QuickStart quickStart = new QuickStart(baseUrl);
+
+ // Shows how to create types in DGI for your meta model
+ quickStart.createTypes();
+
+ // Shows how to create entities (instances) for the added types in DGI
+ quickStart.createEntities();
+
+ // Shows some search queries using DSL based on types
+ quickStart.search();
+ }
+
+ static String getServerUrl(String[] args) {
+ String baseUrl = "http://localhost:21000";
+ if (args.length > 0) {
+ baseUrl = args[0];
+ }
+
+ return baseUrl;
+ }
+
+ private static final String DATABASE_TYPE = "DB";
+ private static final String COLUMN_TYPE = "Column";
+ private static final String TABLE_TYPE = "Table";
+ private static final String VIEW_TYPE = "View";
+ private static final String LOAD_PROCESS_TYPE = "LoadProcess";
+ private static final String STORAGE_DESC_TYPE = "StorageDesc";
+
+ private static final String[] TYPES = {
+ DATABASE_TYPE, TABLE_TYPE, STORAGE_DESC_TYPE, COLUMN_TYPE, LOAD_PROCESS_TYPE, VIEW_TYPE,
+ "JdbcAccess", "ETL", "Metric", "PII", "Fact", "Dimension"
+ };
+
+ private final MetadataServiceClient metadataServiceClient;
+
+ QuickStart(String baseUrl) {
+ metadataServiceClient = new MetadataServiceClient(baseUrl);
+ }
+
+ void createTypes() throws Exception {
+ TypesDef typesDef = createTypeDefinitions();
+
+ String typesAsJSON = TypesSerialization.toJson(typesDef);
+ System.out.println("typesAsJSON = " + typesAsJSON);
+ metadataServiceClient.createType(typesAsJSON);
+
+ // verify types created
+ verifyTypesCreated();
+ }
+
+ TypesDef createTypeDefinitions() throws Exception {
+ HierarchicalTypeDefinition<ClassType> dbClsDef
+ = TypesUtil.createClassTypeDef(DATABASE_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("description", DataTypes.STRING_TYPE),
+ attrDef("locationUri", DataTypes.STRING_TYPE),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> storageDescClsDef =
+ TypesUtil.createClassTypeDef(STORAGE_DESC_TYPE, null,
+ attrDef("location", DataTypes.STRING_TYPE),
+ attrDef("inputFormat", DataTypes.STRING_TYPE),
+ attrDef("outputFormat", DataTypes.STRING_TYPE),
+ attrDef("compressed", DataTypes.STRING_TYPE,
+ Multiplicity.REQUIRED, false, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> columnClsDef =
+ TypesUtil.createClassTypeDef(COLUMN_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ attrDef("dataType", DataTypes.STRING_TYPE),
+ attrDef("comment", DataTypes.STRING_TYPE)
+ );
+
+ HierarchicalTypeDefinition<ClassType> tblClsDef =
+ TypesUtil.createClassTypeDef(TABLE_TYPE, ImmutableList.of("DataSet"),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("sd", STORAGE_DESC_TYPE,
+ Multiplicity.REQUIRED, true, null),
+ attrDef("owner", DataTypes.STRING_TYPE),
+ attrDef("createTime", DataTypes.INT_TYPE),
+ attrDef("lastAccessTime", DataTypes.INT_TYPE),
+ attrDef("retention", DataTypes.INT_TYPE),
+ attrDef("viewOriginalText", DataTypes.STRING_TYPE),
+ attrDef("viewExpandedText", DataTypes.STRING_TYPE),
+ attrDef("tableType", DataTypes.STRING_TYPE),
+ attrDef("temporary", DataTypes.BOOLEAN_TYPE),
+ new AttributeDefinition("columns",
+ DataTypes.arrayTypeName(COLUMN_TYPE),
+ Multiplicity.COLLECTION, true, null)
+ );
+
+ HierarchicalTypeDefinition<ClassType> loadProcessClsDef =
+ TypesUtil.createClassTypeDef(LOAD_PROCESS_TYPE, ImmutableList.of("Process"),
+ attrDef("userName", DataTypes.STRING_TYPE),
+ attrDef("startTime", DataTypes.INT_TYPE),
+ attrDef("endTime", DataTypes.INT_TYPE),
+ attrDef("queryText", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryPlan", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryId", DataTypes.STRING_TYPE, Multiplicity.REQUIRED),
+ attrDef("queryGraph", DataTypes.STRING_TYPE, Multiplicity.REQUIRED)
+ );
+
+ HierarchicalTypeDefinition<ClassType> viewClsDef =
+ TypesUtil.createClassTypeDef(VIEW_TYPE, null,
+ attrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("db", DATABASE_TYPE,
+ Multiplicity.REQUIRED, false, null),
+ new AttributeDefinition("inputTables",
+ DataTypes.arrayTypeName(TABLE_TYPE),
+ Multiplicity.COLLECTION, false, null)
+ );
+
+ HierarchicalTypeDefinition<TraitType> dimTraitDef =
+ TypesUtil.createTraitTypeDef("Dimension", null);
+
+ HierarchicalTypeDefinition<TraitType> factTraitDef =
+ TypesUtil.createTraitTypeDef("Fact", null);
+
+ HierarchicalTypeDefinition<TraitType> piiTraitDef =
+ TypesUtil.createTraitTypeDef("PII", null);
+
+ HierarchicalTypeDefinition<TraitType> metricTraitDef =
+ TypesUtil.createTraitTypeDef("Metric", null);
+
+ HierarchicalTypeDefinition<TraitType> etlTraitDef =
+ TypesUtil.createTraitTypeDef("ETL", null);
+
+ HierarchicalTypeDefinition<TraitType> jdbcTraitDef =
+ TypesUtil.createTraitTypeDef("JdbcAccess", null);
+
+ return TypeUtils.getTypesDef(
+ ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of(dimTraitDef, factTraitDef,
+ piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
+ ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef,
+ tblClsDef, loadProcessClsDef, viewClsDef)
+ );
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT) {
+ return attrDef(name, dT, Multiplicity.OPTIONAL, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT, Multiplicity m) {
+ return attrDef(name, dT, m, false, null);
+ }
+
+ AttributeDefinition attrDef(String name, IDataType dT,
+ Multiplicity m, boolean isComposite, String reverseAttributeName) {
+ Preconditions.checkNotNull(name);
+ Preconditions.checkNotNull(dT);
+ return new AttributeDefinition(name, dT.getName(), m, isComposite, reverseAttributeName);
+ }
+
+ void createEntities() throws Exception {
+ Id salesDB = database(
+ "Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
+
+
+ Referenceable sd = rawStorageDescriptor("hdfs://host:8000/apps/warehouse/sales",
+ "TextInputFormat", "TextOutputFormat", true);
+
+ List<Referenceable> salesFactColumns = ImmutableList.of(
+ rawColumn("time_id", "int", "time id"),
+ rawColumn("product_id", "int", "product id"),
+ rawColumn("customer_id", "int", "customer id", "PII"),
+ rawColumn("sales", "double", "product id", "Metric")
+ );
+
+ Id salesFact = table("sales_fact", "sales fact table",
+ salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
+
+ List<Referenceable> productDimColumns = ImmutableList.of(
+ rawColumn("product_id", "int", "product id"),
+ rawColumn("product_name", "string", "product name"),
+ rawColumn("brand_name", "int", "brand name")
+ );
+
+ Id productDim = table("product_dim", "product dimension table",
+ salesDB, sd, "John Doe", "Managed", productDimColumns, "Dimension");
+
+ List<Referenceable> timeDimColumns = ImmutableList.of(
+ rawColumn("time_id", "int", "time id"),
+ rawColumn("dayOfYear", "int", "day Of Year"),
+ rawColumn("weekDay", "int", "week Day")
+ );
+
+ Id timeDim = table("time_dim", "time dimension table",
+ salesDB, sd, "John Doe", "External", timeDimColumns, "Dimension");
+
+
+ List<Referenceable> customerDimColumns = ImmutableList.of(
+ rawColumn("customer_id", "int", "customer id", "PII"),
+ rawColumn("name", "string", "customer name", "PII"),
+ rawColumn("address", "string", "customer address", "PII")
+ );
+
+ Id customerDim = table("customer_dim", "customer dimension table",
+ salesDB, sd, "fetl", "External", customerDimColumns, "Dimension");
+
+
+ Id reportingDB = database("Reporting", "reporting database", "Jane BI",
+ "hdfs://host:8000/apps/warehouse/reporting");
+
+ Id salesFactDaily = table("sales_fact_daily_mv",
+ "sales fact daily materialized view", reportingDB, sd,
+ "Joe BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesDaily", "hive query for daily summary", "John ETL",
+ ImmutableList.of(salesFact, timeDim), ImmutableList.of(salesFactDaily),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+
+ view("product_dim_view", reportingDB,
+ ImmutableList.of(productDim), "Dimension", "JdbcAccess");
+
+ view("customer_dim_view", reportingDB,
+ ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
+
+ Id salesFactMonthly = table("sales_fact_monthly_mv",
+ "sales fact monthly materialized view",
+ reportingDB, sd, "Jane BI", "Managed", salesFactColumns, "Metric");
+
+ loadProcess("loadSalesMonthly", "hive query for monthly summary", "John ETL",
+ ImmutableList.of(salesFactDaily), ImmutableList.of(salesFactMonthly),
+ "create table as select ", "plan", "id", "graph",
+ "ETL");
+ }
+
+ private Id createInstance(Referenceable referenceable) throws Exception {
+ String typeName = referenceable.getTypeName();
+
+ String entityJSON = InstanceSerialization.toJson(referenceable, true);
+ System.out.println("Submitting new entity= " + entityJSON);
+ JSONObject jsonObject = metadataServiceClient.createEntity(entityJSON);
+ String guid = jsonObject.getString(MetadataServiceClient.GUID);
+ System.out.println("created instance for type " + typeName + ", guid: " + guid);
+
+ // return the Id for created instance with guid
+ return new Id(guid, referenceable.getId().getVersion(), referenceable.getTypeName());
+ }
+
+ Id database(String name, String description,
+ String owner, String locationUri,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(DATABASE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("locationUri", locationUri);
+ referenceable.set("createTime", System.currentTimeMillis());
+
+ return createInstance(referenceable);
+ }
+
+ Referenceable rawStorageDescriptor(String location, String inputFormat,
+ String outputFormat,
+ boolean compressed) throws Exception {
+ Referenceable referenceable = new Referenceable(STORAGE_DESC_TYPE);
+ referenceable.set("location", location);
+ referenceable.set("inputFormat", inputFormat);
+ referenceable.set("outputFormat", outputFormat);
+ referenceable.set("compressed", compressed);
+
+ return referenceable;
+ }
+
+ Referenceable rawColumn(String name, String dataType, String comment,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(COLUMN_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("dataType", dataType);
+ referenceable.set("comment", comment);
+
+ return referenceable;
+ }
+
+ Id table(String name, String description,
+ Id dbId, Referenceable sd,
+ String owner, String tableType,
+ List<Referenceable> columns,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(TABLE_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("owner", owner);
+ referenceable.set("tableType", tableType);
+ referenceable.set("createTime", System.currentTimeMillis());
+ referenceable.set("lastAccessTime", System.currentTimeMillis());
+ referenceable.set("retention", System.currentTimeMillis());
+ referenceable.set("db", dbId);
+ referenceable.set("sd", sd);
+ referenceable.set("columns", columns);
+
+ return createInstance(referenceable);
+ }
+
+ Id loadProcess(String name, String description, String user,
+ List<Id> inputTables,
+ List<Id> outputTables,
+ String queryText, String queryPlan,
+ String queryId, String queryGraph,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(LOAD_PROCESS_TYPE, traitNames);
+ // super type attributes
+ referenceable.set("name", name);
+ referenceable.set("description", description);
+ referenceable.set("inputs", inputTables);
+ referenceable.set("outputs", outputTables);
+
+ referenceable.set("user", user);
+ referenceable.set("startTime", System.currentTimeMillis());
+ referenceable.set("endTime", System.currentTimeMillis() + 10000);
+
+ referenceable.set("queryText", queryText);
+ referenceable.set("queryPlan", queryPlan);
+ referenceable.set("queryId", queryId);
+ referenceable.set("queryGraph", queryGraph);
+
+ return createInstance(referenceable);
+ }
+
+ Id view(String name, Id dbId,
+ List<Id> inputTables,
+ String... traitNames) throws Exception {
+ Referenceable referenceable = new Referenceable(VIEW_TYPE, traitNames);
+ referenceable.set("name", name);
+ referenceable.set("db", dbId);
+
+ referenceable.set("inputTables", inputTables);
+
+ return createInstance(referenceable);
+ }
+
+ private void verifyTypesCreated() throws Exception {
+ List<String> types = metadataServiceClient.listTypes();
+ for (String type : TYPES) {
+ assert types.contains(type);
+ }
+ }
+
+ private String[] getDSLQueries() {
+ return new String[]{
+ "from DB",
+ "DB",
+ "DB where name=\"Reporting\"",
+ "DB where DB.name=\"Reporting\"",
+ "DB name = \"Reporting\"",
+ "DB DB.name = \"Reporting\"",
+ "DB where name=\"Reporting\" select name, owner",
+ "DB where DB.name=\"Reporting\" select name, owner",
+ "DB has name",
+ "DB where DB has name",
+ "DB, Table",
+ "DB is JdbcAccess",
+ /*
+ "DB, hive_process has name",
+ "DB as db1, Table where db1.name = \"Reporting\"",
+ "DB where DB.name=\"Reporting\" and DB.createTime < " + System.currentTimeMillis()},
+ */
+ "from Table",
+ "Table",
+ "Table is Dimension",
+ "Column where Column isa PII",
+ "View is Dimension",
+ /*"Column where Column isa PII select Column.name",*/
+ "Column select Column.name",
+ "Column select name",
+ "Column where Column.name=\"customer_id\"",
+ "from Table select Table.name",
+ "DB where (name = \"Reporting\")",
+ "DB where (name = \"Reporting\") select name as _col_0, owner as _col_1",
+ "DB where DB is JdbcAccess",
+ "DB where DB has name",
+ "DB Table",
+ "DB where DB has name",
+ "DB as db1 Table where (db1.name = \"Reporting\")",
+ "DB where (name = \"Reporting\") select name as _col_0, (createTime + 1) as _col_1 ",
+ /*
+ todo: does not work
+ "DB where (name = \"Reporting\") and ((createTime + 1) > 0)",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) or (db1.name = \"Reporting\") select db1.name as dbName, tab.name as tabName",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
+ "DB as db1 Table as tab where ((db1.createTime + 1) > 0) and (db1.name = \"Reporting\") or db1 has owner select db1.name as dbName, tab.name as tabName",
+ */
+ // trait searches
+ "Dimension",
+ /*"Fact", - todo: does not work*/
+ "JdbcAccess",
+ "ETL",
+ "Metric",
+ "PII",
+ /*
+ // Lineage - todo - fix this, its not working
+ "Table hive_process outputTables",
+ "Table loop (hive_process outputTables)",
+ "Table as _loop0 loop (hive_process outputTables) withPath",
+ "Table as src loop (hive_process outputTables) as dest select src.name as srcTable, dest.name as destTable withPath",
+ */
+ "Table where name=\"sales_fact\", columns",
+ "Table where name=\"sales_fact\", columns as column select column.name, column.dataType, column.comment",
+ "from DataSet",
+ "from Process",
+ };
+ }
+
+ private void search() throws Exception {
+ for (String dslQuery : getDSLQueries()) {
+ JSONObject response = metadataServiceClient.searchEntity(dslQuery);
+ JSONObject results = response.getJSONObject(MetadataServiceClient.RESULTS);
+ if (!results.isNull("rows")) {
+ JSONArray rows = results.getJSONArray("rows");
+ System.out.println("query [" + dslQuery + "] returned [" + rows.length() + "] rows");
+ } else {
+ System.out.println("query [" + dslQuery + "] failed, results:" + results.toString());
+ }
+ }
+ }
+}